Merge pull request #504 from bogdando/rework_dnsstack

Make dnsmasq daemon set optional
This commit is contained in:
Smaine Kahlouch 2016-09-23 15:33:34 +02:00 committed by GitHub
commit 284354c0da
11 changed files with 199 additions and 145 deletions

21
docs/dns-stack.md Normal file
View file

@ -0,0 +1,21 @@
K8s DNS stack by Kargo
======================
Here is an approximate picture of how DNS things working and
being configured by Kargo ansible playbooks:
![Image](figures/dns.png?raw=true)
Note that an additional dnsmasq daemon set is installed by Kargo
by default. Kubelet will configure DNS base of all pods to use that
dnsmasq cluster IP. You can disable it with the ``skip_dnsmasq``
var. This may be the case, if you're fine with Linux limit of max 3
nameservers in the ``/etc/resolv.conf``. When skipped and bypassed
directly to Kubedns's dnsmasq cluster IP, it greatly simplifies things
by the price of limited nameservers though.
Nameservers are configured in the hosts' ``/etc/resolv.conf`` files
from the ``nameservers`` (see also ``searchdomains``) vars. While the
``upstream_dns_servers`` will define additional DNS servers for the
dnsmasq daemon set running on all hosts (unless bypassed with
``skip_dnsmasq``).

BIN
docs/figures/dns.png Normal file

Binary file not shown.

View file

@ -103,6 +103,8 @@ kube_apiserver_insecure_port: 8080 # (http)
# You still must manually configure all your containers to use this DNS server,
# Kubernetes won't do this for you (yet).
# Do not install additional dnsmasq
skip_dnsmasq: false
# Upstream dns servers used by dnsmasq
upstream_dns_servers:
- 8.8.8.8

View file

@ -0,0 +1,58 @@
---
- name: ensure dnsmasq.d directory exists
file:
path: /etc/dnsmasq.d
state: directory
- name: ensure dnsmasq.d-available directory exists
file:
path: /etc/dnsmasq.d-available
state: directory
- name: Write dnsmasq configuration
template:
src: 01-kube-dns.conf.j2
dest: /etc/dnsmasq.d-available/01-kube-dns.conf
mode: 0755
backup: yes
- name: Stat dnsmasq configuration
stat: path=/etc/dnsmasq.d/01-kube-dns.conf
register: sym
- name: Move previous configuration
command: mv /etc/dnsmasq.d/01-kube-dns.conf /etc/dnsmasq.d-available/01-kube-dns.conf.bak
changed_when: False
when: sym.stat.islnk is defined and sym.stat.islnk == False
- name: Enable dnsmasq configuration
file:
src: /etc/dnsmasq.d-available/01-kube-dns.conf
dest: /etc/dnsmasq.d/01-kube-dns.conf
state: link
- name: Create dnsmasq manifests
template: src={{item.file}} dest=/etc/kubernetes/{{item.file}}
with_items:
- {file: dnsmasq-ds.yml, type: ds}
- {file: dnsmasq-svc.yml, type: svc}
register: manifests
when: inventory_hostname == groups['kube-master'][0]
- name: Start Resources
kube:
name: dnsmasq
namespace: kube-system
kubectl: "{{bin_dir}}/kubectl"
resource: "{{item.item.type}}"
filename: /etc/kubernetes/{{item.item.file}}
state: "{{item.changed | ternary('latest','present') }}"
with_items: "{{ manifests.results }}"
when: inventory_hostname == groups['kube-master'][0]
- name: Check for dnsmasq port (pulling image and running container)
wait_for:
host: "{{dns_server}}"
port: 53
delay: 5
when: inventory_hostname == groups['kube-node'][0]

View file

@ -1,134 +1,5 @@
---
- name: ensure dnsmasq.d directory exists
file:
path: /etc/dnsmasq.d
state: directory
- include: dnsmasq.yml
when: "{{ not skip_dnsmasq|bool }}"
- name: ensure dnsmasq.d-available directory exists
file:
path: /etc/dnsmasq.d-available
state: directory
- name: Write dnsmasq configuration
template:
src: 01-kube-dns.conf.j2
dest: /etc/dnsmasq.d-available/01-kube-dns.conf
mode: 0755
backup: yes
- name: Stat dnsmasq configuration
stat: path=/etc/dnsmasq.d/01-kube-dns.conf
register: sym
- name: Move previous configuration
command: mv /etc/dnsmasq.d/01-kube-dns.conf /etc/dnsmasq.d-available/01-kube-dns.conf.bak
changed_when: False
when: sym.stat.islnk is defined and sym.stat.islnk == False
- name: Enable dnsmasq configuration
file:
src: /etc/dnsmasq.d-available/01-kube-dns.conf
dest: /etc/dnsmasq.d/01-kube-dns.conf
state: link
- name: Create dnsmasq manifests
template: src={{item.file}} dest=/etc/kubernetes/{{item.file}}
with_items:
- {file: dnsmasq-ds.yml, type: ds}
- {file: dnsmasq-svc.yml, type: svc}
register: manifests
when: inventory_hostname == groups['kube-master'][0]
- name: Start Resources
kube:
name: dnsmasq
namespace: kube-system
kubectl: "{{bin_dir}}/kubectl"
resource: "{{item.item.type}}"
filename: /etc/kubernetes/{{item.item.file}}
state: "{{item.changed | ternary('latest','present') }}"
with_items: "{{ manifests.results }}"
when: inventory_hostname == groups['kube-master'][0]
- name: Check for dnsmasq port (pulling image and running container)
wait_for:
host: "{{dns_server}}"
port: 53
delay: 5
when: inventory_hostname == groups['kube-node'][0]
- name: check resolvconf
shell: which resolvconf
register: resolvconf
ignore_errors: yes
- name: target resolv.conf file
set_fact:
resolvconffile: >-
{%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
- name: generate search domains to resolvconf
set_fact:
searchentries="{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
- name: generate nameservers to resolvconf
set_fact:
nameserverentries="{{ nameservers|default([]) + [ dns_server ] }}"
- name: Add search resolv.conf
lineinfile:
line: "search {{searchentries}}"
dest: "{{resolvconffile}}"
state: present
insertbefore: BOF
backup: yes
follow: yes
- name: Add local dnsmasq to resolv.conf
blockinfile:
dest: "{{resolvconffile}}"
block: |-
{% for item in nameserverentries -%}
nameserver {{ item }}
{% endfor %}
state: present
create: yes
backup: yes
follow: yes
marker: "# Ansible nameservers {mark}"
- name: Add options to resolv.conf
lineinfile:
line: options {{ item }}
dest: "{{resolvconffile}}"
state: present
regexp: "^options.*{{ item }}$"
insertafter: EOF
backup: yes
follow: yes
with_items:
- timeout:2
- attempts:2
- name: Remove search and nameserver options from resolvconf base
lineinfile:
dest: /etc/resolvconf/resolv.conf.d/base
state: absent
regexp: "^{{ item }}.*$"
backup: yes
follow: yes
with_items:
- search
- nameserver
when: resolvconf.rc == 0
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
notify: Dnsmasq | restart network
when: ansible_os_family == "Debian"
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
notify: Dnsmasq | restart network
when: ansible_os_family == "RedHat"
- include: resolvconf.yml

View file

@ -0,0 +1,100 @@
---
- name: check resolvconf
shell: which resolvconf
register: resolvconf
ignore_errors: yes
- name: target resolv.conf file
set_fact:
resolvconffile: >-
{%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
- name: generate search domains to resolvconf
set_fact:
searchentries:
"{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
- name: pick dnsmasq cluster IP
set_fact:
dnsmasq_server: >-
{%- if skip_dnsmasq|bool -%}{{ [ skydns_server ] + upstream_dns_servers|default([]) }}{%- else -%}{{ [ dns_server ] }}{%- endif -%}
- name: generate nameservers to resolvconf
set_fact:
nameserverentries:
"{{ nameservers|default([]) + dnsmasq_server|default([]) }}"
- name: Remove search and nameserver options from resolvconf head
lineinfile:
dest: /etc/resolvconf/resolv.conf.d/head
state: absent
regexp: "^{{ item }}.*$"
backup: yes
follow: yes
with_items:
- search
- nameserver
when: resolvconf.rc == 0
notify: Dnsmasq | update resolvconf
- name: Add search resolv.conf
lineinfile:
line: "search {{searchentries}}"
dest: "{{resolvconffile}}"
state: present
insertbefore: BOF
backup: yes
follow: yes
notify: Dnsmasq | update resolvconf
- name: Add local dnsmasq to resolv.conf
blockinfile:
dest: "{{resolvconffile}}"
block: |-
{% for item in nameserverentries -%}
nameserver {{ item }}
{% endfor %}
state: present
insertafter: "^search.*$"
create: yes
backup: yes
follow: yes
marker: "# Ansible nameservers {mark}"
notify: Dnsmasq | update resolvconf
- name: Add options to resolv.conf
lineinfile:
line: options {{ item }}
dest: "{{resolvconffile}}"
state: present
regexp: "^options.*{{ item }}$"
insertafter: EOF
backup: yes
follow: yes
with_items:
- timeout:2
- attempts:2
notify: Dnsmasq | update resolvconf
- name: Remove search and nameserver options from resolvconf base
lineinfile:
dest: /etc/resolvconf/resolv.conf.d/base
state: absent
regexp: "^{{ item }}.*$"
backup: yes
follow: yes
with_items:
- search
- nameserver
when: resolvconf.rc == 0
notify: Dnsmasq | update resolvconf
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
notify: Dnsmasq | restart network
when: ansible_os_family == "Debian"
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
notify: Dnsmasq | restart network
when: ansible_os_family == "RedHat"

View file

@ -4,6 +4,10 @@ listen-address=0.0.0.0
addn-hosts=/etc/hosts
strict-order
# Forward k8s domain to kube-dns
server=/{{ dns_domain }}/{{ skydns_server }}
#Set upstream dns servers
{% if upstream_dns_servers is defined %}
{% for srv in upstream_dns_servers %}
@ -14,8 +18,9 @@ server={{ srv }}
server=8.8.4.4
{% endif %}
# Forward k8s domain to kube-dns
server=/{{ dns_domain }}/{{ skydns_server }}
# Forward reverse lookups for k8s service addresses to kube-dns
rev-server={{ kube_service_addresses }},{{ skydns_server }}
no-resolv
no-negcache
cache-size=1000
max-cache-ttl=10
max-ttl=20
log-facility=-

View file

@ -50,3 +50,4 @@ spec:
- name: etcdnsmasqdavailable
hostPath:
path: /etc/dnsmasq.d-available
dnsPolicy: Default # Don't use cluster DNS.

View file

@ -65,6 +65,7 @@ spec:
- name: dnsmasq
image: gcr.io/google_containers/kube-dnsmasq-amd64:1.3
args:
- --log-facility=-
- --cache-size=1000
- --no-resolv
- --server=127.0.0.1#10053

View file

@ -8,13 +8,6 @@ kube_resolv_conf: "/etc/resolv.conf"
kube_proxy_mode: iptables
# IP address of the DNS server.
# Kubernetes will create a pod with several containers, serving as the DNS
# server and expose it under this IP address. The IP address must be from
# the range specified as kube_service_addresses. This magic will actually
# pick the 10th ip address in the kube_service_addresses range and use that.
dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}"
# kube_api_runtime_config:
# - extensions/v1beta1/daemonsets=true
# - extensions/v1beta1/deployments=true

View file

@ -19,7 +19,9 @@ KUBELET_HOSTNAME="--hostname-override={{ inventory_hostname }}"
KUBELET_REGISTER_NODE="--register-node=false"
{% endif %}
# location of the api-server
{% if dns_setup %}
{% if dns_setup|bool and skip_dnsmasq|bool %}
KUBELET_ARGS="--cluster_dns={{ skydns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}"
{% elif dns_setup|bool %}
KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}"
{% else %}
KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"