Make dnsmasq daemon set optional

Change additional dnsmasq opts:
- Adjust caching size and TTL
- Disable resolve conf to not create loops
- Change dnsPolicy to default (similarly to kubedns's dnsmasq). The
  ClusterFirst should not be used to not create loops
- Disable negative NXDOMAIN replies to be cached
- Make its very installation as optional step (enabled by default).
  If you don't want more than 3 DNS servers, including 1 for K8s, disable
  it.
- Add docs and a drawing to clarify DNS setup.
- Fix stdout logs for dnsmasq/kubedns app configs
- Add missed notifies to resolvconf -u handler
- Fix idempotency of resolvconf head file changes

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
This commit is contained in:
Bogdan Dobrelya 2016-09-22 19:14:51 +02:00
parent 73a8c24089
commit 82ee60fe8b
11 changed files with 199 additions and 145 deletions

21
docs/dns-stack.md Normal file
View file

@ -0,0 +1,21 @@
K8s DNS stack by Kargo
======================
Here is an approximate picture of how DNS things working and
being configured by Kargo ansible playbooks:
![Image](figures/dns.png?raw=true)
Note that an additional dnsmasq daemon set is installed by Kargo
by default. Kubelet will configure DNS base of all pods to use that
dnsmasq cluster IP. You can disable it with the ``skip_dnsmasq``
var. This may be the case, if you're fine with Linux limit of max 3
nameservers in the ``/etc/resolv.conf``. When skipped and bypassed
directly to Kubedns's dnsmasq cluster IP, it greatly simplifies things
by the price of limited nameservers though.
Nameservers are configured in the hosts' ``/etc/resolv.conf`` files
from the ``nameservers`` (see also ``searchdomains``) vars. While the
``upstream_dns_servers`` will define additional DNS servers for the
dnsmasq daemon set running on all hosts (unless bypassed with
``skip_dnsmasq``).

BIN
docs/figures/dns.png Normal file

Binary file not shown.

View file

@ -103,6 +103,8 @@ kube_apiserver_insecure_port: 8080 # (http)
# You still must manually configure all your containers to use this DNS server, # You still must manually configure all your containers to use this DNS server,
# Kubernetes won't do this for you (yet). # Kubernetes won't do this for you (yet).
# Do not install additional dnsmasq
skip_dnsmasq: false
# Upstream dns servers used by dnsmasq # Upstream dns servers used by dnsmasq
upstream_dns_servers: upstream_dns_servers:
- 8.8.8.8 - 8.8.8.8

View file

@ -0,0 +1,58 @@
---
- name: ensure dnsmasq.d directory exists
file:
path: /etc/dnsmasq.d
state: directory
- name: ensure dnsmasq.d-available directory exists
file:
path: /etc/dnsmasq.d-available
state: directory
- name: Write dnsmasq configuration
template:
src: 01-kube-dns.conf.j2
dest: /etc/dnsmasq.d-available/01-kube-dns.conf
mode: 0755
backup: yes
- name: Stat dnsmasq configuration
stat: path=/etc/dnsmasq.d/01-kube-dns.conf
register: sym
- name: Move previous configuration
command: mv /etc/dnsmasq.d/01-kube-dns.conf /etc/dnsmasq.d-available/01-kube-dns.conf.bak
changed_when: False
when: sym.stat.islnk is defined and sym.stat.islnk == False
- name: Enable dnsmasq configuration
file:
src: /etc/dnsmasq.d-available/01-kube-dns.conf
dest: /etc/dnsmasq.d/01-kube-dns.conf
state: link
- name: Create dnsmasq manifests
template: src={{item.file}} dest=/etc/kubernetes/{{item.file}}
with_items:
- {file: dnsmasq-ds.yml, type: ds}
- {file: dnsmasq-svc.yml, type: svc}
register: manifests
when: inventory_hostname == groups['kube-master'][0]
- name: Start Resources
kube:
name: dnsmasq
namespace: kube-system
kubectl: "{{bin_dir}}/kubectl"
resource: "{{item.item.type}}"
filename: /etc/kubernetes/{{item.item.file}}
state: "{{item.changed | ternary('latest','present') }}"
with_items: "{{ manifests.results }}"
when: inventory_hostname == groups['kube-master'][0]
- name: Check for dnsmasq port (pulling image and running container)
wait_for:
host: "{{dns_server}}"
port: 53
delay: 5
when: inventory_hostname == groups['kube-node'][0]

View file

@ -1,134 +1,5 @@
--- ---
- name: ensure dnsmasq.d directory exists - include: dnsmasq.yml
file: when: "{{ not skip_dnsmasq|bool }}"
path: /etc/dnsmasq.d
state: directory
- name: ensure dnsmasq.d-available directory exists - include: resolvconf.yml
file:
path: /etc/dnsmasq.d-available
state: directory
- name: Write dnsmasq configuration
template:
src: 01-kube-dns.conf.j2
dest: /etc/dnsmasq.d-available/01-kube-dns.conf
mode: 0755
backup: yes
- name: Stat dnsmasq configuration
stat: path=/etc/dnsmasq.d/01-kube-dns.conf
register: sym
- name: Move previous configuration
command: mv /etc/dnsmasq.d/01-kube-dns.conf /etc/dnsmasq.d-available/01-kube-dns.conf.bak
changed_when: False
when: sym.stat.islnk is defined and sym.stat.islnk == False
- name: Enable dnsmasq configuration
file:
src: /etc/dnsmasq.d-available/01-kube-dns.conf
dest: /etc/dnsmasq.d/01-kube-dns.conf
state: link
- name: Create dnsmasq manifests
template: src={{item.file}} dest=/etc/kubernetes/{{item.file}}
with_items:
- {file: dnsmasq-ds.yml, type: ds}
- {file: dnsmasq-svc.yml, type: svc}
register: manifests
when: inventory_hostname == groups['kube-master'][0]
- name: Start Resources
kube:
name: dnsmasq
namespace: kube-system
kubectl: "{{bin_dir}}/kubectl"
resource: "{{item.item.type}}"
filename: /etc/kubernetes/{{item.item.file}}
state: "{{item.changed | ternary('latest','present') }}"
with_items: "{{ manifests.results }}"
when: inventory_hostname == groups['kube-master'][0]
- name: Check for dnsmasq port (pulling image and running container)
wait_for:
host: "{{dns_server}}"
port: 53
delay: 5
when: inventory_hostname == groups['kube-node'][0]
- name: check resolvconf
shell: which resolvconf
register: resolvconf
ignore_errors: yes
- name: target resolv.conf file
set_fact:
resolvconffile: >-
{%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
- name: generate search domains to resolvconf
set_fact:
searchentries="{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
- name: generate nameservers to resolvconf
set_fact:
nameserverentries="{{ nameservers|default([]) + [ dns_server ] }}"
- name: Add search resolv.conf
lineinfile:
line: "search {{searchentries}}"
dest: "{{resolvconffile}}"
state: present
insertbefore: BOF
backup: yes
follow: yes
- name: Add local dnsmasq to resolv.conf
blockinfile:
dest: "{{resolvconffile}}"
block: |-
{% for item in nameserverentries -%}
nameserver {{ item }}
{% endfor %}
state: present
create: yes
backup: yes
follow: yes
marker: "# Ansible nameservers {mark}"
- name: Add options to resolv.conf
lineinfile:
line: options {{ item }}
dest: "{{resolvconffile}}"
state: present
regexp: "^options.*{{ item }}$"
insertafter: EOF
backup: yes
follow: yes
with_items:
- timeout:2
- attempts:2
- name: Remove search and nameserver options from resolvconf base
lineinfile:
dest: /etc/resolvconf/resolv.conf.d/base
state: absent
regexp: "^{{ item }}.*$"
backup: yes
follow: yes
with_items:
- search
- nameserver
when: resolvconf.rc == 0
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
notify: Dnsmasq | restart network
when: ansible_os_family == "Debian"
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
notify: Dnsmasq | restart network
when: ansible_os_family == "RedHat"

View file

@ -0,0 +1,100 @@
---
- name: check resolvconf
shell: which resolvconf
register: resolvconf
ignore_errors: yes
- name: target resolv.conf file
set_fact:
resolvconffile: >-
{%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
- name: generate search domains to resolvconf
set_fact:
searchentries:
"{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
- name: pick dnsmasq cluster IP
set_fact:
dnsmasq_server: >-
{%- if skip_dnsmasq|bool -%}{{ [ skydns_server ] + upstream_dns_servers|default([]) }}{%- else -%}{{ [ dns_server ] }}{%- endif -%}
- name: generate nameservers to resolvconf
set_fact:
nameserverentries:
"{{ nameservers|default([]) + dnsmasq_server|default([]) }}"
- name: Remove search and nameserver options from resolvconf head
lineinfile:
dest: /etc/resolvconf/resolv.conf.d/head
state: absent
regexp: "^{{ item }}.*$"
backup: yes
follow: yes
with_items:
- search
- nameserver
when: resolvconf.rc == 0
notify: Dnsmasq | update resolvconf
- name: Add search resolv.conf
lineinfile:
line: "search {{searchentries}}"
dest: "{{resolvconffile}}"
state: present
insertbefore: BOF
backup: yes
follow: yes
notify: Dnsmasq | update resolvconf
- name: Add local dnsmasq to resolv.conf
blockinfile:
dest: "{{resolvconffile}}"
block: |-
{% for item in nameserverentries -%}
nameserver {{ item }}
{% endfor %}
state: present
insertafter: "^search.*$"
create: yes
backup: yes
follow: yes
marker: "# Ansible nameservers {mark}"
notify: Dnsmasq | update resolvconf
- name: Add options to resolv.conf
lineinfile:
line: options {{ item }}
dest: "{{resolvconffile}}"
state: present
regexp: "^options.*{{ item }}$"
insertafter: EOF
backup: yes
follow: yes
with_items:
- timeout:2
- attempts:2
notify: Dnsmasq | update resolvconf
- name: Remove search and nameserver options from resolvconf base
lineinfile:
dest: /etc/resolvconf/resolv.conf.d/base
state: absent
regexp: "^{{ item }}.*$"
backup: yes
follow: yes
with_items:
- search
- nameserver
when: resolvconf.rc == 0
notify: Dnsmasq | update resolvconf
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
notify: Dnsmasq | restart network
when: ansible_os_family == "Debian"
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
notify: Dnsmasq | restart network
when: ansible_os_family == "RedHat"

View file

@ -4,6 +4,10 @@ listen-address=0.0.0.0
addn-hosts=/etc/hosts addn-hosts=/etc/hosts
strict-order
# Forward k8s domain to kube-dns
server=/{{ dns_domain }}/{{ skydns_server }}
#Set upstream dns servers #Set upstream dns servers
{% if upstream_dns_servers is defined %} {% if upstream_dns_servers is defined %}
{% for srv in upstream_dns_servers %} {% for srv in upstream_dns_servers %}
@ -14,8 +18,9 @@ server={{ srv }}
server=8.8.4.4 server=8.8.4.4
{% endif %} {% endif %}
# Forward k8s domain to kube-dns no-resolv
server=/{{ dns_domain }}/{{ skydns_server }} no-negcache
cache-size=1000
# Forward reverse lookups for k8s service addresses to kube-dns max-cache-ttl=10
rev-server={{ kube_service_addresses }},{{ skydns_server }} max-ttl=20
log-facility=-

View file

@ -50,3 +50,4 @@ spec:
- name: etcdnsmasqdavailable - name: etcdnsmasqdavailable
hostPath: hostPath:
path: /etc/dnsmasq.d-available path: /etc/dnsmasq.d-available
dnsPolicy: Default # Don't use cluster DNS.

View file

@ -65,6 +65,7 @@ spec:
- name: dnsmasq - name: dnsmasq
image: gcr.io/google_containers/kube-dnsmasq-amd64:1.3 image: gcr.io/google_containers/kube-dnsmasq-amd64:1.3
args: args:
- --log-facility=-
- --cache-size=1000 - --cache-size=1000
- --no-resolv - --no-resolv
- --server=127.0.0.1#10053 - --server=127.0.0.1#10053

View file

@ -8,13 +8,6 @@ kube_resolv_conf: "/etc/resolv.conf"
kube_proxy_mode: iptables kube_proxy_mode: iptables
# IP address of the DNS server.
# Kubernetes will create a pod with several containers, serving as the DNS
# server and expose it under this IP address. The IP address must be from
# the range specified as kube_service_addresses. This magic will actually
# pick the 10th ip address in the kube_service_addresses range and use that.
dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}"
# kube_api_runtime_config: # kube_api_runtime_config:
# - extensions/v1beta1/daemonsets=true # - extensions/v1beta1/daemonsets=true
# - extensions/v1beta1/deployments=true # - extensions/v1beta1/deployments=true

View file

@ -19,7 +19,9 @@ KUBELET_HOSTNAME="--hostname-override={{ inventory_hostname }}"
KUBELET_REGISTER_NODE="--register-node=false" KUBELET_REGISTER_NODE="--register-node=false"
{% endif %} {% endif %}
# location of the api-server # location of the api-server
{% if dns_setup %} {% if dns_setup|bool and skip_dnsmasq|bool %}
KUBELET_ARGS="--cluster_dns={{ skydns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}"
{% elif dns_setup|bool %}
KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}" KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}"
{% else %} {% else %}
KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}" KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"