C12s/c12s-kubespray
3
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Make dnsmasq daemon set optional

Browse source 
Change additional dnsmasq opts:
- Adjust caching size and TTL
- Disable resolve conf to not create loops
- Change dnsPolicy to default (similarly to kubedns's dnsmasq). The
  ClusterFirst should not be used to not create loops
- Disable negative NXDOMAIN replies to be cached
- Make its very installation as optional step (enabled by default).
  If you don't want more than 3 DNS servers, including 1 for K8s, disable
  it.
- Add docs and a drawing to clarify DNS setup.
- Fix stdout logs for dnsmasq/kubedns app configs
- Add missed notifies to resolvconf -u handler
- Fix idempotency of resolvconf head file changes

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
This commit is contained in:
Bogdan Dobrelya 2016-09-22 19:14:51 +02:00
parent 73a8c24089
commit 82ee60fe8b
11 changed files with 199 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
docs/dns-stack.md Normal file
View file

@ -0,0 +1,21 @@
K8s DNS stack by Kargo
======================
Here is an approximate picture of how DNS things working and
being configured by Kargo ansible playbooks:
![Image](figures/dns.png?raw=true)
Note that an additional dnsmasq daemon set is installed by Kargo
by default. Kubelet will configure DNS base of all pods to use that
dnsmasq cluster IP. You can disable it with the ``skip_dnsmasq``
var. This may be the case, if you're fine with Linux limit of max 3
nameservers in the ``/etc/resolv.conf``. When skipped and bypassed
directly to Kubedns's dnsmasq cluster IP, it greatly simplifies things
by the price of limited nameservers though.
Nameservers are configured in the hosts' ``/etc/resolv.conf`` files
from the ``nameservers`` (see also ``searchdomains``) vars. While the
``upstream_dns_servers`` will define additional DNS servers for the
dnsmasq daemon set running on all hosts (unless bypassed with
``skip_dnsmasq``).

BIN
docs/figures/dns.png Normal file
View file

Binary file not shown.

2
inventory/group_vars/all.yml
View file

@ -103,6 +103,8 @@ kube_apiserver_insecure_port: 8080 # (http)
# You still must manually configure all your containers to use this DNS server,
# Kubernetes won't do this for you (yet).
# Do not install additional dnsmasq
skip_dnsmasq: false
# Upstream dns servers used by dnsmasq
upstream_dns_servers:
- 8.8.8.8

58
roles/dnsmasq/tasks/dnsmasq.yml Normal file
View file

@ -0,0 +1,58 @@
---
- name: ensure dnsmasq.d directory exists
file:
path: /etc/dnsmasq.d
state: directory
- name: ensure dnsmasq.d-available directory exists
file:
path: /etc/dnsmasq.d-available
state: directory
- name: Write dnsmasq configuration
template:
src: 01-kube-dns.conf.j2
dest: /etc/dnsmasq.d-available/01-kube-dns.conf
mode: 0755
backup: yes
- name: Stat dnsmasq configuration
stat: path=/etc/dnsmasq.d/01-kube-dns.conf
register: sym
- name: Move previous configuration
command: mv /etc/dnsmasq.d/01-kube-dns.conf /etc/dnsmasq.d-available/01-kube-dns.conf.bak
changed_when: False
when: sym.stat.islnk is defined and sym.stat.islnk == False
- name: Enable dnsmasq configuration
file:
src: /etc/dnsmasq.d-available/01-kube-dns.conf
dest: /etc/dnsmasq.d/01-kube-dns.conf
state: link
- name: Create dnsmasq manifests
template: src={{item.file}} dest=/etc/kubernetes/{{item.file}}
with_items:
- {file: dnsmasq-ds.yml, type: ds}
- {file: dnsmasq-svc.yml, type: svc}
register: manifests
when: inventory_hostname == groups['kube-master'][0]
- name: Start Resources
kube:
name: dnsmasq
namespace: kube-system
kubectl: "{{bin_dir}}/kubectl"
resource: "{{item.item.type}}"
filename: /etc/kubernetes/{{item.item.file}}
state: "{{item.changed | ternary('latest','present') }}"
with_items: "{{ manifests.results }}"
when: inventory_hostname == groups['kube-master'][0]
- name: Check for dnsmasq port (pulling image and running container)
wait_for:
host: "{{dns_server}}"
port: 53
delay: 5
when: inventory_hostname == groups['kube-node'][0]

135
roles/dnsmasq/tasks/main.yml
View file

@ -1,134 +1,5 @@
---
- name: ensure dnsmasq.d directory exists
file:
path: /etc/dnsmasq.d
state: directory
- include: dnsmasq.yml
when: "{{ not skip_dnsmasq|bool }}"
- name: ensure dnsmasq.d-available directory exists
file:
path: /etc/dnsmasq.d-available
state: directory
- name: Write dnsmasq configuration
template:
src: 01-kube-dns.conf.j2
dest: /etc/dnsmasq.d-available/01-kube-dns.conf
mode: 0755
backup: yes
- name: Stat dnsmasq configuration
stat: path=/etc/dnsmasq.d/01-kube-dns.conf
register: sym
- name: Move previous configuration
command: mv /etc/dnsmasq.d/01-kube-dns.conf /etc/dnsmasq.d-available/01-kube-dns.conf.bak
changed_when: False
when: sym.stat.islnk is defined and sym.stat.islnk == False
- name: Enable dnsmasq configuration
file:
src: /etc/dnsmasq.d-available/01-kube-dns.conf
dest: /etc/dnsmasq.d/01-kube-dns.conf
state: link
- name: Create dnsmasq manifests
template: src={{item.file}} dest=/etc/kubernetes/{{item.file}}
with_items:
- {file: dnsmasq-ds.yml, type: ds}
- {file: dnsmasq-svc.yml, type: svc}
register: manifests
when: inventory_hostname == groups['kube-master'][0]
- name: Start Resources
kube:
name: dnsmasq
namespace: kube-system
kubectl: "{{bin_dir}}/kubectl"
resource: "{{item.item.type}}"
filename: /etc/kubernetes/{{item.item.file}}
state: "{{item.changed | ternary('latest','present') }}"
with_items: "{{ manifests.results }}"
when: inventory_hostname == groups['kube-master'][0]
- name: Check for dnsmasq port (pulling image and running container)
wait_for:
host: "{{dns_server}}"
port: 53
delay: 5
when: inventory_hostname == groups['kube-node'][0]
- name: check resolvconf
shell: which resolvconf
register: resolvconf
ignore_errors: yes
- name: target resolv.conf file
set_fact:
resolvconffile: >-
{%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
- name: generate search domains to resolvconf
set_fact:
searchentries="{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
- name: generate nameservers to resolvconf
set_fact:
nameserverentries="{{ nameservers|default([]) + [ dns_server ] }}"
- name: Add search resolv.conf
lineinfile:
line: "search {{searchentries}}"
dest: "{{resolvconffile}}"
state: present
insertbefore: BOF
backup: yes
follow: yes
- name: Add local dnsmasq to resolv.conf
blockinfile:
dest: "{{resolvconffile}}"
block: |-
{% for item in nameserverentries -%}
nameserver {{ item }}
{% endfor %}
state: present
create: yes
backup: yes
follow: yes
marker: "# Ansible nameservers {mark}"
- name: Add options to resolv.conf
lineinfile:
line: options {{ item }}
dest: "{{resolvconffile}}"
state: present
regexp: "^options.*{{ item }}$"
insertafter: EOF
backup: yes
follow: yes
with_items:
- timeout:2
- attempts:2
- name: Remove search and nameserver options from resolvconf base
lineinfile:
dest: /etc/resolvconf/resolv.conf.d/base
state: absent
regexp: "^{{ item }}.*$"
backup: yes
follow: yes
with_items:
- search
- nameserver
when: resolvconf.rc == 0
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
notify: Dnsmasq | restart network
when: ansible_os_family == "Debian"
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
notify: Dnsmasq | restart network
when: ansible_os_family == "RedHat"
- include: resolvconf.yml

100
roles/dnsmasq/tasks/resolvconf.yml Normal file
View file

@ -0,0 +1,100 @@
---
- name: check resolvconf
shell: which resolvconf
register: resolvconf
ignore_errors: yes
- name: target resolv.conf file
set_fact:
resolvconffile: >-
{%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
- name: generate search domains to resolvconf
set_fact:
searchentries:
"{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
- name: pick dnsmasq cluster IP
set_fact:
dnsmasq_server: >-
{%- if skip_dnsmasq|bool -%}{{ [ skydns_server ] + upstream_dns_servers|default([]) }}{%- else -%}{{ [ dns_server ] }}{%- endif -%}
- name: generate nameservers to resolvconf
set_fact:
nameserverentries:
"{{ nameservers|default([]) + dnsmasq_server|default([]) }}"
- name: Remove search and nameserver options from resolvconf head
lineinfile:
dest: /etc/resolvconf/resolv.conf.d/head
state: absent
regexp: "^{{ item }}.*$"
backup: yes
follow: yes
with_items:
- search
- nameserver
when: resolvconf.rc == 0
notify: Dnsmasq | update resolvconf
- name: Add search resolv.conf
lineinfile:
line: "search {{searchentries}}"
dest: "{{resolvconffile}}"
state: present
insertbefore: BOF
backup: yes
follow: yes
notify: Dnsmasq | update resolvconf
- name: Add local dnsmasq to resolv.conf
blockinfile:
dest: "{{resolvconffile}}"
block: |-
{% for item in nameserverentries -%}
nameserver {{ item }}
{% endfor %}
state: present
insertafter: "^search.*$"
create: yes
backup: yes
follow: yes
marker: "# Ansible nameservers {mark}"
notify: Dnsmasq | update resolvconf
- name: Add options to resolv.conf
lineinfile:
line: options {{ item }}
dest: "{{resolvconffile}}"
state: present
regexp: "^options.*{{ item }}$"
insertafter: EOF
backup: yes
follow: yes
with_items:
- timeout:2
- attempts:2
notify: Dnsmasq | update resolvconf
- name: Remove search and nameserver options from resolvconf base
lineinfile:
dest: /etc/resolvconf/resolv.conf.d/base
state: absent
regexp: "^{{ item }}.*$"
backup: yes
follow: yes
with_items:
- search
- nameserver
when: resolvconf.rc == 0
notify: Dnsmasq | update resolvconf
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
notify: Dnsmasq | restart network
when: ansible_os_family == "Debian"
- name: disable resolv.conf modification by dhclient
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
notify: Dnsmasq | restart network
when: ansible_os_family == "RedHat"

15
roles/dnsmasq/templates/01-kube-dns.conf.j2
View file

@ -4,6 +4,10 @@ listen-address=0.0.0.0
addn-hosts=/etc/hosts
strict-order
# Forward k8s domain to kube-dns
server=/{{ dns_domain }}/{{ skydns_server }}
#Set upstream dns servers
{% if upstream_dns_servers is defined %}
{% for srv in upstream_dns_servers %}
@ -14,8 +18,9 @@ server={{ srv }}
server=8.8.4.4
{% endif %}
# Forward k8s domain to kube-dns
server=/{{ dns_domain }}/{{ skydns_server }}
# Forward reverse lookups for k8s service addresses to kube-dns
rev-server={{ kube_service_addresses }},{{ skydns_server }}
no-resolv
no-negcache
cache-size=1000
max-cache-ttl=10
max-ttl=20
log-facility=-

1
roles/dnsmasq/templates/dnsmasq-ds.yml
View file

@ -50,3 +50,4 @@ spec:
- name: etcdnsmasqdavailable
hostPath:
path: /etc/dnsmasq.d-available
dnsPolicy: Default # Don't use cluster DNS.

1
roles/kubernetes-apps/ansible/templates/kubedns-rc.yml
View file

@ -65,6 +65,7 @@ spec:
- name: dnsmasq
image: gcr.io/google_containers/kube-dnsmasq-amd64:1.3
args:
- --log-facility=-
- --cache-size=1000
- --no-resolv
- --server=127.0.0.1#10053

7
roles/kubernetes/node/defaults/main.yml
View file

@ -8,13 +8,6 @@ kube_resolv_conf: "/etc/resolv.conf"
kube_proxy_mode: iptables
# IP address of the DNS server.
# Kubernetes will create a pod with several containers, serving as the DNS
# server and expose it under this IP address. The IP address must be from
# the range specified as kube_service_addresses. This magic will actually
# pick the 10th ip address in the kube_service_addresses range and use that.
dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}"
# kube_api_runtime_config:
# - extensions/v1beta1/daemonsets=true
# - extensions/v1beta1/deployments=true

4
roles/kubernetes/node/templates/kubelet.j2
View file

@ -19,7 +19,9 @@ KUBELET_HOSTNAME="--hostname-override={{ inventory_hostname }}"
KUBELET_REGISTER_NODE="--register-node=false"
{% endif %}
# location of the api-server
{% if dns_setup %}
{% if dns_setup|bool and skip_dnsmasq|bool %}
KUBELET_ARGS="--cluster_dns={{ skydns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}"
{% elif dns_setup|bool %}
KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}"
{% else %}
KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"