Make dnsmasq daemon set optional

Change additional dnsmasq opts:
- Adjust caching size and TTL
- Disable resolve conf to not create loops
- Change dnsPolicy to default (similarly to kubedns's dnsmasq). The
  ClusterFirst should not be used to not create loops
- Disable negative NXDOMAIN replies to be cached
- Make its very installation as optional step (enabled by default).
  If you don't want more than 3 DNS servers, including 1 for K8s, disable
  it.
- Add docs and a drawing to clarify DNS setup.
- Fix stdout logs for dnsmasq/kubedns app configs
- Add missed notifies to resolvconf -u handler
- Fix idempotency of resolvconf head file changes

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

docs/dns-stack.md

@@ -0,0 +1,21 @@
+K8s DNS stack by Kargo
+======================
+
+Here is an approximate picture of how DNS things working and
+being configured by Kargo ansible playbooks:
+
+![Image](figures/dns.png?raw=true)
+
+Note that an additional dnsmasq daemon set is installed by Kargo
+by default. Kubelet will configure DNS base of all pods to use that
+dnsmasq cluster IP. You can disable it with the ``skip_dnsmasq``
+var. This may be the case, if you're fine with Linux limit of max 3
+nameservers in the ``/etc/resolv.conf``. When skipped and bypassed
+directly to Kubedns's dnsmasq cluster IP, it greatly simplifies things
+by the price of limited nameservers though.
+
+Nameservers are configured in the hosts' ``/etc/resolv.conf`` files
+from the ``nameservers`` (see also ``searchdomains``) vars. While the
+``upstream_dns_servers`` will define additional DNS servers for the
+dnsmasq daemon set running on all hosts (unless bypassed with
+``skip_dnsmasq``).

inventory/group_vars/all.yml

@@ -103,6 +103,8 @@ kube_apiserver_insecure_port: 8080 # (http)
 # You still must manually configure all your containers to use this DNS server,
 # Kubernetes won't do this for you (yet).
 
+# Do not install additional dnsmasq
+skip_dnsmasq: false
 # Upstream dns servers used by dnsmasq
 upstream_dns_servers:
   - 8.8.8.8

roles/dnsmasq/tasks/dnsmasq.yml

@@ -0,0 +1,58 @@
+---
+- name: ensure dnsmasq.d directory exists
+  file:
+    path: /etc/dnsmasq.d
+    state: directory
+
+- name: ensure dnsmasq.d-available directory exists
+  file:
+    path: /etc/dnsmasq.d-available
+    state: directory
+
+- name: Write dnsmasq configuration
+  template:
+    src: 01-kube-dns.conf.j2
+    dest: /etc/dnsmasq.d-available/01-kube-dns.conf
+    mode: 0755
+    backup: yes
+
+- name: Stat dnsmasq configuration
+  stat: path=/etc/dnsmasq.d/01-kube-dns.conf
+  register: sym
+
+- name: Move previous configuration
+  command: mv /etc/dnsmasq.d/01-kube-dns.conf /etc/dnsmasq.d-available/01-kube-dns.conf.bak
+  changed_when: False
+  when: sym.stat.islnk is defined and sym.stat.islnk == False
+
+- name: Enable dnsmasq configuration
+  file:
+    src: /etc/dnsmasq.d-available/01-kube-dns.conf
+    dest: /etc/dnsmasq.d/01-kube-dns.conf
+    state: link
+
+- name: Create dnsmasq manifests
+  template: src={{item.file}} dest=/etc/kubernetes/{{item.file}}
+  with_items:
+    - {file: dnsmasq-ds.yml, type: ds}
+    - {file: dnsmasq-svc.yml, type: svc}
+  register: manifests
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: Start Resources
+  kube:
+    name: dnsmasq
+    namespace: kube-system
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "{{item.item.type}}"
+    filename: /etc/kubernetes/{{item.item.file}}
+    state: "{{item.changed | ternary('latest','present') }}"
+  with_items: "{{ manifests.results }}"
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: Check for dnsmasq port (pulling image and running container)
+  wait_for:
+    host: "{{dns_server}}"
+    port: 53
+    delay: 5
+  when: inventory_hostname == groups['kube-node'][0]

roles/dnsmasq/tasks/main.yml

@@ -1,134 +1,5 @@
 ---
-- name: ensure dnsmasq.d directory exists
-  file:
-    path: /etc/dnsmasq.d
-    state: directory
+- include: dnsmasq.yml
+  when: "{{ not skip_dnsmasq|bool }}"
 
-- name: ensure dnsmasq.d-available directory exists
-  file:
-    path: /etc/dnsmasq.d-available
-    state: directory
-
-- name: Write dnsmasq configuration
-  template:
-    src: 01-kube-dns.conf.j2
-    dest: /etc/dnsmasq.d-available/01-kube-dns.conf
-    mode: 0755
-    backup: yes
-
-- name: Stat dnsmasq configuration
-  stat: path=/etc/dnsmasq.d/01-kube-dns.conf
-  register: sym
-
-- name: Move previous configuration
-  command: mv /etc/dnsmasq.d/01-kube-dns.conf /etc/dnsmasq.d-available/01-kube-dns.conf.bak
-  changed_when: False
-  when: sym.stat.islnk is defined and sym.stat.islnk == False
-
-- name: Enable dnsmasq configuration
-  file:
-    src: /etc/dnsmasq.d-available/01-kube-dns.conf
-    dest: /etc/dnsmasq.d/01-kube-dns.conf
-    state: link
-
-- name: Create dnsmasq manifests
-  template: src={{item.file}} dest=/etc/kubernetes/{{item.file}}
-  with_items:
-    - {file: dnsmasq-ds.yml, type: ds}
-    - {file: dnsmasq-svc.yml, type: svc}
-  register: manifests
-  when: inventory_hostname == groups['kube-master'][0]
-
-- name: Start Resources
-  kube:
-    name: dnsmasq
-    namespace: kube-system
-    kubectl: "{{bin_dir}}/kubectl"
-    resource: "{{item.item.type}}"
-    filename: /etc/kubernetes/{{item.item.file}}
-    state: "{{item.changed | ternary('latest','present') }}"
-  with_items: "{{ manifests.results }}"
-  when: inventory_hostname == groups['kube-master'][0]
-
-- name: Check for dnsmasq port (pulling image and running container)
-  wait_for:
-    host: "{{dns_server}}"
-    port: 53
-    delay: 5
-  when: inventory_hostname == groups['kube-node'][0]
-
-
-- name: check resolvconf
-  shell: which resolvconf
-  register: resolvconf
-  ignore_errors: yes
-
-- name: target resolv.conf file
-  set_fact:
-    resolvconffile: >-
-      {%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
-
-- name: generate search domains to resolvconf
-  set_fact:
-    searchentries="{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
-
-- name: generate nameservers to resolvconf
-  set_fact:
-    nameserverentries="{{ nameservers|default([]) + [ dns_server ] }}"
-
-- name: Add search resolv.conf
-  lineinfile:
-    line: "search {{searchentries}}"
-    dest: "{{resolvconffile}}"
-    state: present
-    insertbefore: BOF
-    backup: yes
-    follow: yes
-
-- name: Add local dnsmasq to resolv.conf
-  blockinfile:
-    dest: "{{resolvconffile}}"
-    block: |-
-      {% for item in nameserverentries -%}
-      nameserver {{ item }}
-      {% endfor %}
-    state: present
-    create: yes
-    backup: yes
-    follow: yes
-    marker: "# Ansible nameservers {mark}"
-
-- name: Add options to resolv.conf
-  lineinfile:
-    line: options {{ item }}
-    dest: "{{resolvconffile}}"
-    state: present
-    regexp: "^options.*{{ item }}$"
-    insertafter: EOF
-    backup: yes
-    follow: yes
-  with_items:
-    - timeout:2
-    - attempts:2
-
-- name: Remove search and nameserver options from resolvconf base
-  lineinfile:
-    dest: /etc/resolvconf/resolv.conf.d/base
-    state: absent
-    regexp: "^{{ item }}.*$"
-    backup: yes
-    follow: yes
-  with_items:
-    - search
-    - nameserver
-  when: resolvconf.rc == 0
-
-- name: disable resolv.conf modification by dhclient
-  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
-  notify: Dnsmasq | restart network
-  when: ansible_os_family == "Debian"
-
-- name: disable resolv.conf modification by dhclient
-  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
-  notify: Dnsmasq | restart network
-  when: ansible_os_family == "RedHat"
+- include: resolvconf.yml

roles/dnsmasq/tasks/resolvconf.yml

@@ -0,0 +1,100 @@
+---
+- name: check resolvconf
+  shell: which resolvconf
+  register: resolvconf
+  ignore_errors: yes
+
+- name: target resolv.conf file
+  set_fact:
+    resolvconffile: >-
+      {%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
+
+- name: generate search domains to resolvconf
+  set_fact:
+    searchentries:
+      "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
+
+- name: pick dnsmasq cluster IP
+  set_fact:
+    dnsmasq_server: >-
+      {%- if skip_dnsmasq|bool -%}{{ [ skydns_server ] + upstream_dns_servers|default([]) }}{%- else -%}{{ [ dns_server ] }}{%- endif -%}
+
+- name: generate nameservers to resolvconf
+  set_fact:
+    nameserverentries:
+      "{{ nameservers|default([]) + dnsmasq_server|default([]) }}"
+
+- name: Remove search and nameserver options from resolvconf head
+  lineinfile:
+    dest: /etc/resolvconf/resolv.conf.d/head
+    state: absent
+    regexp: "^{{ item }}.*$"
+    backup: yes
+    follow: yes
+  with_items:
+    - search
+    - nameserver
+  when: resolvconf.rc == 0
+  notify: Dnsmasq | update resolvconf
+
+- name: Add search resolv.conf
+  lineinfile:
+    line: "search {{searchentries}}"
+    dest: "{{resolvconffile}}"
+    state: present
+    insertbefore: BOF
+    backup: yes
+    follow: yes
+  notify: Dnsmasq | update resolvconf
+
+- name: Add local dnsmasq to resolv.conf
+  blockinfile:
+    dest: "{{resolvconffile}}"
+    block: |-
+      {% for item in nameserverentries -%}
+      nameserver {{ item }}
+      {% endfor %}
+    state: present
+    insertafter: "^search.*$"
+    create: yes
+    backup: yes
+    follow: yes
+    marker: "# Ansible nameservers {mark}"
+  notify: Dnsmasq | update resolvconf
+
+- name: Add options to resolv.conf
+  lineinfile:
+    line: options {{ item }}
+    dest: "{{resolvconffile}}"
+    state: present
+    regexp: "^options.*{{ item }}$"
+    insertafter: EOF
+    backup: yes
+    follow: yes
+  with_items:
+    - timeout:2
+    - attempts:2
+  notify: Dnsmasq | update resolvconf
+
+- name: Remove search and nameserver options from resolvconf base
+  lineinfile:
+    dest: /etc/resolvconf/resolv.conf.d/base
+    state: absent
+    regexp: "^{{ item }}.*$"
+    backup: yes
+    follow: yes
+  with_items:
+    - search
+    - nameserver
+  when: resolvconf.rc == 0
+  notify: Dnsmasq | update resolvconf
+
+- name: disable resolv.conf modification by dhclient
+  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
+  notify: Dnsmasq | restart network
+  when: ansible_os_family == "Debian"
+
+- name: disable resolv.conf modification by dhclient
+  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
+  notify: Dnsmasq | restart network
+  when: ansible_os_family == "RedHat"

roles/dnsmasq/templates/01-kube-dns.conf.j2

@@ -4,6 +4,10 @@ listen-address=0.0.0.0
 
 addn-hosts=/etc/hosts
 
+strict-order
+# Forward k8s domain to kube-dns
+server=/{{ dns_domain }}/{{ skydns_server }}
+
 #Set upstream dns servers
 {% if upstream_dns_servers is defined %}
 {% for srv in upstream_dns_servers %}
@@ -14,8 +18,9 @@ server={{ srv }}
  server=8.8.4.4
 {% endif %}
 
-# Forward k8s domain to kube-dns
-server=/{{ dns_domain }}/{{ skydns_server }}
-
-# Forward reverse lookups for k8s service addresses to kube-dns
-rev-server={{ kube_service_addresses }},{{ skydns_server }}
+no-resolv
+no-negcache
+cache-size=1000
+max-cache-ttl=10
+max-ttl=20
+log-facility=-

roles/dnsmasq/templates/dnsmasq-ds.yml

@@ -50,3 +50,4 @@ spec:
         - name: etcdnsmasqdavailable
           hostPath:
             path: /etc/dnsmasq.d-available
+      dnsPolicy: Default  # Don't use cluster DNS.

roles/kubernetes-apps/ansible/templates/kubedns-rc.yml

@@ -65,6 +65,7 @@ spec:
       - name: dnsmasq
         image: gcr.io/google_containers/kube-dnsmasq-amd64:1.3
         args:
+        - --log-facility=-
         - --cache-size=1000
         - --no-resolv
         - --server=127.0.0.1#10053

roles/kubernetes/node/defaults/main.yml

@@ -8,13 +8,6 @@ kube_resolv_conf: "/etc/resolv.conf"
 
 kube_proxy_mode: iptables
 
-# IP address of the DNS server.
-# Kubernetes will create a pod with several containers, serving as the DNS
-# server and expose it under this IP address. The IP address must be from
-# the range specified as kube_service_addresses. This magic will actually
-# pick the 10th ip address in the kube_service_addresses range and use that.
-dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}"
-
 # kube_api_runtime_config:
 #   - extensions/v1beta1/daemonsets=true
 #   - extensions/v1beta1/deployments=true

roles/kubernetes/node/templates/kubelet.j2

@@ -19,7 +19,9 @@ KUBELET_HOSTNAME="--hostname-override={{ inventory_hostname }}"
 KUBELET_REGISTER_NODE="--register-node=false"
 {% endif %}
 # location of the api-server
-{% if dns_setup %}
+{% if dns_setup|bool and skip_dnsmasq|bool %}
+KUBELET_ARGS="--cluster_dns={{ skydns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}"
+{% elif dns_setup|bool %}
 KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }}"
 {% else %}
 KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"