Add support for running a nodelocal dns cache (#3861)

* Add support for running a nodelocal dns cache

After encountering dns issues in a cluster I was recently working on I
noticed Kubernetes 1.13 introduced support for running a nodelocal dns
cache.

I believe this can usefull for more people.

73b548db06
https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/0030-nodelocal-dns-cache.md

* Add requested changes

* Add additional requested changes + documentation

* Add requested changes after review

* Replace incorrect variable

docs/dns-stack.md

@@ -84,6 +84,11 @@ leaves you with a non functional cluster.
 ``resolvconf_mode`` configures how Kubespray will setup DNS for ``hostNetwork: true`` PODs and non-k8s containers.
 There are three modes available:
 
+## Nodelocal DNS cache
+Setting ``enable_nodelocaldns`` to ``true`` will make pods reach out to the dns (core-dns) caching agent running on the same node, thereby avoiding iptables DNAT rules and connection tracking. The local caching agent will query kube-dns / core-dns (depending on what main DNS plugin is configured in your cluster) for cache misses of cluster hostnames(cluster.local suffix by default).
+
+More information on the rationale behind this implementation can be found [here](https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/0030-nodelocal-dns-cache.md).
+
 #### docker_dns (default)
 This sets up the docker daemon with additional --dns/--dns-search/--dns-opt flags.
 

inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml

@@ -114,6 +114,8 @@ ndots: 2
 dns_mode: coredns
 # Set manual server if using a custom cluster DNS server
 #manual_dns_server: 10.x.x.x
+# Enable nodelocal dns cache
+enable_nodelocaldns: False
 
 # Can be docker_dns, host_resolvconf or none
 resolvconf_mode: docker_dns

roles/download/defaults/main.yml

@@ -184,6 +184,10 @@ coredns_version: "1.2.6"
 coredns_image_repo: "coredns/coredns"
 coredns_image_tag: "{{ coredns_version }}"
 
+nodelocaldns_version: "1.15.0"
+nodelocaldns_image_repo: "k8s.gcr.io/k8s-dns-node-cache"
+nodelocaldns_image_tag: "{{ nodelocaldns_version }}"
+
 dnsmasq_nanny_image_repo: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-{{ image_arch }}"
 dnsmasq_nanny_image_tag: "{{ kubedns_version }}"
 dnsmasq_sidecar_image_repo: "gcr.io/google_containers/k8s-dns-sidecar-{{ image_arch }}"
@@ -490,6 +494,15 @@ downloads:
     groups:
       - kube-node
 
+  nodelocaldns:
+    enabled: "{{ enable_nodelocaldns == True }}"
+    container: true
+    repo: "{{ nodelocaldns_image_repo }}"
+    tag: "{{ nodelocaldns_image_tag }}"
+    sha256: "{{ nodelocaldns_digest_checksum|default(None) }}"
+    groups:
+      - kube-node
+
   dnsmasq_nanny:
     enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}"
     container: true

roles/kubernetes-apps/ansible/defaults/main.yml

@@ -8,6 +8,12 @@ dns_nodes_per_replica: 10
 dns_cores_per_replica: 20
 dns_prevent_single_point_failure: "{{ 'true' if dns_min_replicas|int > 1 else 'false' }}"
 
+# nodelocaldns
+nodelocaldns_cpu_requests: 100m
+nodelocaldns_ip: 169.254.25.10
+nodelocaldns_memory_limit: 170Mi
+nodelocaldnsdns_memory_requests: 70Mi
+
 # Netchecker
 deploy_netchecker: false
 netchecker_port: 31081

roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml

@@ -13,6 +13,19 @@
   tags:
     - upgrade
 
+- name: Kubernetes Apps | Delete old nodelocalDNS resources
+  kube:
+    name: "nodelocaldns"
+    namespace: "kube-system"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item }}"
+    state: absent
+  with_items:
+    - 'deamonset'
+    - 'configmap'
+  tags:
+    - upgrade
+
 - name: Kubernetes Apps | Delete kubeadm CoreDNS
   kube:
     name: "coredns"

roles/kubernetes-apps/ansible/tasks/main.yml

@@ -20,6 +20,7 @@
     - dnsmasq
     - coredns
     - kubedns
+    - nodelocaldns
 
 - name: Kubernetes Apps | CoreDNS
   import_tasks: "tasks/coredns.yml"
@@ -29,6 +30,14 @@
   tags:
     - coredns
 
+- name: Kubernetes Apps | nodelocalDNS
+  import_tasks: "tasks/nodelocaldns.yml"
+  when:
+    - enable_nodelocaldns == True
+    - inventory_hostname == groups['kube-master'] | first
+  tags:
+    - nodelocaldns
+
 - name: Kubernetes Apps | KubeDNS
   import_tasks: "tasks/kubedns.yml"
   when:
@@ -49,6 +58,7 @@
     - "{{ kubedns_manifests.results | default({}) }}"
     - "{{ coredns_manifests.results | default({}) }}"
     - "{{ coredns_secondary_manifests.results | default({}) }}"
+    - "{{ nodelocaldns_manifests.results | default({}) }}"
   when:
     - dns_mode != 'none'
     - inventory_hostname == groups['kube-master'][0]
@@ -61,6 +71,7 @@
     - dnsmasq
     - coredns
     - kubedns
+    - nodelocaldns
   loop_control:
     label: "{{ item.item.file }}"
 

roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml

@@ -0,0 +1,18 @@
+---
+- name: Kubernetes Apps | Lay Down nodelocaldns Template
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - { name: nodelocaldns, file: nodelocaldns-config.yml, type: configmap }
+    - { name: nodelocaldns, file: nodelocaldns-sa.yml, type: sa }
+    - { name: nodelocaldns, file: nodelocaldns-deamonset.yml, type: daemonset }
+  register: nodelocaldns_manifests
+  vars:
+    clusterIP: "{{ skydns_server }}"
+    secondaryclusterIP: "{{ skydns_server_secondary }}"
+  when:
+    - enable_nodelocaldns == True
+    - inventory_hostname == groups['kube-master'] | first
+  tags:
+    - nodelocaldns

roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2

@@ -0,0 +1,71 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nodelocaldns
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+
+data:
+  Corefile: |
+    {{ dns_domain }}:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ nodelocaldns_ip }}
+{% if secondaryclusterIP is defined and dns_mode == 'coredns_dual' %}
+        forward . {{ clusterIP }} {{ secondaryclusterIP }} {
+{% else %}
+        forward . {{ clusterIP }} {
+{% endif %}
+                force_tcp
+        }
+        prometheus :9253
+        health {{ nodelocaldns_ip }}:8080
+        }
+    in-addr.arpa:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ nodelocaldns_ip }}
+{% if secondaryclusterIP is defined %}
+        forward . {{ clusterIP }} {{ secondaryclusterIP }} {
+{% else %}
+        forward . {{ clusterIP }} {
+{% endif %}
+                force_tcp
+        }
+        prometheus :9253
+        }
+    ip6.arpa:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ nodelocaldns_ip }}
+{% if secondaryclusterIP is defined %}
+        forward . {{ clusterIP }} {{ secondaryclusterIP }} {
+{% else %}
+        forward . {{ clusterIP }} {
+{% endif %}
+             force_tcp
+        }
+        prometheus :9253
+        }
+    .:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ nodelocaldns_ip }}
+{% if resolvconf_mode == 'host_resolvconf' and upstream_dns_servers is defined and upstream_dns_servers|length > 0 %}
+        forward . {{ upstream_dns_servers|join(' ') }} {
+{% else %}
+        forward . /etc/resolv.conf {
+{% endif %}
+                force_tcp
+        }
+        prometheus :9253
+        }

roles/kubernetes-apps/ansible/templates/nodelocaldns-deamonset.yml.j2

@@ -0,0 +1,72 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nodelocaldns
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  selector:
+    matchLabels:
+      k8s-app: nodelocaldns
+  template:
+    metadata:
+       labels:
+          k8s-app: nodelocaldns
+    spec:
+{% if kube_version is version('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
+      serviceAccountName: nodelocaldns
+      hostNetwork: true
+      dnsPolicy: Default  # Don't use cluster DNS.
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      - key: "CriticalAddonsOnly"
+        operator: "Exists"
+      containers:
+      - name: node-cache
+        image: "{{ nodelocaldns_image_repo }}:{{ nodelocaldns_image_tag }}"
+        resources:
+          limits:
+            memory: {{ nodelocaldns_memory_limit }}
+          requests:
+            cpu: {{ nodelocaldns_cpu_requests }}
+            memory: {{ nodelocaldnsdns_memory_requests }}
+        args: [ "-localip", "{{ nodelocaldns_ip }}", "-conf", "/etc/coredns/Corefile" ]
+        securityContext:
+          privileged: true
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9253
+          name: metrics
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            host: {{ nodelocaldns_ip }}
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 3
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+      volumes:
+        - name: config-volume
+          configMap:
+            name: nodelocaldns
+            items:
+            - key: Corefile
+              path: Corefile
+      terminationGracePeriodSeconds: 30

roles/kubernetes-apps/ansible/templates/nodelocaldns-sa.yml.j2

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nodelocaldns
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile

roles/kubespray-defaults/defaults/main.yaml

@@ -46,6 +46,9 @@ ndots: 2
 # Can be dnsmasq_kubedns, kubedns, manual or none
 dns_mode: coredns
 
+# Enable nodelocal dns cache
+enable_nodelocaldns: False
+
 # Should be set to a cluster IP if using a custom cluster DNS
 # manual_dns_server: 10.x.x.x
 

tests/files/gce_centos7-flannel-addons.yml

@@ -20,3 +20,4 @@ cert_manager_enabled: true
 metrics_server_enabled: true
 kube_token_auth: true
 kube_basic_auth: true
+enable_nodelocaldns: true