From 3e3ee0aeb1d0785979170f0e5d7c2205317fc761 Mon Sep 17 00:00:00 2001 From: Thomas Nys Date: Tue, 11 Dec 2018 02:28:03 +0100 Subject: [PATCH] Add support for running a nodelocal dns cache (#3861) * Add support for running a nodelocal dns cache After encountering dns issues in a cluster I was recently working on I noticed Kubernetes 1.13 introduced support for running a nodelocal dns cache. I believe this can usefull for more people. https://github.com/kubernetes/kubernetes/commit/73b548db06c5e293533344c5b6171e955eac9ff1 https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/0030-nodelocal-dns-cache.md * Add requested changes * Add additional requested changes + documentation * Add requested changes after review * Replace incorrect variable --- docs/dns-stack.md | 5 ++ .../group_vars/k8s-cluster/k8s-cluster.yml | 2 + roles/download/defaults/main.yml | 13 ++++ .../kubernetes-apps/ansible/defaults/main.yml | 6 ++ .../ansible/tasks/cleanup_dns.yml | 13 ++++ roles/kubernetes-apps/ansible/tasks/main.yml | 11 +++ .../ansible/tasks/nodelocaldns.yml | 18 +++++ .../templates/nodelocaldns-config.yml.j2 | 71 ++++++++++++++++++ .../templates/nodelocaldns-deamonset.yml.j2 | 72 +++++++++++++++++++ .../ansible/templates/nodelocaldns-sa.yml.j2 | 8 +++ roles/kubespray-defaults/defaults/main.yaml | 3 + tests/files/gce_centos7-flannel-addons.yml | 1 + 12 files changed, 223 insertions(+) create mode 100644 roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml create mode 100644 roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 create mode 100644 roles/kubernetes-apps/ansible/templates/nodelocaldns-deamonset.yml.j2 create mode 100644 roles/kubernetes-apps/ansible/templates/nodelocaldns-sa.yml.j2 diff --git a/docs/dns-stack.md b/docs/dns-stack.md index 92689eee5..cf51e044d 100644 --- a/docs/dns-stack.md +++ b/docs/dns-stack.md @@ -84,6 +84,11 @@ leaves you with a non functional cluster. ``resolvconf_mode`` configures how Kubespray will setup DNS for ``hostNetwork: true`` PODs and non-k8s containers. There are three modes available: +## Nodelocal DNS cache +Setting ``enable_nodelocaldns`` to ``true`` will make pods reach out to the dns (core-dns) caching agent running on the same node, thereby avoiding iptables DNAT rules and connection tracking. The local caching agent will query kube-dns / core-dns (depending on what main DNS plugin is configured in your cluster) for cache misses of cluster hostnames(cluster.local suffix by default). + +More information on the rationale behind this implementation can be found [here](https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/0030-nodelocal-dns-cache.md). + #### docker_dns (default) This sets up the docker daemon with additional --dns/--dns-search/--dns-opt flags. diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index e1a800327..81ee23aa8 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -114,6 +114,8 @@ ndots: 2 dns_mode: coredns # Set manual server if using a custom cluster DNS server #manual_dns_server: 10.x.x.x +# Enable nodelocal dns cache +enable_nodelocaldns: False # Can be docker_dns, host_resolvconf or none resolvconf_mode: docker_dns diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 789c630cb..f075cc464 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -184,6 +184,10 @@ coredns_version: "1.2.6" coredns_image_repo: "coredns/coredns" coredns_image_tag: "{{ coredns_version }}" +nodelocaldns_version: "1.15.0" +nodelocaldns_image_repo: "k8s.gcr.io/k8s-dns-node-cache" +nodelocaldns_image_tag: "{{ nodelocaldns_version }}" + dnsmasq_nanny_image_repo: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-{{ image_arch }}" dnsmasq_nanny_image_tag: "{{ kubedns_version }}" dnsmasq_sidecar_image_repo: "gcr.io/google_containers/k8s-dns-sidecar-{{ image_arch }}" @@ -490,6 +494,15 @@ downloads: groups: - kube-node + nodelocaldns: + enabled: "{{ enable_nodelocaldns == True }}" + container: true + repo: "{{ nodelocaldns_image_repo }}" + tag: "{{ nodelocaldns_image_tag }}" + sha256: "{{ nodelocaldns_digest_checksum|default(None) }}" + groups: + - kube-node + dnsmasq_nanny: enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" container: true diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 5e47740ca..127c1efad 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -8,6 +8,12 @@ dns_nodes_per_replica: 10 dns_cores_per_replica: 20 dns_prevent_single_point_failure: "{{ 'true' if dns_min_replicas|int > 1 else 'false' }}" +# nodelocaldns +nodelocaldns_cpu_requests: 100m +nodelocaldns_ip: 169.254.25.10 +nodelocaldns_memory_limit: 170Mi +nodelocaldnsdns_memory_requests: 70Mi + # Netchecker deploy_netchecker: false netchecker_port: 31081 diff --git a/roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml b/roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml index ee6ba3203..56a2bd8a8 100644 --- a/roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml +++ b/roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml @@ -13,6 +13,19 @@ tags: - upgrade +- name: Kubernetes Apps | Delete old nodelocalDNS resources + kube: + name: "nodelocaldns" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item }}" + state: absent + with_items: + - 'deamonset' + - 'configmap' + tags: + - upgrade + - name: Kubernetes Apps | Delete kubeadm CoreDNS kube: name: "coredns" diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index d0f9b6c7c..f186049d7 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -20,6 +20,7 @@ - dnsmasq - coredns - kubedns + - nodelocaldns - name: Kubernetes Apps | CoreDNS import_tasks: "tasks/coredns.yml" @@ -29,6 +30,14 @@ tags: - coredns +- name: Kubernetes Apps | nodelocalDNS + import_tasks: "tasks/nodelocaldns.yml" + when: + - enable_nodelocaldns == True + - inventory_hostname == groups['kube-master'] | first + tags: + - nodelocaldns + - name: Kubernetes Apps | KubeDNS import_tasks: "tasks/kubedns.yml" when: @@ -49,6 +58,7 @@ - "{{ kubedns_manifests.results | default({}) }}" - "{{ coredns_manifests.results | default({}) }}" - "{{ coredns_secondary_manifests.results | default({}) }}" + - "{{ nodelocaldns_manifests.results | default({}) }}" when: - dns_mode != 'none' - inventory_hostname == groups['kube-master'][0] @@ -61,6 +71,7 @@ - dnsmasq - coredns - kubedns + - nodelocaldns loop_control: label: "{{ item.item.file }}" diff --git a/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml b/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml new file mode 100644 index 000000000..0f56daa25 --- /dev/null +++ b/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml @@ -0,0 +1,18 @@ +--- +- name: Kubernetes Apps | Lay Down nodelocaldns Template + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - { name: nodelocaldns, file: nodelocaldns-config.yml, type: configmap } + - { name: nodelocaldns, file: nodelocaldns-sa.yml, type: sa } + - { name: nodelocaldns, file: nodelocaldns-deamonset.yml, type: daemonset } + register: nodelocaldns_manifests + vars: + clusterIP: "{{ skydns_server }}" + secondaryclusterIP: "{{ skydns_server_secondary }}" + when: + - enable_nodelocaldns == True + - inventory_hostname == groups['kube-master'] | first + tags: + - nodelocaldns diff --git a/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 b/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 new file mode 100644 index 000000000..258289029 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2 @@ -0,0 +1,71 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nodelocaldns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists + +data: + Corefile: | + {{ dns_domain }}:53 { + errors + cache 30 + reload + loop + bind {{ nodelocaldns_ip }} +{% if secondaryclusterIP is defined and dns_mode == 'coredns_dual' %} + forward . {{ clusterIP }} {{ secondaryclusterIP }} { +{% else %} + forward . {{ clusterIP }} { +{% endif %} + force_tcp + } + prometheus :9253 + health {{ nodelocaldns_ip }}:8080 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ nodelocaldns_ip }} +{% if secondaryclusterIP is defined %} + forward . {{ clusterIP }} {{ secondaryclusterIP }} { +{% else %} + forward . {{ clusterIP }} { +{% endif %} + force_tcp + } + prometheus :9253 + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ nodelocaldns_ip }} +{% if secondaryclusterIP is defined %} + forward . {{ clusterIP }} {{ secondaryclusterIP }} { +{% else %} + forward . {{ clusterIP }} { +{% endif %} + force_tcp + } + prometheus :9253 + } + .:53 { + errors + cache 30 + reload + loop + bind {{ nodelocaldns_ip }} +{% if resolvconf_mode == 'host_resolvconf' and upstream_dns_servers is defined and upstream_dns_servers|length > 0 %} + forward . {{ upstream_dns_servers|join(' ') }} { +{% else %} + forward . /etc/resolv.conf { +{% endif %} + force_tcp + } + prometheus :9253 + } diff --git a/roles/kubernetes-apps/ansible/templates/nodelocaldns-deamonset.yml.j2 b/roles/kubernetes-apps/ansible/templates/nodelocaldns-deamonset.yml.j2 new file mode 100644 index 000000000..c5a4c09aa --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/nodelocaldns-deamonset.yml.j2 @@ -0,0 +1,72 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nodelocaldns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + selector: + matchLabels: + k8s-app: nodelocaldns + template: + metadata: + labels: + k8s-app: nodelocaldns + spec: +{% if kube_version is version('v1.11.1', '>=') %} + priorityClassName: system-cluster-critical +{% endif %} + serviceAccountName: nodelocaldns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - name: node-cache + image: "{{ nodelocaldns_image_repo }}:{{ nodelocaldns_image_tag }}" + resources: + limits: + memory: {{ nodelocaldns_memory_limit }} + requests: + cpu: {{ nodelocaldns_cpu_requests }} + memory: {{ nodelocaldnsdns_memory_requests }} + args: [ "-localip", "{{ nodelocaldns_ip }}", "-conf", "/etc/coredns/Corefile" ] + securityContext: + privileged: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + host: {{ nodelocaldns_ip }} + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + volumes: + - name: config-volume + configMap: + name: nodelocaldns + items: + - key: Corefile + path: Corefile + terminationGracePeriodSeconds: 30 diff --git a/roles/kubernetes-apps/ansible/templates/nodelocaldns-sa.yml.j2 b/roles/kubernetes-apps/ansible/templates/nodelocaldns-sa.yml.j2 new file mode 100644 index 000000000..5d18742af --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/nodelocaldns-sa.yml.j2 @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nodelocaldns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 15797558f..b46bf1449 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -46,6 +46,9 @@ ndots: 2 # Can be dnsmasq_kubedns, kubedns, manual or none dns_mode: coredns +# Enable nodelocal dns cache +enable_nodelocaldns: False + # Should be set to a cluster IP if using a custom cluster DNS # manual_dns_server: 10.x.x.x diff --git a/tests/files/gce_centos7-flannel-addons.yml b/tests/files/gce_centos7-flannel-addons.yml index 05a9a837f..5432e5488 100644 --- a/tests/files/gce_centos7-flannel-addons.yml +++ b/tests/files/gce_centos7-flannel-addons.yml @@ -20,3 +20,4 @@ cert_manager_enabled: true metrics_server_enabled: true kube_token_auth: true kube_basic_auth: true +enable_nodelocaldns: true