Add support for running a nodelocal dns cache (#3861)

* Add support for running a nodelocal dns cache

After encountering dns issues in a cluster I was recently working on I
noticed Kubernetes 1.13 introduced support for running a nodelocal dns
cache.

I believe this can usefull for more people.

73b548db06
https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/0030-nodelocal-dns-cache.md

* Add requested changes

* Add additional requested changes + documentation

* Add requested changes after review

* Replace incorrect variable
This commit is contained in:
Thomas Nys 2018-12-11 02:28:03 +01:00 committed by Kubernetes Prow Robot
parent 7b674e0607
commit 3e3ee0aeb1
12 changed files with 223 additions and 0 deletions

View file

@ -84,6 +84,11 @@ leaves you with a non functional cluster.
``resolvconf_mode`` configures how Kubespray will setup DNS for ``hostNetwork: true`` PODs and non-k8s containers. ``resolvconf_mode`` configures how Kubespray will setup DNS for ``hostNetwork: true`` PODs and non-k8s containers.
There are three modes available: There are three modes available:
## Nodelocal DNS cache
Setting ``enable_nodelocaldns`` to ``true`` will make pods reach out to the dns (core-dns) caching agent running on the same node, thereby avoiding iptables DNAT rules and connection tracking. The local caching agent will query kube-dns / core-dns (depending on what main DNS plugin is configured in your cluster) for cache misses of cluster hostnames(cluster.local suffix by default).
More information on the rationale behind this implementation can be found [here](https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/0030-nodelocal-dns-cache.md).
#### docker_dns (default) #### docker_dns (default)
This sets up the docker daemon with additional --dns/--dns-search/--dns-opt flags. This sets up the docker daemon with additional --dns/--dns-search/--dns-opt flags.

View file

@ -114,6 +114,8 @@ ndots: 2
dns_mode: coredns dns_mode: coredns
# Set manual server if using a custom cluster DNS server # Set manual server if using a custom cluster DNS server
#manual_dns_server: 10.x.x.x #manual_dns_server: 10.x.x.x
# Enable nodelocal dns cache
enable_nodelocaldns: False
# Can be docker_dns, host_resolvconf or none # Can be docker_dns, host_resolvconf or none
resolvconf_mode: docker_dns resolvconf_mode: docker_dns

View file

@ -184,6 +184,10 @@ coredns_version: "1.2.6"
coredns_image_repo: "coredns/coredns" coredns_image_repo: "coredns/coredns"
coredns_image_tag: "{{ coredns_version }}" coredns_image_tag: "{{ coredns_version }}"
nodelocaldns_version: "1.15.0"
nodelocaldns_image_repo: "k8s.gcr.io/k8s-dns-node-cache"
nodelocaldns_image_tag: "{{ nodelocaldns_version }}"
dnsmasq_nanny_image_repo: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-{{ image_arch }}" dnsmasq_nanny_image_repo: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-{{ image_arch }}"
dnsmasq_nanny_image_tag: "{{ kubedns_version }}" dnsmasq_nanny_image_tag: "{{ kubedns_version }}"
dnsmasq_sidecar_image_repo: "gcr.io/google_containers/k8s-dns-sidecar-{{ image_arch }}" dnsmasq_sidecar_image_repo: "gcr.io/google_containers/k8s-dns-sidecar-{{ image_arch }}"
@ -490,6 +494,15 @@ downloads:
groups: groups:
- kube-node - kube-node
nodelocaldns:
enabled: "{{ enable_nodelocaldns == True }}"
container: true
repo: "{{ nodelocaldns_image_repo }}"
tag: "{{ nodelocaldns_image_tag }}"
sha256: "{{ nodelocaldns_digest_checksum|default(None) }}"
groups:
- kube-node
dnsmasq_nanny: dnsmasq_nanny:
enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}"
container: true container: true

View file

@ -8,6 +8,12 @@ dns_nodes_per_replica: 10
dns_cores_per_replica: 20 dns_cores_per_replica: 20
dns_prevent_single_point_failure: "{{ 'true' if dns_min_replicas|int > 1 else 'false' }}" dns_prevent_single_point_failure: "{{ 'true' if dns_min_replicas|int > 1 else 'false' }}"
# nodelocaldns
nodelocaldns_cpu_requests: 100m
nodelocaldns_ip: 169.254.25.10
nodelocaldns_memory_limit: 170Mi
nodelocaldnsdns_memory_requests: 70Mi
# Netchecker # Netchecker
deploy_netchecker: false deploy_netchecker: false
netchecker_port: 31081 netchecker_port: 31081

View file

@ -13,6 +13,19 @@
tags: tags:
- upgrade - upgrade
- name: Kubernetes Apps | Delete old nodelocalDNS resources
kube:
name: "nodelocaldns"
namespace: "kube-system"
kubectl: "{{ bin_dir }}/kubectl"
resource: "{{ item }}"
state: absent
with_items:
- 'deamonset'
- 'configmap'
tags:
- upgrade
- name: Kubernetes Apps | Delete kubeadm CoreDNS - name: Kubernetes Apps | Delete kubeadm CoreDNS
kube: kube:
name: "coredns" name: "coredns"

View file

@ -20,6 +20,7 @@
- dnsmasq - dnsmasq
- coredns - coredns
- kubedns - kubedns
- nodelocaldns
- name: Kubernetes Apps | CoreDNS - name: Kubernetes Apps | CoreDNS
import_tasks: "tasks/coredns.yml" import_tasks: "tasks/coredns.yml"
@ -29,6 +30,14 @@
tags: tags:
- coredns - coredns
- name: Kubernetes Apps | nodelocalDNS
import_tasks: "tasks/nodelocaldns.yml"
when:
- enable_nodelocaldns == True
- inventory_hostname == groups['kube-master'] | first
tags:
- nodelocaldns
- name: Kubernetes Apps | KubeDNS - name: Kubernetes Apps | KubeDNS
import_tasks: "tasks/kubedns.yml" import_tasks: "tasks/kubedns.yml"
when: when:
@ -49,6 +58,7 @@
- "{{ kubedns_manifests.results | default({}) }}" - "{{ kubedns_manifests.results | default({}) }}"
- "{{ coredns_manifests.results | default({}) }}" - "{{ coredns_manifests.results | default({}) }}"
- "{{ coredns_secondary_manifests.results | default({}) }}" - "{{ coredns_secondary_manifests.results | default({}) }}"
- "{{ nodelocaldns_manifests.results | default({}) }}"
when: when:
- dns_mode != 'none' - dns_mode != 'none'
- inventory_hostname == groups['kube-master'][0] - inventory_hostname == groups['kube-master'][0]
@ -61,6 +71,7 @@
- dnsmasq - dnsmasq
- coredns - coredns
- kubedns - kubedns
- nodelocaldns
loop_control: loop_control:
label: "{{ item.item.file }}" label: "{{ item.item.file }}"

View file

@ -0,0 +1,18 @@
---
- name: Kubernetes Apps | Lay Down nodelocaldns Template
template:
src: "{{ item.file }}.j2"
dest: "{{ kube_config_dir }}/{{ item.file }}"
with_items:
- { name: nodelocaldns, file: nodelocaldns-config.yml, type: configmap }
- { name: nodelocaldns, file: nodelocaldns-sa.yml, type: sa }
- { name: nodelocaldns, file: nodelocaldns-deamonset.yml, type: daemonset }
register: nodelocaldns_manifests
vars:
clusterIP: "{{ skydns_server }}"
secondaryclusterIP: "{{ skydns_server_secondary }}"
when:
- enable_nodelocaldns == True
- inventory_hostname == groups['kube-master'] | first
tags:
- nodelocaldns

View file

@ -0,0 +1,71 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: nodelocaldns
namespace: kube-system
labels:
addonmanager.kubernetes.io/mode: EnsureExists
data:
Corefile: |
{{ dns_domain }}:53 {
errors
cache 30
reload
loop
bind {{ nodelocaldns_ip }}
{% if secondaryclusterIP is defined and dns_mode == 'coredns_dual' %}
forward . {{ clusterIP }} {{ secondaryclusterIP }} {
{% else %}
forward . {{ clusterIP }} {
{% endif %}
force_tcp
}
prometheus :9253
health {{ nodelocaldns_ip }}:8080
}
in-addr.arpa:53 {
errors
cache 30
reload
loop
bind {{ nodelocaldns_ip }}
{% if secondaryclusterIP is defined %}
forward . {{ clusterIP }} {{ secondaryclusterIP }} {
{% else %}
forward . {{ clusterIP }} {
{% endif %}
force_tcp
}
prometheus :9253
}
ip6.arpa:53 {
errors
cache 30
reload
loop
bind {{ nodelocaldns_ip }}
{% if secondaryclusterIP is defined %}
forward . {{ clusterIP }} {{ secondaryclusterIP }} {
{% else %}
forward . {{ clusterIP }} {
{% endif %}
force_tcp
}
prometheus :9253
}
.:53 {
errors
cache 30
reload
loop
bind {{ nodelocaldns_ip }}
{% if resolvconf_mode == 'host_resolvconf' and upstream_dns_servers is defined and upstream_dns_servers|length > 0 %}
forward . {{ upstream_dns_servers|join(' ') }} {
{% else %}
forward . /etc/resolv.conf {
{% endif %}
force_tcp
}
prometheus :9253
}

View file

@ -0,0 +1,72 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: nodelocaldns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
spec:
selector:
matchLabels:
k8s-app: nodelocaldns
template:
metadata:
labels:
k8s-app: nodelocaldns
spec:
{% if kube_version is version('v1.11.1', '>=') %}
priorityClassName: system-cluster-critical
{% endif %}
serviceAccountName: nodelocaldns
hostNetwork: true
dnsPolicy: Default # Don't use cluster DNS.
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
- key: "CriticalAddonsOnly"
operator: "Exists"
containers:
- name: node-cache
image: "{{ nodelocaldns_image_repo }}:{{ nodelocaldns_image_tag }}"
resources:
limits:
memory: {{ nodelocaldns_memory_limit }}
requests:
cpu: {{ nodelocaldns_cpu_requests }}
memory: {{ nodelocaldnsdns_memory_requests }}
args: [ "-localip", "{{ nodelocaldns_ip }}", "-conf", "/etc/coredns/Corefile" ]
securityContext:
privileged: true
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9253
name: metrics
protocol: TCP
livenessProbe:
httpGet:
host: {{ nodelocaldns_ip }}
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 3
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
volumes:
- name: config-volume
configMap:
name: nodelocaldns
items:
- key: Corefile
path: Corefile
terminationGracePeriodSeconds: 30

View file

@ -0,0 +1,8 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: nodelocaldns
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile

View file

@ -46,6 +46,9 @@ ndots: 2
# Can be dnsmasq_kubedns, kubedns, manual or none # Can be dnsmasq_kubedns, kubedns, manual or none
dns_mode: coredns dns_mode: coredns
# Enable nodelocal dns cache
enable_nodelocaldns: False
# Should be set to a cluster IP if using a custom cluster DNS # Should be set to a cluster IP if using a custom cluster DNS
# manual_dns_server: 10.x.x.x # manual_dns_server: 10.x.x.x

View file

@ -20,3 +20,4 @@ cert_manager_enabled: true
metrics_server_enabled: true metrics_server_enabled: true
kube_token_auth: true kube_token_auth: true
kube_basic_auth: true kube_basic_auth: true
enable_nodelocaldns: true