Set cluster DNS correctly in case of nodelocal dns cache (#3879)

* Set cluster DNS correctly in case of nodelocal dns cache

* Pass in cluster_ip based on dns mode

* Disable nodelocaldns by default

* Fix syntax error

* Fix syntax issue

* Add nodelocadns ip to vars of node installation

* Change location of nodelocaldns_ip

* Try to remove newlines from jinja template

* Add debug for config file

* Move parameter logic outside of template

* Adapt templates after feedback

* Remove debugging

inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml

@@ -122,6 +122,7 @@ dns_mode: coredns
 #manual_dns_server: 10.x.x.x
 # Enable nodelocal dns cache
 enable_nodelocaldns: False
+nodelocaldns_ip: 169.254.25.10
 
 # Can be docker_dns, host_resolvconf or none
 resolvconf_mode: docker_dns

roles/kubernetes-apps/ansible/defaults/main.yml

@@ -10,7 +10,6 @@ dns_prevent_single_point_failure: "{{ 'true' if dns_min_replicas|int > 1 else 'f
 
 # nodelocaldns
 nodelocaldns_cpu_requests: 100m
-nodelocaldns_ip: 169.254.25.10
 nodelocaldns_memory_limit: 170Mi
 nodelocaldnsdns_memory_requests: 70Mi
 

roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml

@@ -1,4 +1,16 @@
 ---
+- name: Kubernetes Apps | set up necessary nodelocaldns parameters
+  set_fact:
+    clusterIP: >-
+      {%- if dns_mode in ['kubedns', 'coredns', 'coredns_dual'] -%}
+      {{ skydns_server }}
+      {%- elif dns_mode == 'dnsmasq_kubedns' -%}
+      {{ dnsmasq_dns_server }}
+      {%- elif dns_mode == 'manual' -%}
+      {{ manual_dns_server }}
+      {%- endif -%}
+    secondaryclusterIP: "{{ skydns_server_secondary }}"
+
 - name: Kubernetes Apps | Lay Down nodelocaldns Template
   template:
     src: "{{ item.file }}.j2"
@@ -9,8 +21,18 @@
     - { name: nodelocaldns, file: nodelocaldns-daemonset.yml, type: daemonset }
   register: nodelocaldns_manifests
   vars:
-    clusterIP: "{{ skydns_server }}"
+    forwardTarget: >-
-    secondaryclusterIP: "{{ skydns_server_secondary }}"
+      {%- if secondaryclusterIP is defined and dns_mode == 'coredns_dual' -%}
+      {{ clusterIP }} {{ secondaryclusterIP }}
+      {%- else -%}
+      {{ clusterIP }}
+      {%- endif -%}
+    upstreamForwardTarget: >-
+      {%- if resolvconf_mode == 'host_resolvconf' and upstream_dns_servers is defined and upstream_dns_servers|length > 0 -%}
+      {{ upstream_dns_servers|join(' ') }}
+      {%- else -%}
+      /etc/resolv.conf
+      {%- endif -%}
   when:
     - enable_nodelocaldns == True
     - inventory_hostname == groups['kube-master'] | first

roles/kubernetes-apps/ansible/templates/nodelocaldns-config.yml.j2

@@ -14,58 +14,42 @@ data:
         reload
         loop
         bind {{ nodelocaldns_ip }}
-{% if secondaryclusterIP is defined and dns_mode == 'coredns_dual' %}
+        forward . {{ forwardTarget }} {
-        forward . {{ clusterIP }} {{ secondaryclusterIP }} {
+            force_tcp
-{% else %}
-        forward . {{ clusterIP }} {
-{% endif %}
-                force_tcp
         }
         prometheus :9253
         health {{ nodelocaldns_ip }}:8080
-        }
+    }
     in-addr.arpa:53 {
         errors
         cache 30
         reload
         loop
         bind {{ nodelocaldns_ip }}
-{% if secondaryclusterIP is defined %}
+        forward . {{ forwardTarget }} {
-        forward . {{ clusterIP }} {{ secondaryclusterIP }} {
+            force_tcp
-{% else %}
-        forward . {{ clusterIP }} {
-{% endif %}
-                force_tcp
         }
         prometheus :9253
-        }
+    }
     ip6.arpa:53 {
         errors
         cache 30
         reload
         loop
         bind {{ nodelocaldns_ip }}
-{% if secondaryclusterIP is defined %}
+        forward . {{ forwardTarget }} {
-        forward . {{ clusterIP }} {{ secondaryclusterIP }} {
+            force_tcp
-{% else %}
-        forward . {{ clusterIP }} {
-{% endif %}
-             force_tcp
         }
         prometheus :9253
-        }
+    }
     .:53 {
         errors
         cache 30
         reload
         loop
         bind {{ nodelocaldns_ip }}
-{% if resolvconf_mode == 'host_resolvconf' and upstream_dns_servers is defined and upstream_dns_servers|length > 0 %}
+        forward . {{ upstreamForwardTarget }} {
-        forward . {{ upstream_dns_servers|join(' ') }} {
+            force_tcp
-{% else %}
-        forward . /etc/resolv.conf {
-{% endif %}
-                force_tcp
         }
         prometheus :9253
-        }
+    }

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -76,6 +76,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% else %}
 {% set kubelet_args_cluster_dns %}{% endset %}
 {% endif %}
+{% if enable_nodelocaldns == True %}
+{% set kubelet_args_cluster_dns %}--cluster-dns={{ nodelocaldns_ip }}{% endset %}
+{% endif %}
 {% set kubelet_args_dns %}{{ kubelet_args_cluster_dns }} --cluster-domain={{ dns_domain }} --resolv-conf={{ kube_resolv_conf }}{% endset %}
 
 {# Kubelet node labels #}

roles/kubespray-defaults/defaults/main.yaml

@@ -57,6 +57,7 @@ dns_mode: coredns
 
 # Enable nodelocal dns cache
 enable_nodelocaldns: False
+nodelocaldns_ip: 169.254.25.10
 
 # Should be set to a cluster IP if using a custom cluster DNS
 # manual_dns_server: 10.x.x.x