Merge pull request #2462 from woopstar/coredns-patch

Add CoreDNS support

docs/dns-stack.md

@@ -62,6 +62,14 @@ other queries are forwardet to the nameservers found in ``upstream_dns_servers``
 This does not install the dnsmasq DaemonSet and instructs kubelet to directly use kubedns/skydns for
 all queries.
 
+#### coredns
+This does not install the dnsmasq DaemonSet and instructs kubelet to directly use CoreDNS for
+all queries.
+
+#### coredns_dual
+This does not install the dnsmasq DaemonSet and instructs kubelet to directly use CoreDNS for
+all queries. It will also deploy a secondary CoreDNS stack
+
 #### manual
 This does not install dnsmasq or kubedns, but allows you to specify
 `manual_dns_server`, which will be configured on nodes for handling Pod DNS.

docs/vars.md

@@ -63,7 +63,8 @@ following default cluster paramters:
   bits in kube_pods_subnet dictates how many kube-nodes can be in cluster.
 * *dns_setup* - Enables dnsmasq
 * *dnsmasq_dns_server* - Cluster IP for dnsmasq (default is 10.233.0.2)
-* *skydns_server* - Cluster IP for KubeDNS (default is 10.233.0.3)
+* *skydns_server* - Cluster IP for DNS (default is 10.233.0.3)
+* *skydns_server_secondary* - Secondary Cluster IP for CoreDNS used with coredns_dual deployment (default is 10.233.0.4)
 * *cloud_provider* - Enable extra Kubelet option if operating inside GCE or
   OpenStack (default is unset)
 * *kube_hostpath_dynamic_provisioner* - Required for use of PetSets type in

inventory/sample/group_vars/k8s-cluster.yml

@@ -121,7 +121,7 @@ kube_encrypt_secret_data: false
 cluster_name: cluster.local
 # Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods
 ndots: 2
-# Can be dnsmasq_kubedns, kubedns, manual or none
+# Can be dnsmasq_kubedns, kubedns, coredns, coredns_dual, manual or none
 dns_mode: kubedns
 # Set manual server if using a custom cluster DNS server
 #manual_dns_server: 10.x.x.x
@@ -132,6 +132,7 @@ resolvconf_mode: docker_dns
 deploy_netchecker: false
 # Ip address of the kubernetes skydns service
 skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}"
+skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}"
 dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
 dns_domain: "{{ cluster_name }}"
 

roles/docker/tasks/set_facts_dns.yml

@@ -3,8 +3,10 @@
 - name: set dns server for docker
   set_fact:
     docker_dns_servers: |-
-      {%- if dns_mode == 'kubedns' -%}
+      {%- if dns_mode in ['kubedns', 'coredns'] -%}
         {{ [ skydns_server ] }}
+      {%- elif dns_mode == 'coredns_dual' -%}
+        {{ [ skydns_server ] + [ skydns_server_secondary ] }}
       {%- elif dns_mode == 'dnsmasq_kubedns' -%}
         {{ [ dnsmasq_dns_server ] }}
       {%- elif dns_mode == 'manual' -%}
@@ -24,7 +26,7 @@
 - name: add upstream dns servers (only when dnsmasq is not used)
   set_fact:
     docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers|default([]) }}"
-  when: dns_mode == 'kubedns'
+  when: dns_mode in ['kubedns', 'coredns', 'coreos_dual']
 
 - name: add global searchdomains
   set_fact:

roles/download/defaults/main.yml

@@ -100,6 +100,9 @@ dnsmasq_image_tag: "{{ dnsmasq_version }}"
 kubedns_version: 1.14.8
 kubedns_image_repo: "gcr.io/google_containers/k8s-dns-kube-dns-amd64"
 kubedns_image_tag: "{{ kubedns_version }}"
+coredns_version: 1.1.0
+coredns_image_repo: "docker.io/coredns/coredns"
+coredns_image_tag: "{{ coredns_version }}"
 dnsmasq_nanny_image_repo: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64"
 dnsmasq_nanny_image_tag: "{{ kubedns_version }}"
 dnsmasq_sidecar_image_repo: "gcr.io/google_containers/k8s-dns-sidecar-amd64"
@@ -274,25 +277,31 @@ downloads:
     tag: "{{ dnsmasq_image_tag }}"
     sha256: "{{ dnsmasq_digest_checksum|default(None) }}"
   kubedns:
-    enabled: true
+    enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}"
     container: true
     repo: "{{ kubedns_image_repo }}"
     tag: "{{ kubedns_image_tag }}"
     sha256: "{{ kubedns_digest_checksum|default(None) }}"
+  coredns:
+    enabled: "{{ dns_mode in ['coredns', 'coredns_dual'] }}"
+    container: true
+    repo: "{{ coredns_image_repo }}"
+    tag: "{{ coredns_image_tag }}"
+    sha256: "{{ coredns_digest_checksum|default(None) }}"
   dnsmasq_nanny:
-    enabled: true
+    enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}"
     container: true
     repo: "{{ dnsmasq_nanny_image_repo }}"
     tag: "{{ dnsmasq_nanny_image_tag }}"
     sha256: "{{ dnsmasq_nanny_digest_checksum|default(None) }}"
   dnsmasq_sidecar:
-    enabled: true
+    enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}"
     container: true
     repo: "{{ dnsmasq_sidecar_image_repo }}"
     tag: "{{ dnsmasq_sidecar_image_tag }}"
     sha256: "{{ dnsmasq_sidecar_digest_checksum|default(None) }}"
   kubednsautoscaler:
-    enabled: true
+    enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}"
     container: true
     repo: "{{ kubednsautoscaler_image_repo }}"
     tag: "{{ kubednsautoscaler_image_tag }}"

roles/kubernetes-apps/ansible/defaults/main.yml

@@ -10,6 +10,9 @@ dns_memory_requests: 70Mi
 kubedns_min_replicas: 2
 kubedns_nodes_per_replica: 10
 
+# CoreDNS
+coredns_replicas: 2
+
 # Images
 kubedns_image_repo: "gcr.io/google_containers/k8s-dns-kube-dns-amd64"
 kubedns_image_tag: "{{ kubedns_version }}"

roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml

@@ -0,0 +1,54 @@
+---
+- name: Kubernetes Apps | Delete old CoreDNS resources
+  kube:
+    name: "coredns"
+    namespace: "{{ system_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item }}"
+    state: absent
+  with_items:
+    - 'deploy'
+    - 'configmap'
+    - 'svc'
+  tags:
+    - upgrade
+
+- name: Kubernetes Apps | Delete kubeadm CoreDNS
+  kube:
+    name: "coredns"
+    namespace: "{{ system_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "deploy"
+    state: absent
+  when:
+    - kubeadm_enabled|default(false)
+    - kubeadm_init.changed|default(false)
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: Kubernetes Apps | Delete old KubeDNS resources
+  kube:
+    name: "kube-dns"
+    namespace: "{{ system_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item }}"
+    state: absent
+  with_items:
+    - 'deploy'
+    - 'svc'
+  tags:
+    - upgrade
+
+- name: Kubernetes Apps | Delete kubeadm KubeDNS
+  kube:
+    name: "kube-dns"
+    namespace: "{{ system_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item }}"
+    state: absent
+  with_items:
+    - 'deploy'
+    - 'svc'
+  when:
+    - kubeadm_enabled|default(false)
+    - kubeadm_init.changed|default(false)
+    - inventory_hostname == groups['kube-master'][0]

roles/kubernetes-apps/ansible/tasks/coredns.yml

@@ -0,0 +1,39 @@
+---
+- name: Kubernetes Apps | Lay Down CoreDNS Template
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - { name: coredns, file: coredns-config.yml, type: configmap }
+    - { name: coredns, file: coredns-sa.yml, type: sa }
+    - { name: coredns, file: coredns-deployment.yml, type: deployment }
+    - { name: coredns, file: coredns-svc.yml, type: svc }
+    - { name: coredns, file: coredns-clusterrole.yml, type: clusterrole }
+    - { name: coredns, file: coredns-clusterrolebinding.yml, type: clusterrolebinding }
+  register: coredns_manifests
+  vars:
+    clusterIP: "{{ skydns_server }}"
+  when:
+    - dns_mode in ['coredns', 'coredns_dual']
+    - inventory_hostname == groups['kube-master'][0]
+    - rbac_enabled or item.type not in rbac_resources
+  tags:
+    - coredns
+
+- name: Kubernetes Apps | Lay Down Secondary CoreDNS Template
+  template:
+    src: "{{ item.src }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - { name: coredns, src: coredns-deployment.yml, file: coredns-deployment-secondary.yml, type: deployment }
+    - { name: coredns, src: coredns-svc.yml, file: coredns-svc-secondary.yml, type: svc }
+  register: coredns_secondary_manifests
+  vars:
+    clusterIP: "{{ skydns_server_secondary }}"
+    coredns_ordinal_suffix: "-secondary"
+  when:
+    - dns_mode == 'coredns_dual'
+    - inventory_hostname == groups['kube-master'][0]
+    - rbac_enabled or item.type not in rbac_resources
+  tags:
+    - coredns

roles/kubernetes-apps/ansible/tasks/kubedns.yml

@@ -0,0 +1,41 @@
+---
+
+- name: Kubernetes Apps | Lay Down KubeDNS Template
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - { name: kube-dns, file: kubedns-sa.yml, type: sa }
+    - { name: kube-dns, file: kubedns-deploy.yml, type: deployment }
+    - { name: kube-dns, file: kubedns-svc.yml, type: svc }
+    - { name: kubedns-autoscaler, file: kubedns-autoscaler-sa.yml, type: sa }
+    - { name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrole.yml, type: clusterrole }
+    - { name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrolebinding.yml, type: clusterrolebinding }
+    - { name: kubedns-autoscaler, file: kubedns-autoscaler.yml, type: deployment }
+  register: kubedns_manifests
+  when:
+    - dns_mode in ['kubedns','dnsmasq_kubedns']
+    - inventory_hostname == groups['kube-master'][0]
+    - rbac_enabled or item.type not in rbac_resources
+  tags:
+    - dnsmasq
+
+# see https://github.com/kubernetes/kubernetes/issues/45084, only needed for "old" kube-dns
+- name: Kubernetes Apps | Patch system:kube-dns ClusterRole
+  command: >
+    {{ bin_dir }}/kubectl patch clusterrole system:kube-dns
+    --patch='{
+               "rules": [
+                 {
+                   "apiGroups" : [""],
+                   "resources" : ["endpoints", "services"],
+                   "verbs": ["list", "watch", "get"]
+                 }
+               ]
+             }'
+  when:
+    - dns_mode in ['kubedns', 'dnsmasq_kubedns']
+    - inventory_hostname == groups['kube-master'][0]
+    - rbac_enabled and kubedns_version|version_compare("1.11.0", "<", strict=True)
+  tags:
+    - dnsmasq

roles/kubernetes-apps/ansible/tasks/main.yml

@@ -11,66 +11,26 @@
   delay: 2
   when: inventory_hostname == groups['kube-master'][0]
 
-- name: Kubernetes Apps | Delete old kubedns resources
+- name: Kubernetes Apps | Cleanup DNS
-  kube:
+  import_tasks: tasks/cleanup_dns.yml
-    name: "kubedns"
+  when:
-    namespace: "{{ system_namespace }}"
+    - inventory_hostname == groups['kube-master'][0]
-    kubectl: "{{ bin_dir }}/kubectl"
-    resource: "{{ item }}"
-    state: absent
-  with_items:
-    - 'deploy'
-    - 'svc'
   tags:
     - upgrade
 
-- name: Kubernetes Apps | Delete kubeadm kubedns
+- name: Kubernetes Apps | CoreDNS
-  kube:
+  import_tasks: "tasks/coredns.yml"
-    name: "kubedns"
-    namespace: "{{ system_namespace }}"
-    kubectl: "{{ bin_dir }}/kubectl"
-    resource: "deploy"
-    state: absent
   when:
-    - kubeadm_enabled|default(false)
+    - dns_mode in ['coredns', 'coredns_dual']
-    - kubeadm_init.changed|default(false)
     - inventory_hostname == groups['kube-master'][0]
-
-- name: Kubernetes Apps | Lay Down KubeDNS Template
-  template:
-    src: "{{ item.file }}.j2"
-    dest: "{{ kube_config_dir }}/{{ item.file }}"
-  with_items:
-    - { name: kube-dns, file: kubedns-sa.yml, type: sa }
-    - { name: kube-dns, file: kubedns-deploy.yml, type: deployment }
-    - { name: kube-dns, file: kubedns-svc.yml, type: svc }
-    - { name: kubedns-autoscaler, file: kubedns-autoscaler-sa.yml, type: sa }
-    - { name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrole.yml, type: clusterrole }
-    - { name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrolebinding.yml, type: clusterrolebinding }
-    - { name: kubedns-autoscaler, file: kubedns-autoscaler.yml, type: deployment }
-  register: manifests
-  when:
-    - dns_mode != 'none' and inventory_hostname == groups['kube-master'][0]
-    - rbac_enabled or item.type not in rbac_resources
   tags:
-    - dnsmasq
+    - coredns
 
-# see https://github.com/kubernetes/kubernetes/issues/45084, only needed for "old" kube-dns
+- name: Kubernetes Apps | KubeDNS
-- name: Kubernetes Apps | Patch system:kube-dns ClusterRole
+  import_tasks: "tasks/kubedns.yml"
-  command: >
-    {{ bin_dir }}/kubectl patch clusterrole system:kube-dns
-    --patch='{
-               "rules": [
-                 {
-                   "apiGroups" : [""],
-                   "resources" : ["endpoints", "services"],
-                   "verbs": ["list", "watch", "get"]
-                 }
-               ]
-             }'
   when:
-    - dns_mode != 'none' and inventory_hostname == groups['kube-master'][0]
+    - dns_mode in ['kubedns', 'dnsmasq_kubedns']
-    - rbac_enabled and kubedns_version|version_compare("1.11.0", "<", strict=True)
+    - inventory_hostname == groups['kube-master'][0]
   tags:
     - dnsmasq
 
@@ -82,7 +42,10 @@
     resource: "{{ item.item.type }}"
     filename: "{{ kube_config_dir }}/{{ item.item.file }}"
     state: "latest"
-  with_items: "{{ manifests.results }}"
+  with_items:
+    - "{{ kubedns_manifests.results | default({}) }}"
+    - "{{ coredns_manifests.results | default({}) }}"
+    - "{{ coredns_secondary_manifests.results | default({}) }}"
   when:
     - dns_mode != 'none'
     - inventory_hostname == groups['kube-master'][0]

roles/kubernetes-apps/ansible/templates/coredns-clusterrole.yml.j2

@@ -0,0 +1,19 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: Reconcile
+  name: system:coredns
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - services
+  - pods
+  - namespaces
+  verbs:
+  - list
+  - watch

roles/kubernetes-apps/ansible/templates/coredns-clusterrolebinding.yml.j2

@@ -0,0 +1,18 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: EnsureExists
+  name: system:coredns
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:coredns
+subjects:
+- kind: ServiceAccount
+  name: coredns
+  namespace: {{ system_namespace }}

roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: {{ system_namespace }}
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health
+        kubernetes {{ cluster_name }} in-addr.arpa ip6.arpa {
+          pods insecure
+          upstream /etc/resolv.conf
+          fallthrough in-addr.arpa ip6.arpa
+        }
+        prometheus :9153
+        proxy . /etc/resolv.conf
+        cache 30
+    }

roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2

@@ -0,0 +1,81 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: coredns{{ coredns_ordinal_suffix | default('') }}
+  namespace: {{ system_namespace }}
+  labels:
+    k8s-app: coredns{{ coredns_ordinal_suffix | default('') }}
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/name: "CoreDNS"
+spec:
+  replicas: {{ coredns_replicas }}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: 10%
+  selector:
+    matchLabels:
+      k8s-app: coredns{{ coredns_ordinal_suffix | default('') }}
+  template:
+    metadata:
+      labels:
+        k8s-app: coredns{{ coredns_ordinal_suffix | default('') }}
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+{% if rbac_enabled %}
+      serviceAccountName: coredns
+{% endif %}
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+      containers:
+      - name: coredns
+        image: "{{ coredns_image_repo }}:{{ coredns_image_tag }}"
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        resources:
+          # TODO: Set memory limits when we've profiled the container for large
+          # clusters, then set request = limit to keep this container in
+          # guaranteed class. Currently, this container falls into the
+          # "burstable" category so the kubelet doesn't backoff from restarting it.
+          limits:
+            memory: {{ dns_memory_limit }}
+          requests:
+            cpu: {{ dns_cpu_requests }}
+            memory: {{ dns_memory_requests }}
+        args: [ "-conf", "/etc/coredns/Corefile" ]
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9153
+          name: metrics
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+      dnsPolicy: Default
+      volumes:
+        - name: config-volume
+          configMap:
+            name: coredns
+            items:
+            - key: Corefile
+              path: Corefile

roles/kubernetes-apps/ansible/templates/coredns-sa.yml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: coredns
+  namespace: {{ system_namespace }}
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile

roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: coredns{{ coredns_ordinal_suffix | default('') }}
+  namespace: {{ system_namespace }}
+  labels:
+    k8s-app: coredns{{ coredns_ordinal_suffix | default('') }}
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/name: "CoreDNS"
+spec:
+  selector:
+    k8s-app: coredns{{ coredns_ordinal_suffix | default('') }}
+  clusterIP: {{ clusterIP }}
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    protocol: TCP

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -51,8 +51,10 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% endif %}
 
 {# DNS settings for kubelet #}
-{% if dns_mode == 'kubedns' %}
+{% if dns_mode in ['kubedns', 'coredns'] %}
 {% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }}{% endset %}
+{% elif dns_mode == 'coredns_dual' %}
+{% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }},{{ skydns_server_secondary }}{% endset %}
 {% elif dns_mode == 'dnsmasq_kubedns' %}
 {% set kubelet_args_cluster_dns %}--cluster-dns={{ dnsmasq_dns_server }}{% endset %}
 {% elif dns_mode == 'manual' %}

roles/kubernetes/node/templates/kubelet.standard.env.j2

@@ -42,8 +42,10 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 --enforce-node-allocatable={{ kubelet_enforce_node_allocatable }} {% endif %}{% endset %}
 
 {# DNS settings for kubelet #}
-{% if dns_mode == 'kubedns' %}
+{% if dns_mode in ['kubedns', 'coredns'] %}
 {% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }}{% endset %}
+{% elif dns_mode == 'coredns_dual' %}
+{% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }},{{ skydns_server_secondary }}{% endset %}
 {% elif dns_mode == 'dnsmasq_kubedns' %}
 {% set kubelet_args_cluster_dns %}--cluster-dns={{ dnsmasq_dns_server }}{% endset %}
 {% elif dns_mode == 'manual' %}

roles/kubernetes/preinstall/tasks/set_resolv_facts.yml

@@ -93,8 +93,10 @@
 - name: pick dnsmasq cluster IP or default resolver
   set_fact:
     dnsmasq_server: |-
-      {%- if dns_mode == 'kubedns' and not dns_early|bool -%}
+      {%- if dns_mode in ['kubedns', 'coredns'] and not dns_early|bool -%}
         {{ [ skydns_server ] + upstream_dns_servers|default([]) }}
+      {%- elif dns_mode == 'coredns_dual' and not dns_early|bool -%}
+        {{ [ skydns_server ] + [ skydns_server_secondary ] + upstream_dns_servers|default([]) }}
       {%- elif dns_mode == 'manual' and not dns_early|bool -%}
         {{ [ manual_dns_server ] + upstream_dns_servers|default([]) }}
       {%- elif dns_early|bool -%}

roles/kubespray-defaults/defaults/main.yaml

@@ -49,6 +49,7 @@ resolvconf_mode: docker_dns
 deploy_netchecker: false
 # Ip address of the kubernetes skydns service
 skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}"
+skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}"
 dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
 dns_domain: "{{ cluster_name }}"