From b5542465020288b85f546e6079957e27139ff8b0 Mon Sep 17 00:00:00 2001 From: Mac Chaffee Date: Sat, 26 Feb 2022 13:29:23 -0500 Subject: [PATCH] Fix host DNS config 1) being edited too soon and 2) not working with NM (#8575) Signed-off-by: Mac Chaffee --- cluster.yml | 3 +- roles/kubernetes/preinstall/defaults/main.yml | 1 + roles/kubernetes/preinstall/handlers/main.yml | 3 +- .../preinstall/tasks/0040-set_facts.yml | 29 +++++++++++++++---- .../preinstall/tasks/0060-resolvconf.yml | 5 +++- .../0062-networkmanager-unmanaged-devices.yml | 11 ------- .../tasks/0063-networkmanager-dns.yml | 8 ++--- roles/kubernetes/preinstall/tasks/main.yml | 6 ++-- scale.yml | 5 ++-- upgrade-cluster.yml | 5 ++-- 10 files changed, 46 insertions(+), 30 deletions(-) diff --git a/cluster.yml b/cluster.yml index e13575e9c..cc169f80b 100644 --- a/cluster.yml +++ b/cluster.yml @@ -118,7 +118,8 @@ - { role: kubernetes-apps/external_provisioner, tags: external-provisioner } - { role: kubernetes-apps, tags: apps } -- hosts: k8s_cluster +- name: Apply resolv.conf changes now that cluster DNS is up + hosts: k8s_cluster gather_facts: False any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index 10f22cb5b..fc17b79d4 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -3,6 +3,7 @@ ignore_assert_errors: false epel_enabled: false +# Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf dns_late: false common_required_pkgs: diff --git a/roles/kubernetes/preinstall/handlers/main.yml b/roles/kubernetes/preinstall/handlers/main.yml index 667465b6f..078833186 100644 --- a/roles/kubernetes/preinstall/handlers/main.yml +++ b/roles/kubernetes/preinstall/handlers/main.yml @@ -23,12 +23,11 @@ command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }} when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] -- name: Preinstall | update resolvconf for Fedora CoreOS +- name: Preinstall | update resolvconf for networkmanager command: /bin/true notify: - Preinstall | reload NetworkManager - Preinstall | reload kubelet - when: is_fedora_coreos - name: Preinstall | reload NetworkManager service: diff --git a/roles/kubernetes/preinstall/tasks/0040-set_facts.yml b/roles/kubernetes/preinstall/tasks/0040-set_facts.yml index fce7c485c..6c5ba4b89 100644 --- a/roles/kubernetes/preinstall/tasks/0040-set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/0040-set_facts.yml @@ -67,6 +67,14 @@ when: resolvconf_stat.stat.exists is defined and resolvconf_stat.stat.exists +- name: NetworkManager | Check if host has NetworkManager + # noqa 303 Should we use service_facts for this? + command: systemctl is-active --quiet NetworkManager.service + register: networkmanager_enabled + failed_when: false + changed_when: false + check_mode: false + - name: check systemd-resolved # noqa 303 Should we use service_facts for this? command: systemctl is-active systemd-resolved @@ -98,8 +106,7 @@ - name: check if early DNS configuration stage set_fact: - dns_early: >- - {%- if kubelet_configured.stat.exists -%}false{%- else -%}true{%- endif -%} + dns_early: "{{ not kubelet_configured.stat.exists }}" - name: target resolv.conf files set_fact: @@ -177,12 +184,24 @@ {{ upstream_dns_servers|default([]) }} {%- endif -%} -- name: generate nameservers to resolvconf +# This task should only run after cluster/nodelocal DNS is up, otherwise all DNS lookups will timeout +- name: generate nameservers for resolvconf, including cluster DNS set_fact: - nameserverentries: - nameserver {{ ( ( [nodelocaldns_ip] if enable_nodelocaldns else []) + coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([]) + configured_nameservers|d([])) | unique | join(',nameserver ') }} + nameserverentries: |- + {{ ( ( [nodelocaldns_ip] if enable_nodelocaldns else []) + coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([]) + configured_nameservers|d([])) | unique | join(',') }} supersede_nameserver: supersede domain-name-servers {{ ( coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(', ') }}; + when: not dns_early or dns_late + +# This task should run instead of the above task when cluster/nodelocal DNS hasn't +# been deployed yet (like scale.yml/cluster.yml) or when it's down (reset.yml) +- name: generate nameservers for resolvconf, not including cluster DNS + set_fact: + nameserverentries: |- + {{ ( nameservers|d([]) + cloud_resolver|d([]) + configured_nameservers|d([])) | unique | join(',') }} + supersede_nameserver: + supersede domain-name-servers {{ ( nameservers|d([]) + cloud_resolver|d([])) | unique | join(', ') }}; + when: dns_early and not dns_late - name: gather os specific variables include_vars: "{{ item }}" diff --git a/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml b/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml index 65b55d7fb..2759b53e1 100644 --- a/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml +++ b/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml @@ -7,9 +7,12 @@ blockinfile: path: "{{ resolvconffile }}" block: |- - {% for item in [domainentry] + [searchentries] + nameserverentries.split(',') -%} + {% for item in [domainentry] + [searchentries] -%} {{ item }} {% endfor %} + {% for item in nameserverentries.split(',') %} + nameserver {{ item }} + {% endfor %} options ndots:{{ ndots }} options timeout:2 options attempts:2 diff --git a/roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml b/roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml index 4afde1fdb..1cd56d496 100644 --- a/roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml +++ b/roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml @@ -1,18 +1,9 @@ --- -- name: NetworkManager | Check if host has NetworkManager - # noqa 303 Should we use service_facts for this? - command: systemctl is-active --quiet NetworkManager.service - register: nm_check - failed_when: false - changed_when: false - check_mode: false - - name: NetworkManager | Ensure NetworkManager conf.d dir file: path: "/etc/NetworkManager/conf.d" state: directory recurse: yes - when: nm_check.rc == 0 - name: NetworkManager | Prevent NetworkManager from managing Calico interfaces (cali*/tunl*/vxlan.calico) copy: @@ -22,7 +13,6 @@ dest: /etc/NetworkManager/conf.d/calico.conf mode: 0644 when: - - nm_check.rc == 0 - kube_network_plugin == "calico" notify: Preinstall | reload NetworkManager @@ -35,5 +25,4 @@ unmanaged-devices+=interface-name:kube-ipvs0;interface-name:nodelocaldns dest: /etc/NetworkManager/conf.d/k8s.conf mode: 0644 - when: nm_check.rc == 0 notify: Preinstall | reload NetworkManager diff --git a/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml b/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml index 50f933a2e..851f236ac 100644 --- a/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml +++ b/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml @@ -4,10 +4,10 @@ path: /etc/NetworkManager/conf.d/dns.conf section: global-dns-domain-* option: servers - value: "{{ ( coredns_server + nameservers|d([]) + cloud_resolver|d([])) | unique | join(',') }}" + value: "{{ nameserverentries }}" mode: '0600' backup: yes - notify: Preinstall | update resolvconf for Fedora CoreOS + notify: Preinstall | update resolvconf for networkmanager - name: NetworkManager | Add DNS search to NM configuration ini_file: @@ -17,7 +17,7 @@ value: "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(',') }}" mode: '0600' backup: yes - notify: Preinstall | update resolvconf for Fedora CoreOS + notify: Preinstall | update resolvconf for networkmanager - name: NetworkManager | Add DNS options to NM configuration ini_file: @@ -27,4 +27,4 @@ value: "ndots:{{ ndots }};timeout:2;attempts:2;" mode: '0600' backup: yes - notify: Preinstall | update resolvconf for Fedora CoreOS + notify: Preinstall | update resolvconf for networkmanager diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 1e3383cde..2429cd1ad 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -25,7 +25,7 @@ - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - systemd_resolved_enabled.rc != 0 - - not is_fedora_coreos + - networkmanager_enabled.rc != 0 tags: - bootstrap-os - resolvconf @@ -40,6 +40,8 @@ - resolvconf - import_tasks: 0062-networkmanager-unmanaged-devices.yml + when: + - networkmanager_enabled.rc == 0 tags: - bootstrap-os @@ -47,7 +49,7 @@ when: - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' - - is_fedora_coreos + - networkmanager_enabled.rc == 0 tags: - bootstrap-os - resolvconf diff --git a/scale.yml b/scale.yml index de1d9b5fe..089201ef4 100644 --- a/scale.yml +++ b/scale.yml @@ -99,10 +99,11 @@ - { role: kubernetes/node-label, tags: node-label } - { role: network_plugin, tags: network } -- hosts: k8s_cluster +- name: Apply resolv.conf changes now that cluster DNS is up + hosts: k8s_cluster gather_facts: False any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray-defaults } - - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf } + - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true } diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml index 010d64271..c4e1fa4c6 100644 --- a/upgrade-cluster.yml +++ b/upgrade-cluster.yml @@ -155,10 +155,11 @@ - { role: kubespray-defaults } - { role: kubernetes-apps, tags: apps } -- hosts: k8s_cluster +- name: Apply resolv.conf changes now that cluster DNS is up + hosts: k8s_cluster gather_facts: False any_errors_fatal: "{{ any_errors_fatal | default(true) }}" environment: "{{ proxy_disable_env }}" roles: - { role: kubespray-defaults } - - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf } + - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }