From 3117858dcdf4888ce13f902edb4d7d647978516a Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Wed, 30 Nov 2016 14:06:11 +0100 Subject: [PATCH] Rework DNS stack to meet hostnet pods needs * For Debian/RedHat OS families (with NetworkManager/dhclient/resolvconf optionally enabled) prepend /etc/resolv.conf with required nameservers, options, and supersede domain and search domains via the dhclient/resolvconf hooks. * Drop (z)nodnsupdate dhclient hook and re-implement it to complement the resolvconf -u command, which is distro/cloud provider specific. Update docs as well. * Enable network restart to apply and persist changes and simplify handlers to rely on network restart only. This fixes DNS resolve for hostnet K8s pods for Red Hat OS family. Skip network restart for canal/calico plugins, unless https://github.com/projectcalico/felix/issues/1185 fixed. * Replace linefiles line plus with_items to block mode as it's faster. Signed-off-by: Bogdan Dobrelya Co-authored-by: Matthew Mosesohn --- docs/dns-stack.md | 11 ++ roles/kubernetes/preinstall/handlers/main.yml | 16 +- .../preinstall/tasks/dhclient-hooks.yml | 33 +++++ .../preinstall/tasks/resolvconf.yml | 139 +++--------------- .../kubernetes/preinstall/tasks/set_facts.yml | 6 +- .../preinstall/tasks/set_resolv_facts.yml | 88 +++++++++++ .../templates/dhclient_dnsupdate.sh.j2 | 13 ++ .../templates/dhclient_dnsupdate_rh.sh.j2 | 17 +++ 8 files changed, 185 insertions(+), 138 deletions(-) create mode 100644 roles/kubernetes/preinstall/tasks/dhclient-hooks.yml create mode 100644 roles/kubernetes/preinstall/tasks/set_resolv_facts.yml create mode 100644 roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 create mode 100644 roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 diff --git a/docs/dns-stack.md b/docs/dns-stack.md index 9905eb496..54a0081c1 100644 --- a/docs/dns-stack.md +++ b/docs/dns-stack.md @@ -38,6 +38,17 @@ or `8.8.8.8`. And domain is set to the default ``dns_domain`` value as well. Later, the nameservers will be reconfigured to the DNS service IP that Kargo configures for K8s cluster. +Also note, existing records will be purged from the `/etc/resolv.conf`, +including base/head/cloud-init config files and those that come from dhclient. +This is required for hostnet pods networking and for [kubelet to not exceed search domains +limits](https://github.com/kubernetes/kubernetes/issues/9229). + +New search, nameserver records and options will be defined from the aforementioned vars: +* Via resolvconf's head file, if resolvconf installed. +* Via dhclient's DNS update hook. +* Via cloud-init (CoreOS only). +* Statically in the `/etc/resolv.conf`, if none of above is applicable. + DNS configuration details ------------------------- diff --git a/roles/kubernetes/preinstall/handlers/main.yml b/roles/kubernetes/preinstall/handlers/main.yml index e760a8d8b..481bb1a34 100644 --- a/roles/kubernetes/preinstall/handlers/main.yml +++ b/roles/kubernetes/preinstall/handlers/main.yml @@ -2,9 +2,10 @@ command: /bin/true notify: - Preinstall | reload network - - Preinstall | update resolvconf + - Preinstall | reload kubelet when: ansible_os_family != "CoreOS" + # FIXME(bogdando) https://github.com/projectcalico/felix/issues/1185 - name: Preinstall | reload network service: name: >- @@ -14,14 +15,7 @@ networking {%- endif %} state: restarted - when: ansible_os_family != "RedHat" and ansible_os_family != "CoreOS" - -- name: Preinstall | update resolvconf - command: /bin/true - notify: - - Preinstall | reload resolvconf - - Preinstall | reload kubelet - when: ansible_os_family != "CoreOS" + when: ansible_os_family != "CoreOS" or kube_network_plugin not in ['canal', 'calico'] - name: Preinstall | update resolvconf for CoreOS command: /bin/true @@ -30,10 +24,6 @@ - Preinstall | reload kubelet when: ansible_os_family == "CoreOS" -- name: Preinstall | reload resolvconf - command: /sbin/resolvconf -u - ignore_errors: true - - name: Preinstall | apply resolvconf cloud-init command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }} when: ansible_os_family == "CoreOS" diff --git a/roles/kubernetes/preinstall/tasks/dhclient-hooks.yml b/roles/kubernetes/preinstall/tasks/dhclient-hooks.yml new file mode 100644 index 000000000..914ffca92 --- /dev/null +++ b/roles/kubernetes/preinstall/tasks/dhclient-hooks.yml @@ -0,0 +1,33 @@ +--- +- name: Configure dhclient to prepend nameservers and supersede search/domain + blockinfile: + block: |- + {% for item in [ supersede_domain, supersede_search, prepend_nameserver ] -%} + {{ item }} + {% endfor %} + dest: "{{dhclientconffile}}" + create: yes + state: present + insertbefore: BOF + backup: yes + follow: yes + marker: "# Ansible entries {mark}" + notify: Preinstall | restart network + +- name: Configue dhclient hooks for resolv.conf (non-RH) + template: + src: dhclient_dnsupdate.sh.j2 + dest: "{{ dhclienthookfile }}" + owner: root + mode: 0755 + notify: Preinstall | restart network + when: ansible_os_family != "RedHat" + +- name: Configue dhclient hooks for resolv.conf (RH-only) + template: + src: dhclient_dnsupdate_rh.sh.j2 + dest: "{{ dhclienthookfile }}" + owner: root + mode: 0755 + notify: Preinstall | restart network + when: ansible_os_family == "RedHat" diff --git a/roles/kubernetes/preinstall/tasks/resolvconf.yml b/roles/kubernetes/preinstall/tasks/resolvconf.yml index f82e9ddfb..faf3fc671 100644 --- a/roles/kubernetes/preinstall/tasks/resolvconf.yml +++ b/roles/kubernetes/preinstall/tasks/resolvconf.yml @@ -1,120 +1,34 @@ --- -- name: check resolvconf - shell: which resolvconf - register: resolvconf - ignore_errors: yes - changed_when: false - tags: facts - -- name: check kubelet - stat: - path: "{{ bin_dir }}/kubelet" - register: kubelet - changed_when: false - tags: facts - -- name: check if early DNS configuration stage - set_fact: - dns_early: >- - {%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%} - tags: facts - -- name: target resolv.conf file - set_fact: - resolvconffile: >- - {%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%} - when: ansible_os_family != "CoreOS" - tags: facts - -- name: target temporary resolvconf cloud init file - set_fact: - resolvconffile: /tmp/resolveconf_cloud_init_conf - when: ansible_os_family == "CoreOS" - tags: facts - - name: create temporary resolveconf cloud init file command: cp -f /etc/resolv.conf "{{ resolvconffile }}" when: ansible_os_family == "CoreOS" -- name: generate search domains to resolvconf - set_fact: - searchentries: - "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}" - tags: facts - -- name: decide on dns server IP - set_fact: - dns_server_real: >- - {%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%} - -- name: pick dnsmasq cluster IP or default resolver - set_fact: - dnsmasq_server: |- - {%- if skip_dnsmasq|bool and not dns_early|bool -%} - {{ [ skydns_server ] + upstream_dns_servers|default([]) }} - {%- elif dns_early|bool -%} - {{ [ dns_server_real ] + upstream_dns_servers|default([]) }} - {%- else -%} - {{ [ dns_server ] }} - {%- endif -%} - tags: facts - -- name: generate nameservers to resolvconf - set_fact: - nameserverentries: - "{{ dnsmasq_server|default([]) + nameservers|default([]) }}" - tags: facts - -- name: Remove search and nameserver options from resolvconf head +- name: Remove search/domain/nameserver options lineinfile: - dest: /etc/resolvconf/resolv.conf.d/head + dest: "{{item[0]}}" state: absent - regexp: "^{{ item }}.*$" + regexp: "^{{ item[1] }}.*$" backup: yes follow: yes - with_items: - - search - - nameserver - when: resolvconf.rc == 0 - notify: Preinstall | update resolvconf + with_nested: + - "{{ [resolvconffile] + [base|default('')] + [head|default('')] }}" + - [ 'search ', 'nameserver ', 'domain ', 'options ' ] + notify: Preinstall | restart network -- name: Remove search and nameserver options from resolvconf cloud init temporary file - lineinfile: - dest: "{{resolvconffile}}" - state: absent - regexp: "^{{ item }}.*$" - backup: yes - follow: yes - with_items: - - search - - nameserver - when: ansible_os_family == "CoreOS" - notify: Preinstall | update resolvconf for CoreOS - -- name: Add search domains to resolvconf file - lineinfile: - line: "search {{searchentries}}" - dest: "{{resolvconffile}}" - state: present - insertbefore: BOF - backup: yes - follow: yes - notify: Preinstall | update resolvconf - -- name: Add nameservers to resolv.conf +- name: Add domain/search/nameservers to resolv.conf blockinfile: dest: "{{resolvconffile}}" block: |- - {% for item in nameserverentries -%} - nameserver {{ item }} + {% for item in [domainentry] + [searchentries] + nameserverentries.split(',') -%} + {{ item }} {% endfor %} state: present - insertafter: "^search default.svc.*$" + insertbefore: BOF create: yes backup: yes follow: yes - marker: "# Ansible nameservers {mark}" - notify: Preinstall | update resolvconf + marker: "# Ansible entries {mark}" + notify: Preinstall | restart network - name: Add options to resolv.conf lineinfile: @@ -129,30 +43,7 @@ - ndots:{{ ndots }} - timeout:2 - attempts:2 - notify: Preinstall | update resolvconf - -- name: Remove search and nameserver options from resolvconf base - lineinfile: - dest: /etc/resolvconf/resolv.conf.d/base - state: absent - regexp: "^{{ item }}.*$" - backup: yes - follow: yes - with_items: - - search - - nameserver - when: resolvconf.rc == 0 - notify: Preinstall | update resolvconf - -- name: disable resolv.conf modification by dhclient - copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755 notify: Preinstall | restart network - when: ansible_os_family == "Debian" - -- name: disable resolv.conf modification by dhclient - copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x - notify: Preinstall | restart network - when: ansible_os_family == "RedHat" - name: get temporary resolveconf cloud init file content command: cat {{ resolvconffile }} @@ -167,3 +58,7 @@ mode: 0644 notify: Preinstall | update resolvconf for CoreOS when: ansible_os_family == "CoreOS" + +- include: dhclient-hooks.yml + when: ansible_os_family != "CoreOS" + tags: [bootstrap-os, resolvconf] diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index 0f66acd14..cbe4d9203 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -49,6 +49,6 @@ etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=") - set_fact: etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}" -- set_fact: - default_resolver: >- - {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%} + +- include: set_resolv_facts.yml + tags: [bootstrap-os, resolvconf, facts] diff --git a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml new file mode 100644 index 000000000..288c80afe --- /dev/null +++ b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml @@ -0,0 +1,88 @@ +--- +- name: check resolvconf + shell: which resolvconf + register: resolvconf + ignore_errors: yes + changed_when: false + +- set_fact: + resolvconf: >- + {%- if resolvconf.rc == 0 -%}true{%- else -%}false{%- endif -%} + +- set_fact: + private_domains: |- + {% for d in [ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([]) -%} + {{dns_domain}}.{{d}}./{{d}}.{{d}}./com.{{d}}./ + {%- endfor %} + default_resolver: >- + {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%} + +- name: check kubelet + stat: + path: "{{ bin_dir }}/kubelet" + register: kubelet + changed_when: false + +- name: check if early DNS configuration stage + set_fact: + dns_early: >- + {%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%} + +- name: target resolv.conf files + set_fact: + resolvconffile: /etc/resolv.conf + base: >- + {%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%} + head: >- + {%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%} + when: ansible_os_family != "CoreOS" + +- name: target temporary resolvconf cloud init file (CoreOS) + set_fact: resolvconffile=/tmp/resolveconf_cloud_init_conf + when: ansible_os_family == "CoreOS" + +- name: target dhclient conf/hook files for Red Hat family + set_fact: + dhclientconffile: /etc/dhclient.conf + dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh + when: ansible_os_family == "RedHat" + +- name: target dhclient conf/hook files for Debian family + set_fact: + dhclientconffile: /etc/dhcp/dhclient.conf + dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate + when: ansible_os_family == "Debian" + +- name: generate search domains to resolvconf + set_fact: + searchentries: + search {{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }} + domainentry: + domain {{ dns_domain }} + supersede_search: + supersede domain-search "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join('", "') }}"; + supersede_domain: + supersede domain-name "{{ dns_domain }}"; + +- name: decide on dns server IP + set_fact: + dns_server_real: >- + {%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%} + +- name: pick dnsmasq cluster IP or default resolver + set_fact: + dnsmasq_server: |- + {%- if skip_dnsmasq|bool and not dns_early|bool -%} + {{ [ skydns_server ] + upstream_dns_servers|default([]) }} + {%- elif dns_early|bool -%} + {{ [ dns_server_real ] + upstream_dns_servers|default([]) }} + {%- else -%} + {{ [ dns_server ] }} + {%- endif -%} + +- name: generate nameservers to resolvconf + set_fact: + nameserverentries: + nameserver {{( dnsmasq_server|default([]) + nameservers|default([])) | join(',nameserver ')}} + prepend_nameserver: + prepend domain-name-servers {{( dnsmasq_server|default([]) + nameservers|default([])) | join(', ') }}; diff --git a/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 new file mode 100644 index 000000000..84eb23926 --- /dev/null +++ b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 @@ -0,0 +1,13 @@ +#!/bin/sh +# +# Prepend resolver options to /etc/resolv.conf after dhclient` +# regenerates the file. See man (5) resolver for more details. +# +if [ $reason = "BOUND" ]; then + if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then + RESOLV_CONF=$(cat /etc/resolv.conf) + OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}" + + printf "%b\n" "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf + fi +fi diff --git a/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 new file mode 100644 index 000000000..514863663 --- /dev/null +++ b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 @@ -0,0 +1,17 @@ +#!/bin/sh +# +# Prepend resolver options to /etc/resolv.conf after dhclient` +# regenerates the file. See man (5) resolver for more details. +# +zdnsupdate_config() { + if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then + RESOLV_CONF=$(cat /etc/resolv.conf) + OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}" + + echo -e "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf + fi +} + +zdnsupdate_restore() { + : +}