diff --git a/docs/dns-stack.md b/docs/dns-stack.md index 9905eb496..54a0081c1 100644 --- a/docs/dns-stack.md +++ b/docs/dns-stack.md @@ -38,6 +38,17 @@ or `8.8.8.8`. And domain is set to the default ``dns_domain`` value as well. Later, the nameservers will be reconfigured to the DNS service IP that Kargo configures for K8s cluster. +Also note, existing records will be purged from the `/etc/resolv.conf`, +including base/head/cloud-init config files and those that come from dhclient. +This is required for hostnet pods networking and for [kubelet to not exceed search domains +limits](https://github.com/kubernetes/kubernetes/issues/9229). + +New search, nameserver records and options will be defined from the aforementioned vars: +* Via resolvconf's head file, if resolvconf installed. +* Via dhclient's DNS update hook. +* Via cloud-init (CoreOS only). +* Statically in the `/etc/resolv.conf`, if none of above is applicable. + DNS configuration details ------------------------- diff --git a/roles/kubernetes/preinstall/handlers/main.yml b/roles/kubernetes/preinstall/handlers/main.yml index e760a8d8b..481bb1a34 100644 --- a/roles/kubernetes/preinstall/handlers/main.yml +++ b/roles/kubernetes/preinstall/handlers/main.yml @@ -2,9 +2,10 @@ command: /bin/true notify: - Preinstall | reload network - - Preinstall | update resolvconf + - Preinstall | reload kubelet when: ansible_os_family != "CoreOS" + # FIXME(bogdando) https://github.com/projectcalico/felix/issues/1185 - name: Preinstall | reload network service: name: >- @@ -14,14 +15,7 @@ networking {%- endif %} state: restarted - when: ansible_os_family != "RedHat" and ansible_os_family != "CoreOS" - -- name: Preinstall | update resolvconf - command: /bin/true - notify: - - Preinstall | reload resolvconf - - Preinstall | reload kubelet - when: ansible_os_family != "CoreOS" + when: ansible_os_family != "CoreOS" or kube_network_plugin not in ['canal', 'calico'] - name: Preinstall | update resolvconf for CoreOS command: /bin/true @@ -30,10 +24,6 @@ - Preinstall | reload kubelet when: ansible_os_family == "CoreOS" -- name: Preinstall | reload resolvconf - command: /sbin/resolvconf -u - ignore_errors: true - - name: Preinstall | apply resolvconf cloud-init command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }} when: ansible_os_family == "CoreOS" diff --git a/roles/kubernetes/preinstall/tasks/dhclient-hooks.yml b/roles/kubernetes/preinstall/tasks/dhclient-hooks.yml new file mode 100644 index 000000000..914ffca92 --- /dev/null +++ b/roles/kubernetes/preinstall/tasks/dhclient-hooks.yml @@ -0,0 +1,33 @@ +--- +- name: Configure dhclient to prepend nameservers and supersede search/domain + blockinfile: + block: |- + {% for item in [ supersede_domain, supersede_search, prepend_nameserver ] -%} + {{ item }} + {% endfor %} + dest: "{{dhclientconffile}}" + create: yes + state: present + insertbefore: BOF + backup: yes + follow: yes + marker: "# Ansible entries {mark}" + notify: Preinstall | restart network + +- name: Configue dhclient hooks for resolv.conf (non-RH) + template: + src: dhclient_dnsupdate.sh.j2 + dest: "{{ dhclienthookfile }}" + owner: root + mode: 0755 + notify: Preinstall | restart network + when: ansible_os_family != "RedHat" + +- name: Configue dhclient hooks for resolv.conf (RH-only) + template: + src: dhclient_dnsupdate_rh.sh.j2 + dest: "{{ dhclienthookfile }}" + owner: root + mode: 0755 + notify: Preinstall | restart network + when: ansible_os_family == "RedHat" diff --git a/roles/kubernetes/preinstall/tasks/resolvconf.yml b/roles/kubernetes/preinstall/tasks/resolvconf.yml index f82e9ddfb..faf3fc671 100644 --- a/roles/kubernetes/preinstall/tasks/resolvconf.yml +++ b/roles/kubernetes/preinstall/tasks/resolvconf.yml @@ -1,120 +1,34 @@ --- -- name: check resolvconf - shell: which resolvconf - register: resolvconf - ignore_errors: yes - changed_when: false - tags: facts - -- name: check kubelet - stat: - path: "{{ bin_dir }}/kubelet" - register: kubelet - changed_when: false - tags: facts - -- name: check if early DNS configuration stage - set_fact: - dns_early: >- - {%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%} - tags: facts - -- name: target resolv.conf file - set_fact: - resolvconffile: >- - {%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%} - when: ansible_os_family != "CoreOS" - tags: facts - -- name: target temporary resolvconf cloud init file - set_fact: - resolvconffile: /tmp/resolveconf_cloud_init_conf - when: ansible_os_family == "CoreOS" - tags: facts - - name: create temporary resolveconf cloud init file command: cp -f /etc/resolv.conf "{{ resolvconffile }}" when: ansible_os_family == "CoreOS" -- name: generate search domains to resolvconf - set_fact: - searchentries: - "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}" - tags: facts - -- name: decide on dns server IP - set_fact: - dns_server_real: >- - {%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%} - -- name: pick dnsmasq cluster IP or default resolver - set_fact: - dnsmasq_server: |- - {%- if skip_dnsmasq|bool and not dns_early|bool -%} - {{ [ skydns_server ] + upstream_dns_servers|default([]) }} - {%- elif dns_early|bool -%} - {{ [ dns_server_real ] + upstream_dns_servers|default([]) }} - {%- else -%} - {{ [ dns_server ] }} - {%- endif -%} - tags: facts - -- name: generate nameservers to resolvconf - set_fact: - nameserverentries: - "{{ dnsmasq_server|default([]) + nameservers|default([]) }}" - tags: facts - -- name: Remove search and nameserver options from resolvconf head +- name: Remove search/domain/nameserver options lineinfile: - dest: /etc/resolvconf/resolv.conf.d/head + dest: "{{item[0]}}" state: absent - regexp: "^{{ item }}.*$" + regexp: "^{{ item[1] }}.*$" backup: yes follow: yes - with_items: - - search - - nameserver - when: resolvconf.rc == 0 - notify: Preinstall | update resolvconf + with_nested: + - "{{ [resolvconffile] + [base|default('')] + [head|default('')] }}" + - [ 'search ', 'nameserver ', 'domain ', 'options ' ] + notify: Preinstall | restart network -- name: Remove search and nameserver options from resolvconf cloud init temporary file - lineinfile: - dest: "{{resolvconffile}}" - state: absent - regexp: "^{{ item }}.*$" - backup: yes - follow: yes - with_items: - - search - - nameserver - when: ansible_os_family == "CoreOS" - notify: Preinstall | update resolvconf for CoreOS - -- name: Add search domains to resolvconf file - lineinfile: - line: "search {{searchentries}}" - dest: "{{resolvconffile}}" - state: present - insertbefore: BOF - backup: yes - follow: yes - notify: Preinstall | update resolvconf - -- name: Add nameservers to resolv.conf +- name: Add domain/search/nameservers to resolv.conf blockinfile: dest: "{{resolvconffile}}" block: |- - {% for item in nameserverentries -%} - nameserver {{ item }} + {% for item in [domainentry] + [searchentries] + nameserverentries.split(',') -%} + {{ item }} {% endfor %} state: present - insertafter: "^search default.svc.*$" + insertbefore: BOF create: yes backup: yes follow: yes - marker: "# Ansible nameservers {mark}" - notify: Preinstall | update resolvconf + marker: "# Ansible entries {mark}" + notify: Preinstall | restart network - name: Add options to resolv.conf lineinfile: @@ -129,30 +43,7 @@ - ndots:{{ ndots }} - timeout:2 - attempts:2 - notify: Preinstall | update resolvconf - -- name: Remove search and nameserver options from resolvconf base - lineinfile: - dest: /etc/resolvconf/resolv.conf.d/base - state: absent - regexp: "^{{ item }}.*$" - backup: yes - follow: yes - with_items: - - search - - nameserver - when: resolvconf.rc == 0 - notify: Preinstall | update resolvconf - -- name: disable resolv.conf modification by dhclient - copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755 notify: Preinstall | restart network - when: ansible_os_family == "Debian" - -- name: disable resolv.conf modification by dhclient - copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x - notify: Preinstall | restart network - when: ansible_os_family == "RedHat" - name: get temporary resolveconf cloud init file content command: cat {{ resolvconffile }} @@ -167,3 +58,7 @@ mode: 0644 notify: Preinstall | update resolvconf for CoreOS when: ansible_os_family == "CoreOS" + +- include: dhclient-hooks.yml + when: ansible_os_family != "CoreOS" + tags: [bootstrap-os, resolvconf] diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index 0f66acd14..cbe4d9203 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -49,6 +49,6 @@ etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=") - set_fact: etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}" -- set_fact: - default_resolver: >- - {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%} + +- include: set_resolv_facts.yml + tags: [bootstrap-os, resolvconf, facts] diff --git a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml new file mode 100644 index 000000000..288c80afe --- /dev/null +++ b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml @@ -0,0 +1,88 @@ +--- +- name: check resolvconf + shell: which resolvconf + register: resolvconf + ignore_errors: yes + changed_when: false + +- set_fact: + resolvconf: >- + {%- if resolvconf.rc == 0 -%}true{%- else -%}false{%- endif -%} + +- set_fact: + private_domains: |- + {% for d in [ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([]) -%} + {{dns_domain}}.{{d}}./{{d}}.{{d}}./com.{{d}}./ + {%- endfor %} + default_resolver: >- + {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%} + +- name: check kubelet + stat: + path: "{{ bin_dir }}/kubelet" + register: kubelet + changed_when: false + +- name: check if early DNS configuration stage + set_fact: + dns_early: >- + {%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%} + +- name: target resolv.conf files + set_fact: + resolvconffile: /etc/resolv.conf + base: >- + {%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%} + head: >- + {%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%} + when: ansible_os_family != "CoreOS" + +- name: target temporary resolvconf cloud init file (CoreOS) + set_fact: resolvconffile=/tmp/resolveconf_cloud_init_conf + when: ansible_os_family == "CoreOS" + +- name: target dhclient conf/hook files for Red Hat family + set_fact: + dhclientconffile: /etc/dhclient.conf + dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh + when: ansible_os_family == "RedHat" + +- name: target dhclient conf/hook files for Debian family + set_fact: + dhclientconffile: /etc/dhcp/dhclient.conf + dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate + when: ansible_os_family == "Debian" + +- name: generate search domains to resolvconf + set_fact: + searchentries: + search {{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }} + domainentry: + domain {{ dns_domain }} + supersede_search: + supersede domain-search "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join('", "') }}"; + supersede_domain: + supersede domain-name "{{ dns_domain }}"; + +- name: decide on dns server IP + set_fact: + dns_server_real: >- + {%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%} + +- name: pick dnsmasq cluster IP or default resolver + set_fact: + dnsmasq_server: |- + {%- if skip_dnsmasq|bool and not dns_early|bool -%} + {{ [ skydns_server ] + upstream_dns_servers|default([]) }} + {%- elif dns_early|bool -%} + {{ [ dns_server_real ] + upstream_dns_servers|default([]) }} + {%- else -%} + {{ [ dns_server ] }} + {%- endif -%} + +- name: generate nameservers to resolvconf + set_fact: + nameserverentries: + nameserver {{( dnsmasq_server|default([]) + nameservers|default([])) | join(',nameserver ')}} + prepend_nameserver: + prepend domain-name-servers {{( dnsmasq_server|default([]) + nameservers|default([])) | join(', ') }}; diff --git a/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 new file mode 100644 index 000000000..84eb23926 --- /dev/null +++ b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 @@ -0,0 +1,13 @@ +#!/bin/sh +# +# Prepend resolver options to /etc/resolv.conf after dhclient` +# regenerates the file. See man (5) resolver for more details. +# +if [ $reason = "BOUND" ]; then + if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then + RESOLV_CONF=$(cat /etc/resolv.conf) + OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}" + + printf "%b\n" "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf + fi +fi diff --git a/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 new file mode 100644 index 000000000..514863663 --- /dev/null +++ b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 @@ -0,0 +1,17 @@ +#!/bin/sh +# +# Prepend resolver options to /etc/resolv.conf after dhclient` +# regenerates the file. See man (5) resolver for more details. +# +zdnsupdate_config() { + if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then + RESOLV_CONF=$(cat /etc/resolv.conf) + OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}" + + echo -e "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf + fi +} + +zdnsupdate_restore() { + : +}