Rework DNS stack to meet hostnet pods needs
* For Debian/RedHat OS families (with NetworkManager/dhclient/resolvconf optionally enabled) prepend /etc/resolv.conf with required nameservers, options, and supersede domain and search domains via the dhclient/resolvconf hooks. * Drop (z)nodnsupdate dhclient hook and re-implement it to complement the resolvconf -u command, which is distro/cloud provider specific. Update docs as well. * Enable network restart to apply and persist changes and simplify handlers to rely on network restart only. This fixes DNS resolve for hostnet K8s pods for Red Hat OS family. Skip network restart for canal/calico plugins, unless https://github.com/projectcalico/felix/issues/1185 fixed. * Replace linefiles line plus with_items to block mode as it's faster. Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com> Co-authored-by: Matthew Mosesohn <mmosesohn@mirantis.com>
This commit is contained in:
Bogdan Dobrelya
parent
e5ad0836bc
commit
3117858dcd
8 changed files with 185 additions and 138 deletions
|
|
@ -38,6 +38,17 @@ or `8.8.8.8`. And domain is set to the default ``dns_domain`` value as well.
|
|||
Later, the nameservers will be reconfigured to the DNS service IP that Kargo
|
||||
configures for K8s cluster.
|
||||
|
||||
Also note, existing records will be purged from the `/etc/resolv.conf`,
|
||||
including base/head/cloud-init config files and those that come from dhclient.
|
||||
This is required for hostnet pods networking and for [kubelet to not exceed search domains
|
||||
limits](https://github.com/kubernetes/kubernetes/issues/9229).
|
||||
|
||||
New search, nameserver records and options will be defined from the aforementioned vars:
|
||||
* Via resolvconf's head file, if resolvconf installed.
|
||||
* Via dhclient's DNS update hook.
|
||||
* Via cloud-init (CoreOS only).
|
||||
* Statically in the `/etc/resolv.conf`, if none of above is applicable.
|
||||
|
||||
DNS configuration details
|
||||
-------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@
|
|||
command: /bin/true
|
||||
notify:
|
||||
- Preinstall | reload network
|
||||
- Preinstall | update resolvconf
|
||||
- Preinstall | reload kubelet
|
||||
when: ansible_os_family != "CoreOS"
|
||||
|
||||
# FIXME(bogdando) https://github.com/projectcalico/felix/issues/1185
|
||||
- name: Preinstall | reload network
|
||||
service:
|
||||
name: >-
|
||||
|
|
@ -14,14 +15,7 @@
|
|||
networking
|
||||
{%- endif %}
|
||||
state: restarted
|
||||
when: ansible_os_family != "RedHat" and ansible_os_family != "CoreOS"
|
||||
|
||||
- name: Preinstall | update resolvconf
|
||||
command: /bin/true
|
||||
notify:
|
||||
- Preinstall | reload resolvconf
|
||||
- Preinstall | reload kubelet
|
||||
when: ansible_os_family != "CoreOS"
|
||||
when: ansible_os_family != "CoreOS" or kube_network_plugin not in ['canal', 'calico']
|
||||
|
||||
- name: Preinstall | update resolvconf for CoreOS
|
||||
command: /bin/true
|
||||
|
|
@ -30,10 +24,6 @@
|
|||
- Preinstall | reload kubelet
|
||||
when: ansible_os_family == "CoreOS"
|
||||
|
||||
- name: Preinstall | reload resolvconf
|
||||
command: /sbin/resolvconf -u
|
||||
ignore_errors: true
|
||||
|
||||
- name: Preinstall | apply resolvconf cloud-init
|
||||
command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }}
|
||||
when: ansible_os_family == "CoreOS"
|
||||
|
|
|
|||
33
roles/kubernetes/preinstall/tasks/dhclient-hooks.yml
Normal file
33
roles/kubernetes/preinstall/tasks/dhclient-hooks.yml
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
---
|
||||
- name: Configure dhclient to prepend nameservers and supersede search/domain
|
||||
blockinfile:
|
||||
block: |-
|
||||
{% for item in [ supersede_domain, supersede_search, prepend_nameserver ] -%}
|
||||
{{ item }}
|
||||
{% endfor %}
|
||||
dest: "{{dhclientconffile}}"
|
||||
create: yes
|
||||
state: present
|
||||
insertbefore: BOF
|
||||
backup: yes
|
||||
follow: yes
|
||||
marker: "# Ansible entries {mark}"
|
||||
notify: Preinstall | restart network
|
||||
|
||||
- name: Configue dhclient hooks for resolv.conf (non-RH)
|
||||
template:
|
||||
src: dhclient_dnsupdate.sh.j2
|
||||
dest: "{{ dhclienthookfile }}"
|
||||
owner: root
|
||||
mode: 0755
|
||||
notify: Preinstall | restart network
|
||||
when: ansible_os_family != "RedHat"
|
||||
|
||||
- name: Configue dhclient hooks for resolv.conf (RH-only)
|
||||
template:
|
||||
src: dhclient_dnsupdate_rh.sh.j2
|
||||
dest: "{{ dhclienthookfile }}"
|
||||
owner: root
|
||||
mode: 0755
|
||||
notify: Preinstall | restart network
|
||||
when: ansible_os_family == "RedHat"
|
||||
|
|
@ -1,120 +1,34 @@
|
|||
---
|
||||
- name: check resolvconf
|
||||
shell: which resolvconf
|
||||
register: resolvconf
|
||||
ignore_errors: yes
|
||||
changed_when: false
|
||||
tags: facts
|
||||
|
||||
- name: check kubelet
|
||||
stat:
|
||||
path: "{{ bin_dir }}/kubelet"
|
||||
register: kubelet
|
||||
changed_when: false
|
||||
tags: facts
|
||||
|
||||
- name: check if early DNS configuration stage
|
||||
set_fact:
|
||||
dns_early: >-
|
||||
{%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%}
|
||||
tags: facts
|
||||
|
||||
- name: target resolv.conf file
|
||||
set_fact:
|
||||
resolvconffile: >-
|
||||
{%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
|
||||
when: ansible_os_family != "CoreOS"
|
||||
tags: facts
|
||||
|
||||
- name: target temporary resolvconf cloud init file
|
||||
set_fact:
|
||||
resolvconffile: /tmp/resolveconf_cloud_init_conf
|
||||
when: ansible_os_family == "CoreOS"
|
||||
tags: facts
|
||||
|
||||
- name: create temporary resolveconf cloud init file
|
||||
command: cp -f /etc/resolv.conf "{{ resolvconffile }}"
|
||||
when: ansible_os_family == "CoreOS"
|
||||
|
||||
- name: generate search domains to resolvconf
|
||||
set_fact:
|
||||
searchentries:
|
||||
"{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
|
||||
tags: facts
|
||||
|
||||
- name: decide on dns server IP
|
||||
set_fact:
|
||||
dns_server_real: >-
|
||||
{%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%}
|
||||
|
||||
- name: pick dnsmasq cluster IP or default resolver
|
||||
set_fact:
|
||||
dnsmasq_server: |-
|
||||
{%- if skip_dnsmasq|bool and not dns_early|bool -%}
|
||||
{{ [ skydns_server ] + upstream_dns_servers|default([]) }}
|
||||
{%- elif dns_early|bool -%}
|
||||
{{ [ dns_server_real ] + upstream_dns_servers|default([]) }}
|
||||
{%- else -%}
|
||||
{{ [ dns_server ] }}
|
||||
{%- endif -%}
|
||||
tags: facts
|
||||
|
||||
- name: generate nameservers to resolvconf
|
||||
set_fact:
|
||||
nameserverentries:
|
||||
"{{ dnsmasq_server|default([]) + nameservers|default([]) }}"
|
||||
tags: facts
|
||||
|
||||
- name: Remove search and nameserver options from resolvconf head
|
||||
- name: Remove search/domain/nameserver options
|
||||
lineinfile:
|
||||
dest: /etc/resolvconf/resolv.conf.d/head
|
||||
dest: "{{item[0]}}"
|
||||
state: absent
|
||||
regexp: "^{{ item }}.*$"
|
||||
regexp: "^{{ item[1] }}.*$"
|
||||
backup: yes
|
||||
follow: yes
|
||||
with_items:
|
||||
- search
|
||||
- nameserver
|
||||
when: resolvconf.rc == 0
|
||||
notify: Preinstall | update resolvconf
|
||||
with_nested:
|
||||
- "{{ [resolvconffile] + [base|default('')] + [head|default('')] }}"
|
||||
- [ 'search ', 'nameserver ', 'domain ', 'options ' ]
|
||||
notify: Preinstall | restart network
|
||||
|
||||
- name: Remove search and nameserver options from resolvconf cloud init temporary file
|
||||
lineinfile:
|
||||
dest: "{{resolvconffile}}"
|
||||
state: absent
|
||||
regexp: "^{{ item }}.*$"
|
||||
backup: yes
|
||||
follow: yes
|
||||
with_items:
|
||||
- search
|
||||
- nameserver
|
||||
when: ansible_os_family == "CoreOS"
|
||||
notify: Preinstall | update resolvconf for CoreOS
|
||||
|
||||
- name: Add search domains to resolvconf file
|
||||
lineinfile:
|
||||
line: "search {{searchentries}}"
|
||||
dest: "{{resolvconffile}}"
|
||||
state: present
|
||||
insertbefore: BOF
|
||||
backup: yes
|
||||
follow: yes
|
||||
notify: Preinstall | update resolvconf
|
||||
|
||||
- name: Add nameservers to resolv.conf
|
||||
- name: Add domain/search/nameservers to resolv.conf
|
||||
blockinfile:
|
||||
dest: "{{resolvconffile}}"
|
||||
block: |-
|
||||
{% for item in nameserverentries -%}
|
||||
nameserver {{ item }}
|
||||
{% for item in [domainentry] + [searchentries] + nameserverentries.split(',') -%}
|
||||
{{ item }}
|
||||
{% endfor %}
|
||||
state: present
|
||||
insertafter: "^search default.svc.*$"
|
||||
insertbefore: BOF
|
||||
create: yes
|
||||
backup: yes
|
||||
follow: yes
|
||||
marker: "# Ansible nameservers {mark}"
|
||||
notify: Preinstall | update resolvconf
|
||||
marker: "# Ansible entries {mark}"
|
||||
notify: Preinstall | restart network
|
||||
|
||||
- name: Add options to resolv.conf
|
||||
lineinfile:
|
||||
|
|
@ -129,30 +43,7 @@
|
|||
- ndots:{{ ndots }}
|
||||
- timeout:2
|
||||
- attempts:2
|
||||
notify: Preinstall | update resolvconf
|
||||
|
||||
- name: Remove search and nameserver options from resolvconf base
|
||||
lineinfile:
|
||||
dest: /etc/resolvconf/resolv.conf.d/base
|
||||
state: absent
|
||||
regexp: "^{{ item }}.*$"
|
||||
backup: yes
|
||||
follow: yes
|
||||
with_items:
|
||||
- search
|
||||
- nameserver
|
||||
when: resolvconf.rc == 0
|
||||
notify: Preinstall | update resolvconf
|
||||
|
||||
- name: disable resolv.conf modification by dhclient
|
||||
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
|
||||
notify: Preinstall | restart network
|
||||
when: ansible_os_family == "Debian"
|
||||
|
||||
- name: disable resolv.conf modification by dhclient
|
||||
copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
|
||||
notify: Preinstall | restart network
|
||||
when: ansible_os_family == "RedHat"
|
||||
|
||||
- name: get temporary resolveconf cloud init file content
|
||||
command: cat {{ resolvconffile }}
|
||||
|
|
@ -167,3 +58,7 @@
|
|||
mode: 0644
|
||||
notify: Preinstall | update resolvconf for CoreOS
|
||||
when: ansible_os_family == "CoreOS"
|
||||
|
||||
- include: dhclient-hooks.yml
|
||||
when: ansible_os_family != "CoreOS"
|
||||
tags: [bootstrap-os, resolvconf]
|
||||
|
|
|
|||
|
|
@ -49,6 +49,6 @@
|
|||
etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=")
|
||||
- set_fact:
|
||||
etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}"
|
||||
- set_fact:
|
||||
default_resolver: >-
|
||||
{%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%}
|
||||
|
||||
- include: set_resolv_facts.yml
|
||||
tags: [bootstrap-os, resolvconf, facts]
|
||||
|
|
|
|||
88
roles/kubernetes/preinstall/tasks/set_resolv_facts.yml
Normal file
88
roles/kubernetes/preinstall/tasks/set_resolv_facts.yml
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
---
|
||||
- name: check resolvconf
|
||||
shell: which resolvconf
|
||||
register: resolvconf
|
||||
ignore_errors: yes
|
||||
changed_when: false
|
||||
|
||||
- set_fact:
|
||||
resolvconf: >-
|
||||
{%- if resolvconf.rc == 0 -%}true{%- else -%}false{%- endif -%}
|
||||
|
||||
- set_fact:
|
||||
private_domains: |-
|
||||
{% for d in [ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([]) -%}
|
||||
{{dns_domain}}.{{d}}./{{d}}.{{d}}./com.{{d}}./
|
||||
{%- endfor %}
|
||||
default_resolver: >-
|
||||
{%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%}
|
||||
|
||||
- name: check kubelet
|
||||
stat:
|
||||
path: "{{ bin_dir }}/kubelet"
|
||||
register: kubelet
|
||||
changed_when: false
|
||||
|
||||
- name: check if early DNS configuration stage
|
||||
set_fact:
|
||||
dns_early: >-
|
||||
{%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%}
|
||||
|
||||
- name: target resolv.conf files
|
||||
set_fact:
|
||||
resolvconffile: /etc/resolv.conf
|
||||
base: >-
|
||||
{%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%}
|
||||
head: >-
|
||||
{%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%}
|
||||
when: ansible_os_family != "CoreOS"
|
||||
|
||||
- name: target temporary resolvconf cloud init file (CoreOS)
|
||||
set_fact: resolvconffile=/tmp/resolveconf_cloud_init_conf
|
||||
when: ansible_os_family == "CoreOS"
|
||||
|
||||
- name: target dhclient conf/hook files for Red Hat family
|
||||
set_fact:
|
||||
dhclientconffile: /etc/dhclient.conf
|
||||
dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh
|
||||
when: ansible_os_family == "RedHat"
|
||||
|
||||
- name: target dhclient conf/hook files for Debian family
|
||||
set_fact:
|
||||
dhclientconffile: /etc/dhcp/dhclient.conf
|
||||
dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate
|
||||
when: ansible_os_family == "Debian"
|
||||
|
||||
- name: generate search domains to resolvconf
|
||||
set_fact:
|
||||
searchentries:
|
||||
search {{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}
|
||||
domainentry:
|
||||
domain {{ dns_domain }}
|
||||
supersede_search:
|
||||
supersede domain-search "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join('", "') }}";
|
||||
supersede_domain:
|
||||
supersede domain-name "{{ dns_domain }}";
|
||||
|
||||
- name: decide on dns server IP
|
||||
set_fact:
|
||||
dns_server_real: >-
|
||||
{%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%}
|
||||
|
||||
- name: pick dnsmasq cluster IP or default resolver
|
||||
set_fact:
|
||||
dnsmasq_server: |-
|
||||
{%- if skip_dnsmasq|bool and not dns_early|bool -%}
|
||||
{{ [ skydns_server ] + upstream_dns_servers|default([]) }}
|
||||
{%- elif dns_early|bool -%}
|
||||
{{ [ dns_server_real ] + upstream_dns_servers|default([]) }}
|
||||
{%- else -%}
|
||||
{{ [ dns_server ] }}
|
||||
{%- endif -%}
|
||||
|
||||
- name: generate nameservers to resolvconf
|
||||
set_fact:
|
||||
nameserverentries:
|
||||
nameserver {{( dnsmasq_server|default([]) + nameservers|default([])) | join(',nameserver ')}}
|
||||
prepend_nameserver:
|
||||
prepend domain-name-servers {{( dnsmasq_server|default([]) + nameservers|default([])) | join(', ') }};
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/sh
|
||||
#
|
||||
# Prepend resolver options to /etc/resolv.conf after dhclient`
|
||||
# regenerates the file. See man (5) resolver for more details.
|
||||
#
|
||||
if [ $reason = "BOUND" ]; then
|
||||
if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
|
||||
RESOLV_CONF=$(cat /etc/resolv.conf)
|
||||
OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
|
||||
|
||||
printf "%b\n" "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf
|
||||
fi
|
||||
fi
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
#!/bin/sh
|
||||
#
|
||||
# Prepend resolver options to /etc/resolv.conf after dhclient`
|
||||
# regenerates the file. See man (5) resolver for more details.
|
||||
#
|
||||
zdnsupdate_config() {
|
||||
if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
|
||||
RESOLV_CONF=$(cat /etc/resolv.conf)
|
||||
OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
|
||||
|
||||
echo -e "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf
|
||||
fi
|
||||
}
|
||||
|
||||
zdnsupdate_restore() {
|
||||
:
|
||||
}
|
||||
Loading…
Reference in a new issue