Merge pull request #667 from bogdando/fix_dns

Rework DNS stack to meet hostnet pods needs

docs/dns-stack.md

@@ -38,6 +38,17 @@ or `8.8.8.8`. And domain is set to the default ``dns_domain`` value as well.
 Later, the nameservers will be reconfigured to the DNS service IP that Kargo
 configures for K8s cluster.
 
+Also note, existing records will be purged from the `/etc/resolv.conf`,
+including base/head/cloud-init config files and those that come from dhclient.
+This is required for hostnet pods networking and for [kubelet to not exceed search domains
+limits](https://github.com/kubernetes/kubernetes/issues/9229).
+
+New search, nameserver records and options will be defined from the aforementioned vars:
+* Via resolvconf's head file, if resolvconf installed.
+* Via dhclient's DNS update hook.
+* Via cloud-init (CoreOS only).
+* Statically in the `/etc/resolv.conf`, if none of above is applicable.
+
 DNS configuration details
 -------------------------
 

roles/kubernetes/preinstall/handlers/main.yml

@@ -2,9 +2,10 @@
   command: /bin/true
   notify:
     - Preinstall | reload network
-    - Preinstall | update resolvconf
+    - Preinstall | reload kubelet
   when: ansible_os_family != "CoreOS"
 
+  # FIXME(bogdando) https://github.com/projectcalico/felix/issues/1185
 - name: Preinstall | reload network
   service:
     name: >-
@@ -14,14 +15,7 @@
       networking
       {%- endif %}
     state: restarted
-  when: ansible_os_family != "RedHat" and ansible_os_family != "CoreOS"
-
-- name: Preinstall | update resolvconf
-  command: /bin/true
-  notify:
-    - Preinstall | reload resolvconf
-    - Preinstall | reload kubelet
-  when: ansible_os_family != "CoreOS"
+  when: ansible_os_family != "CoreOS" and kube_network_plugin not in ['canal', 'calico']
 
 - name: Preinstall | update resolvconf for CoreOS
   command: /bin/true
@@ -30,10 +24,6 @@
     - Preinstall | reload kubelet
   when: ansible_os_family == "CoreOS"
 
-- name: Preinstall | reload resolvconf
-  command: /sbin/resolvconf -u
-  ignore_errors: true
-
 - name: Preinstall | apply resolvconf cloud-init
   command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }}
   when: ansible_os_family == "CoreOS"

roles/kubernetes/preinstall/tasks/dhclient-hooks.yml

@@ -0,0 +1,33 @@
+---
+- name: Configure dhclient to prepend nameservers and supersede search/domain
+  blockinfile:
+    block: |-
+      {% for item in [ supersede_domain, supersede_search, prepend_nameserver ] -%}
+      {{ item }}
+      {% endfor %}
+    dest: "{{dhclientconffile}}"
+    create: yes
+    state: present
+    insertbefore: BOF
+    backup: yes
+    follow: yes
+    marker: "# Ansible entries {mark}"
+  notify: Preinstall | restart network
+
+- name: Configue dhclient hooks for resolv.conf (non-RH)
+  template:
+    src: dhclient_dnsupdate.sh.j2
+    dest: "{{ dhclienthookfile }}"
+    owner: root
+    mode: 0755
+  notify: Preinstall | restart network
+  when: ansible_os_family != "RedHat"
+
+- name: Configue dhclient hooks for resolv.conf (RH-only)
+  template:
+    src: dhclient_dnsupdate_rh.sh.j2
+    dest: "{{ dhclienthookfile }}"
+    owner: root
+    mode: 0755
+  notify: Preinstall | restart network
+  when: ansible_os_family == "RedHat"

roles/kubernetes/preinstall/tasks/resolvconf.yml

@@ -1,120 +1,34 @@
 ---
-- name: check resolvconf
-  shell: which resolvconf
-  register: resolvconf
-  ignore_errors: yes
-  changed_when: false
-  tags: facts
-
-- name: check kubelet
-  stat:
-    path: "{{ bin_dir }}/kubelet"
-  register: kubelet
-  changed_when: false
-  tags: facts
-
-- name: check if early DNS configuration stage
-  set_fact:
-    dns_early: >-
-      {%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%}
-  tags: facts
-
-- name: target resolv.conf file
-  set_fact:
-    resolvconffile: >-
-      {%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%}
-  when: ansible_os_family != "CoreOS"
-  tags: facts
-
-- name: target temporary resolvconf cloud init file
-  set_fact:
-    resolvconffile: /tmp/resolveconf_cloud_init_conf
-  when: ansible_os_family == "CoreOS"
-  tags: facts
-
 - name: create temporary resolveconf cloud init file
   command: cp -f /etc/resolv.conf "{{ resolvconffile }}"
   when: ansible_os_family == "CoreOS"
 
-- name: generate search domains to resolvconf
-  set_fact:
-    searchentries:
-      "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}"
-  tags: facts
-
-- name: decide on dns server IP
-  set_fact:
-    dns_server_real: >-
-      {%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%}
-
-- name: pick dnsmasq cluster IP or default resolver
-  set_fact:
-    dnsmasq_server: |-
-      {%- if skip_dnsmasq|bool and not dns_early|bool -%}
-        {{ [ skydns_server ] + upstream_dns_servers|default([]) }}
-      {%- elif dns_early|bool -%}
-        {{ [ dns_server_real ] + upstream_dns_servers|default([]) }}
-      {%- else -%}
-        {{ [ dns_server ] }}
-      {%- endif -%}
-  tags: facts
-
-- name: generate nameservers to resolvconf
-  set_fact:
-    nameserverentries:
-      "{{ dnsmasq_server|default([]) + nameservers|default([]) }}"
-  tags: facts
-
-- name: Remove search and nameserver options from resolvconf head
+- name: Remove search/domain/nameserver options
   lineinfile:
-    dest: /etc/resolvconf/resolv.conf.d/head
+    dest: "{{item[0]}}"
     state: absent
-    regexp: "^{{ item }}.*$"
+    regexp: "^{{ item[1] }}.*$"
     backup: yes
     follow: yes
-  with_items:
-    - search
-    - nameserver
-  when: resolvconf.rc == 0
-  notify: Preinstall | update resolvconf
+  with_nested:
+    - "{{ [resolvconffile] + [base|default('')] + [head|default('')] }}"
+    - [ 'search ', 'nameserver ', 'domain ', 'options ' ]
+  notify: Preinstall | restart network
 
-- name: Remove search and nameserver options from resolvconf cloud init temporary file
-  lineinfile:
-    dest: "{{resolvconffile}}"
-    state: absent
-    regexp: "^{{ item }}.*$"
-    backup: yes
-    follow: yes
-  with_items:
-    - search
-    - nameserver
-  when: ansible_os_family == "CoreOS"
-  notify: Preinstall | update resolvconf for CoreOS
-
-- name: Add search domains to resolvconf file
-  lineinfile:
-    line: "search {{searchentries}}"
-    dest: "{{resolvconffile}}"
-    state: present
-    insertbefore: BOF
-    backup: yes
-    follow: yes
-  notify: Preinstall | update resolvconf
-
-- name: Add nameservers to resolv.conf
+- name: Add domain/search/nameservers to resolv.conf
   blockinfile:
     dest: "{{resolvconffile}}"
     block: |-
-      {% for item in nameserverentries -%}
-      nameserver {{ item }}
+      {% for item in [domainentry] + [searchentries] + nameserverentries.split(',') -%}
+      {{ item }}
       {% endfor %}
     state: present
-    insertafter: "^search default.svc.*$"
+    insertbefore: BOF
     create: yes
     backup: yes
     follow: yes
-    marker: "# Ansible nameservers {mark}"
-  notify: Preinstall | update resolvconf
+    marker: "# Ansible entries {mark}"
+  notify: Preinstall | restart network
 
 - name: Add options to resolv.conf
   lineinfile:
@@ -129,30 +43,7 @@
     - ndots:{{ ndots }}
     - timeout:2
     - attempts:2
-  notify: Preinstall | update resolvconf
-
-- name: Remove search and nameserver options from resolvconf base
-  lineinfile:
-    dest: /etc/resolvconf/resolv.conf.d/base
-    state: absent
-    regexp: "^{{ item }}.*$"
-    backup: yes
-    follow: yes
-  with_items:
-    - search
-    - nameserver
-  when: resolvconf.rc == 0
-  notify: Preinstall | update resolvconf
-
-- name: disable resolv.conf modification by dhclient
-  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755
   notify: Preinstall | restart network
-  when: ansible_os_family == "Debian"
-
-- name: disable resolv.conf modification by dhclient
-  copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x
-  notify: Preinstall | restart network
-  when: ansible_os_family == "RedHat"
 
 - name: get temporary resolveconf cloud init file content
   command: cat {{ resolvconffile }}
@@ -167,3 +58,7 @@
     mode: 0644
   notify: Preinstall | update resolvconf for CoreOS
   when: ansible_os_family == "CoreOS"
+
+- include: dhclient-hooks.yml
+  when: ansible_os_family != "CoreOS"
+  tags: [bootstrap-os, resolvconf]

roles/kubernetes/preinstall/tasks/set_facts.yml

@@ -49,6 +49,6 @@
     etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=")
 - set_fact:
     etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}"
-- set_fact:
-    default_resolver: >-
-      {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%}
+
+- include: set_resolv_facts.yml
+  tags: [bootstrap-os, resolvconf, facts]

roles/kubernetes/preinstall/tasks/set_resolv_facts.yml

@@ -0,0 +1,88 @@
+---
+- name: check resolvconf
+  shell: which resolvconf
+  register: resolvconf
+  ignore_errors: yes
+  changed_when: false
+
+- set_fact:
+    resolvconf: >-
+      {%- if resolvconf.rc == 0 -%}true{%- else -%}false{%- endif -%}
+
+- set_fact:
+    private_domains: |-
+      {% for d in [ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([]) -%}
+      {{dns_domain}}.{{d}}./{{d}}.{{d}}./com.{{d}}./
+      {%- endfor %}
+    default_resolver: >-
+      {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%}
+
+- name: check kubelet
+  stat:
+    path: "{{ bin_dir }}/kubelet"
+  register: kubelet
+  changed_when: false
+
+- name: check if early DNS configuration stage
+  set_fact:
+    dns_early: >-
+      {%- if kubelet.stat.exists -%}false{%- else -%}true{%- endif -%}
+
+- name: target resolv.conf files
+  set_fact:
+    resolvconffile: /etc/resolv.conf
+    base: >-
+      {%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/base{%- endif -%}
+    head: >-
+      {%- if resolvconf|bool -%}/etc/resolvconf/resolv.conf.d/head{%- endif -%}
+  when: ansible_os_family != "CoreOS"
+
+- name: target temporary resolvconf cloud init file (CoreOS)
+  set_fact: resolvconffile=/tmp/resolveconf_cloud_init_conf
+  when: ansible_os_family == "CoreOS"
+
+- name: target dhclient conf/hook files for Red Hat family
+  set_fact:
+    dhclientconffile: /etc/dhclient.conf
+    dhclienthookfile: /etc/dhcp/dhclient.d/zdnsupdate.sh
+  when: ansible_os_family == "RedHat"
+
+- name: target dhclient conf/hook files for Debian family
+  set_fact:
+    dhclientconffile: /etc/dhcp/dhclient.conf
+    dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate
+  when: ansible_os_family == "Debian"
+
+- name: generate search domains to resolvconf
+  set_fact:
+    searchentries:
+      search {{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}
+    domainentry:
+      domain {{ dns_domain }}
+    supersede_search:
+      supersede domain-search "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join('", "') }}";
+    supersede_domain:
+      supersede domain-name "{{ dns_domain }}";
+
+- name: decide on dns server IP
+  set_fact:
+    dns_server_real: >-
+      {%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%}
+
+- name: pick dnsmasq cluster IP or default resolver
+  set_fact:
+    dnsmasq_server: |-
+      {%- if skip_dnsmasq|bool and not dns_early|bool -%}
+        {{ [ skydns_server ] + upstream_dns_servers|default([]) }}
+      {%- elif dns_early|bool -%}
+        {{ [ dns_server_real ] + upstream_dns_servers|default([]) }}
+      {%- else -%}
+        {{ [ dns_server ] }}
+      {%- endif -%}
+
+- name: generate nameservers to resolvconf
+  set_fact:
+    nameserverentries:
+      nameserver {{( dnsmasq_server|default([]) + nameservers|default([])) | join(',nameserver ')}}
+    prepend_nameserver:
+      prepend domain-name-servers {{( dnsmasq_server|default([]) + nameservers|default([])) | join(', ') }};

roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2

@@ -0,0 +1,13 @@
+#!/bin/sh
+#
+# Prepend resolver options to /etc/resolv.conf after dhclient`
+# regenerates the file. See man (5) resolver for more details.
+#
+if [ $reason = "BOUND" ]; then
+  if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
+    RESOLV_CONF=$(cat /etc/resolv.conf)
+    OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
+
+    printf "%b\n" "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf
+  fi
+fi

roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2

@@ -0,0 +1,17 @@
+#!/bin/sh
+#
+# Prepend resolver options to /etc/resolv.conf after dhclient`
+# regenerates the file. See man (5) resolver for more details.
+#
+zdnsupdate_config() {
+  if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
+    RESOLV_CONF=$(cat /etc/resolv.conf)
+    OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
+
+    echo -e "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf
+  fi
+}
+
+zdnsupdate_restore() {
+  :
+}