diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cc86762ae..2402552f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,6 +48,7 @@ before_script: GS_SECRET_ACCESS_KEY: $GS_SECRET ANSIBLE_KEEP_REMOTE_FILES: "1" BOOTSTRAP_OS: none + RESOLVCONF_MODE: docker_dns LOG_LEVEL: "-vv" ETCD_DEPLOYMENT: "docker" KUBELET_DEPLOYMENT: "docker" @@ -104,6 +105,7 @@ before_script: -e download_run_once=true -e download_localhost=true -e deploy_netchecker=true + -e resolvconf_mode=${RESOLVCONF_MODE} -e local_release_dir=${PWD}/downloads -e etcd_deployment_type=${ETCD_DEPLOYMENT} -e kubelet_deployment_type=${KUBELET_DEPLOYMENT} @@ -141,6 +143,7 @@ before_script: CLOUD_REGION: us-west1-b CLUSTER_MODE: separated BOOTSTRAP_OS: coreos + RESOLVCONF_MODE: host_resolvconf # This is required as long as the CoreOS stable channel uses docker < 1.12 .debian8_canal_ha_variables: &debian8_canal_ha_variables # stage: deploy-gce-part1 @@ -177,6 +180,7 @@ before_script: CLOUD_REGION: us-east1-b CLUSTER_MODE: default BOOTSTRAP_OS: coreos + RESOLVCONF_MODE: host_resolvconf # This is required as long as the CoreOS stable channel uses docker < 1.12 .rhel7_canal_sep_variables: &rhel7_canal_sep_variables # stage: deploy-gce-special diff --git a/cluster.yml b/cluster.yml index 553261063..24da63500 100644 --- a/cluster.yml +++ b/cluster.yml @@ -57,8 +57,8 @@ - hosts: k8s-cluster any_errors_fatal: true roles: - - { role: dnsmasq, tags: dnsmasq } - - { role: kubernetes/preinstall, tags: resolvconf } + - { role: dnsmasq, when: "dns_mode == 'dnsmasq_kubedns'", tags: dnsmasq } + - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf } - hosts: kube-master[0] any_errors_fatal: true diff --git a/contrib/terraform/openstack/group_vars/all.yml b/contrib/terraform/openstack/group_vars/all.yml index f49292c3f..3582c48e7 100644 --- a/contrib/terraform/openstack/group_vars/all.yml +++ b/contrib/terraform/openstack/group_vars/all.yml @@ -105,21 +105,21 @@ kube_apiserver_insecure_port: 8080 # (http) # into appropriate IP addresses. It's highly advisable to run such DNS server, # as it greatly simplifies configuration of your applications - you can use # service names instead of magic environment variables. -# You still must manually configure all your containers to use this DNS server, -# Kubernetes won't do this for you (yet). -# Do not install additional dnsmasq -skip_dnsmasq: false -# Upstream dns servers used by dnsmasq +# Can be dnsmasq_kubedns, kubedns or none +dns_mode: dnsmasq_kubedns + +# Can be docker_dns, host_resolvconf or none +resolvconf_mode: docker_dns + +## Upstream dns servers used by dnsmasq #upstream_dns_servers: # - 8.8.8.8 # - 8.8.4.4 -# -# # Use dns server : https://github.com/ansibl8s/k8s-skydns/blob/master/skydns-README.md -dns_setup: true + dns_domain: "{{ cluster_name }}" -# -# # Ip address of the kubernetes skydns service + +# Ip address of the kubernetes skydns service skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}" dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}" diff --git a/docs/dns-stack.md b/docs/dns-stack.md index 09609f602..67afb6b43 100644 --- a/docs/dns-stack.md +++ b/docs/dns-stack.md @@ -9,47 +9,14 @@ to serve as an authoritative DNS server for a given ``dns_domain`` and its Other nodes in the inventory, like external storage nodes or a separate etcd cluster node group, considered non-cluster and left up to the user to configure DNS resolve. -Note, custom ``ndots`` values affect only the dnsmasq daemon set (explained below). -While the kubedns has the ``ndots=5`` hardcoded, which is not recommended due to -[DNS performance reasons](https://github.com/kubernetes/kubernetes/issues/14051). -You can use config maps for the kubedns app to workaround the issue, which is -yet in the Kargo scope. -Additional search (sub)domains may be defined in the ``searchdomains`` -and ``ndots`` vars. And additional recursive DNS resolvers in the `` upstream_dns_servers``, -``nameservers`` vars. Intranet/cloud provider DNS resolvers should be specified -in the first place, followed by external resolvers, for example: +DNS variables +============= -``` -skip_dnsmasq: true -nameservers: [8.8.8.8] -upstream_dns_servers: [172.18.32.6] -``` -or -``` -skip_dnsmasq: false -upstream_dns_servers: [172.18.32.6, 172.18.32.7, 8.8.8.8, 8.8.8.4] -``` -The vars are explained below. For the early cluster deployment stage, when there -is yet K8s cluster and apps exist, a user may expect local repos to be -accessible via authoritative intranet resolvers. For that case, if none custom vars -was specified, the default resolver is set to either the cloud provider default -or `8.8.8.8`. And domain is set to the default ``dns_domain`` value as well. -Later, the nameservers will be reconfigured to the DNS service IP that Kargo -configures for K8s cluster. +There are several global variables which can be used to modify DNS settings: -Also note, existing records will be purged from the `/etc/resolv.conf`, -including resolvconf's base/head/cloud-init config files and those that come from dhclient. -This is required for hostnet pods networking and for [kubelet to not exceed search domains -limits](https://github.com/kubernetes/kubernetes/issues/9229). - -Instead, new domain, search, nameserver records and options will be defined from the -aforementioned vars: -* Superseded via dhclient's DNS update hook. -* Generated via cloud-init (CoreOS only). -* Statically defined in the `/etc/resolv.conf`, if none of above is applicable. -* Resolvconf's head/base files are disabled from populating anything into the - `/etc/resolv.conf`. +#### ndots +ndots value to be used in ``/etc/resolv.conf`` It is important to note that multiple search domains combined with high ``ndots`` values lead to poor performance of DNS stack, so please choose it wisely. @@ -58,48 +25,97 @@ replies for [bogus internal FQDNS](https://github.com/kubernetes/kubernetes/issu before it even hits the kubedns app. This enables dnsmasq to serve as a protective, but still recursive resolver in front of kubedns. -DNS configuration details -------------------------- +#### searchdomains +Custom search domains to be added in addition to the cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``). -Here is an approximate picture of how DNS things working and -being configured by Kargo ansible playbooks: +Most Linux systems limit the total number of search domains to 6 and the total length of all search domains +to 256 characters. Depending on the length of ``dns_domain``, you're limitted to less then the total limit. -![Image](figures/dns.jpeg?raw=true) +Please note that ``resolvconf_mode: docker_dns`` will automatically add your systems search domains as +additional search domains. Please take this into the accounts for the limits. -Note that an additional dnsmasq daemon set is installed by Kargo -by default. Kubelet will configure DNS base of all pods to use the -given dnsmasq cluster IP, which is defined via the ``dns_server`` var. -The dnsmasq forwards requests for a given cluster ``dns_domain`` to -Kubedns's SkyDns service. The SkyDns server is configured to be an -authoritative DNS server for the given cluser domain (and its subdomains -up to ``ndots:5`` depth). Note: you should scale its replication controller -up, if SkyDns chokes. These two layered DNS forwarders provide HA for the -DNS cluster IP endpoint, which is a critical moving part for Kubernetes apps. +#### nameservers +This variable is only used by ``resolvconf_mode: host_resolvconf``. These nameservers are added to the hosts +``/etc/resolv.conf`` *after* ``upstream_dns_servers`` and thus serve as backup nameservers. If this variable +is not set, a default resolver is chosen (depending on cloud provider or 8.8.8.8 when no cloud provider is specified). -Nameservers are as well configured in the hosts' ``/etc/resolv.conf`` files, -as the given DNS cluster IP merged with ``nameservers`` values. While the -DNS cluster IP merged with the ``upstream_dns_servers`` defines additional -nameservers for the aforementioned nsmasq daemon set running on all hosts. -This mitigates existing Linux limitation of max 3 nameservers in the -``/etc/resolv.conf`` and also brings an additional caching layer for the -clustered DNS services. +#### upstream_dns_servers +DNS servers to be added *after* the cluster DNS. Used by all ``resolvconf_mode`` modes. These serve as backup +DNS servers in early cluster deployment when no cluster DNS is available yet. These are also added as upstream +DNS servers used by ``dnsmasq`` (when deployed with ``dns_mode: dnsmasq_kubedns``). -You can skip the dnsmasq daemon set install steps by setting the -``skip_dnsmasq: true``. This may be the case, if you're fine with -the nameservers limitation. Sadly, there is no way to work around the -search domain limitations of a 256 chars and 6 domains. Thus, you can -use the ``searchdomains`` var to define no more than a three custom domains. -Remaining three slots are reserved for K8s cluster default subdomains. +DNS modes supported by kargo +============================ + +You can modify how kargo sets up DNS for your cluster with the variables ``dns_mode`` and ``resolvconf_mode``. + +## dns_mode +``dns_mode`` configures how kargo will setup cluster DNS. There are three modes available: + +#### dnsmasq_kubedns (default) +This installs an additional dnsmasq DaemonSet which gives more flexibility and lifts some +limitations (e.g. number of nameservers). Kubelet is instructed to use dnsmasq instead of kubedns/skydns. +It is configured to forward all DNS queries belonging to cluster services to kubedns/skydns. All +other queries are forwardet to the nameservers found in ``upstream_dns_servers`` or ``default_resolver`` + +#### kubedns +This does not install the dnsmasq DaemonSet and instructs kubelet to directly use kubedns/skydns for +all queries. + +#### none +This does not install any of dnsmasq and kubedns/skydns. This basically disables cluster DNS completely and +leaves you with a non functional cluster. + +## resolvconf_mode +``resolvconf_mode`` configures how kargo will setup DNS for ``hostNetwork: true`` PODs and non-k8s containers. +There are three modes available: + +#### docker_dns (default) +This sets up the docker daemon with additional --dns/--dns-search/--dns-opt flags. + +The following nameservers are added to the docker daemon (in the same order as listed here): +* cluster nameserver (depends on dns_mode) +* content of optional upstream_dns_servers variable +* host system nameservers (read from hosts /etc/resolv.conf) + +The following search domains are added to the docker daemon (in the same order as listed here): +* cluster domains (``default.svc.{{ dns_domain }}``, ``svc.{{ dns_domain }}``) +* content of optional searchdomains variable +* host system search domains (read from hosts /etc/resolv.conf) + +The following dns options are added to the docker daemon +* ndots:{{ ndots }} +* timeout:2 +* attempts:2 + +For normal PODs, k8s will ignore these options and setup its own DNS settings for the PODs, taking +the --cluster_dns (either dnsmasq or kubedns, depending on dns_mode) kubelet option into account. +For ``hostNetwork: true`` PODs however, k8s will let docker setup DNS settings. Docker containers which +are not started/managed by k8s will also use these docker options. + +The host system name servers are added to ensure name resolution is also working while cluster DNS is not +running yet. This is especially important in early stages of cluster deployment. In this early stage, +DNS queries to the cluster DNS will timeout after a few seconds, resulting in the system nameserver being +used as a backup nameserver. After cluster DNS is running, all queries will be answered by the cluster DNS +servers, which in turn will forward queries to the system nameserver if required. + +#### host_resolvconf +This activates the classic kargo behaviour that modifies the hosts ``/etc/resolv.conf`` file and dhclient +configuration to point to the cluster dns server (either dnsmasq or kubedns, depending on dns_mode). + +As cluster DNS is not available on early deployment stage, this mode is split into 2 stages. In the first +stage (``dns_early: true``), ``/etc/resolv.conf`` is configured to use the DNS servers found in ``upstream_dns_servers`` +and ``nameservers``. Later, ``/etc/resolv.conf`` is reconfigured to use the cluster DNS server first, leaving +the other nameservers as backups. + +Also note, existing records will be purged from the `/etc/resolv.conf`, +including resolvconf's base/head/cloud-init config files and those that come from dhclient. + +#### none +Does nothing regarding ``/etc/resolv.conf``. This leaves you with a cluster that works as expected in most cases. +The only exception is that ``hostNetwork: true`` PODs and non-k8s managed containers will not be able to resolve +cluster service names. -When dnsmasq skipped, Kargo redefines the DNS cluster IP to point directly -to SkyDns cluster IP ``skydns_server`` and configures Kubelet's -``--dns_cluster`` to use that IP as well. While this greatly simplifies -things, it comes by the price of limited nameservers though. As you know now, -the DNS cluster IP takes a slot in the ``/etc/resolv.conf``, thus you can -specify no more than a two nameservers for infra and/or external use. -Those may be specified either in ``nameservers`` or ``upstream_dns_servers`` -and will be merged together with the ``skydns_server`` IP into the hots' -``/etc/resolv.conf``. Limitations ----------- diff --git a/docs/figures/dns.jpeg b/docs/figures/dns.jpeg deleted file mode 100644 index 56986252d..000000000 Binary files a/docs/figures/dns.jpeg and /dev/null differ diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 90dabdeca..308922a99 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -133,21 +133,21 @@ kube_apiserver_insecure_port: 8080 # (http) # into appropriate IP addresses. It's highly advisable to run such DNS server, # as it greatly simplifies configuration of your applications - you can use # service names instead of magic environment variables. -# You still must manually configure all your containers to use this DNS server, -# Kubernetes won't do this for you (yet). -# Do not install additional dnsmasq -skip_dnsmasq: false -# Upstream dns servers used by dnsmasq +# Can be dnsmasq_kubedns, kubedns or none +dns_mode: dnsmasq_kubedns + +# Can be docker_dns, host_resolvconf or none +resolvconf_mode: docker_dns + +## Upstream dns servers used by dnsmasq #upstream_dns_servers: # - 8.8.8.8 # - 8.8.4.4 -# -# # Use dns server : https://github.com/ansibl8s/k8s-skydns/blob/master/skydns-README.md -dns_setup: true + dns_domain: "{{ cluster_name }}" -# -# # Ip address of the kubernetes skydns service + +# Ip address of the kubernetes skydns service skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}" dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}" diff --git a/roles/dnsmasq/defaults/main.yml b/roles/dnsmasq/defaults/main.yml index b867820bf..d1cc17f1d 100644 --- a/roles/dnsmasq/defaults/main.yml +++ b/roles/dnsmasq/defaults/main.yml @@ -18,9 +18,6 @@ dnsmasq_version: 2.72 dnsmasq_image_repo: "andyshinn/dnsmasq" dnsmasq_image_tag: "{{ dnsmasq_version }}" -# Skip dnsmasq setup -skip_dnsmasq: false - # Limits for dnsmasq/kubedns apps dns_cpu_limit: 100m dns_memory_limit: 170Mi diff --git a/roles/dnsmasq/meta/main.yml b/roles/dnsmasq/meta/main.yml index 240fbab9e..290754e33 100644 --- a/roles/dnsmasq/meta/main.yml +++ b/roles/dnsmasq/meta/main.yml @@ -2,5 +2,5 @@ dependencies: - role: download file: "{{ downloads.dnsmasq }}" - when: not skip_dnsmasq|default(false) and download_localhost|default(false) + when: dns_mode == 'dnsmasq_kubedns' and download_localhost|default(false) tags: [download, dnsmasq] diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index 4494ac611..845f0ab35 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -14,6 +14,10 @@ skip: true tags: facts +- include: set_facts_dns.yml + when: dns_mode != 'none' and resolvconf_mode == 'docker_dns' + tags: facts + - name: check for minimum kernel version fail: msg: > @@ -63,6 +67,13 @@ with_items: "{{ docker_package_info.pkgs }}" when: (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]) and (docker_package_info.pkgs|length > 0) +- name: check minimum docker version for docker_dns mode. You need at least docker version >= 1.12 for resolvconf_mode=docker_dns + shell: docker version -f "{{ '{{' }}.Client.Version{{ '}}' }}" + register: docker_version + failed_when: docker_version.stdout|version_compare('1.12', '<') + changed_when: false + when: dns_mode != 'none' and resolvconf_mode == 'docker_dns' + - name: Set docker systemd config include: systemd.yml diff --git a/roles/docker/tasks/set_facts_dns.yml b/roles/docker/tasks/set_facts_dns.yml new file mode 100644 index 000000000..647d29829 --- /dev/null +++ b/roles/docker/tasks/set_facts_dns.yml @@ -0,0 +1,61 @@ +--- + +- name: set dns server for docker + set_fact: + docker_dns_servers: |- + {%- if dns_mode == 'kubedns' -%} + {{ [ skydns_server ] }} + {%- elif dns_mode == 'dnsmasq_kubedns' -%} + {{ [ dns_server ] }} + {%- endif -%} + +- name: set base docker dns facts + set_fact: + docker_dns_search_domains: + - 'default.svc.{{ dns_domain }}' + - 'svc.{{ dns_domain }}' + docker_dns_options: + - ndots:{{ ndots }} + - timeout:2 + - attempts:2 + +- name: add upstream dns servers (only when dnsmasq is not used) + set_fact: + docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers|default([]) }}" + when: dns_mode == 'kubedns' + +- name: add global searchdomains + set_fact: + docker_dns_search_domains: "{{ docker_dns_search_domains + searchdomains|default([]) }}" + +- name: check system nameservers + shell: grep "^nameserver" /etc/resolv.conf | sed 's/^nameserver\s*//' + changed_when: False + register: system_nameservers + +- name: check system search domains + shell: grep "^search" /etc/resolv.conf | sed 's/^search\s*//' + changed_when: False + register: system_search_domains + +- name: add system nameservers to docker options + set_fact: + docker_dns_servers: "{{ docker_dns_servers + [item] }}" + with_items: "{{ system_nameservers.stdout_lines|default([]) }}" + +- name: add system search domains to docker options + set_fact: + docker_dns_search_domains: "{{ docker_dns_search_domains + [item] }}" + with_items: "{{ system_search_domains.stdout.split(' ') }}" + +- name: check number of nameservers + fail: msg="Too many nameservers" + when: docker_dns_servers|length > 3 + +- name: check number of search domains + fail: msg="Too many search domains" + when: docker_dns_search_domains|length > 6 + +- name: check length of search domains + fail: msg="Search domains exceeded limit of 256 characters" + when: docker_dns_search_domains|join(' ')|length > 256 diff --git a/roles/docker/tasks/systemd.yml b/roles/docker/tasks/systemd.yml index 090641f85..a107ab462 100644 --- a/roles/docker/tasks/systemd.yml +++ b/roles/docker/tasks/systemd.yml @@ -21,4 +21,11 @@ dest: "/etc/systemd/system/docker.service.d/docker-options.conf" notify: restart docker +- name: Write docker dns systemd drop-in + template: + src: docker-dns.conf.j2 + dest: "/etc/systemd/system/docker.service.d/docker-dns.conf" + notify: restart docker + when: dns_mode != 'none' and resolvconf_mode == 'docker_dns' + - meta: flush_handlers \ No newline at end of file diff --git a/roles/docker/templates/docker-dns.conf.j2 b/roles/docker/templates/docker-dns.conf.j2 new file mode 100644 index 000000000..01dbd3b20 --- /dev/null +++ b/roles/docker/templates/docker-dns.conf.j2 @@ -0,0 +1,6 @@ +[Service] +Environment="DOCKER_DNS_OPTIONS=\ + {% for d in docker_dns_servers %}--dns {{ d }} {% endfor %} \ + {% for d in docker_dns_search_domains %}--dns-search {{ d }} {% endfor %} \ + {% for o in docker_dns_options %}--dns-opt {{ o }} {% endfor %} \ +" diff --git a/roles/docker/templates/docker.service.j2 b/roles/docker/templates/docker.service.j2 index 6838868cd..54e4b7c06 100644 --- a/roles/docker/templates/docker.service.j2 +++ b/roles/docker/templates/docker.service.j2 @@ -22,6 +22,7 @@ ExecStart={{ docker_bin_dir }}/docker daemon \ $DOCKER_OPTS \ $DOCKER_STORAGE_OPTIONS \ $DOCKER_NETWORK_OPTIONS \ + $DOCKER_DNS_OPTIONS \ $INSECURE_REGISTRY TasksMax=infinity LimitNOFILE=1048576 diff --git a/roles/kubernetes-apps/ansible/tasks/main.yaml b/roles/kubernetes-apps/ansible/tasks/main.yaml index d9a548949..b693055a5 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yaml +++ b/roles/kubernetes-apps/ansible/tasks/main.yaml @@ -12,7 +12,7 @@ - {file: kubedns-rc.yml, type: rc} - {file: kubedns-svc.yml, type: svc} register: manifests - when: inventory_hostname == groups['kube-master'][0] + when: dns_mode != 'none' and inventory_hostname == groups['kube-master'][0] tags: dnsmasq - name: Kubernetes Apps | Start Resources @@ -24,7 +24,7 @@ filename: "{{kube_config_dir}}/{{item.item.file}}" state: "{{item.changed | ternary('latest','present') }}" with_items: "{{ manifests.results }}" - when: inventory_hostname == groups['kube-master'][0] + when: dns_mode != 'none' and inventory_hostname == groups['kube-master'][0] tags: dnsmasq - include: tasks/calico-policy-controller.yml diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index 1de3ba966..65f98d432 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -12,9 +12,9 @@ KUBELET_HOSTNAME="--hostname-override={{ ansible_hostname }}" {% set kubelet_args_base %}--pod-manifest-path={{ kube_manifest_dir }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}{% endset %} {# DNS settings for kubelet #} -{% if dns_setup|bool and skip_dnsmasq|bool %} +{% if dns_mode == 'kubedns' %} {% set kubelet_args_cluster_dns %}--cluster_dns={{ skydns_server }}{% endset %} -{% elif dns_setup|bool %} +{% elif dns_mode == 'dnsmasq_kubedns' %} {% set kubelet_args_cluster_dns %}--cluster_dns={{ dns_server }}{% endset %} {% else %} {% set kubelet_args_cluster_dns %}{% endset %} diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 0423ab93f..a5a706da4 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -172,6 +172,7 @@ tags: [bootstrap-os, etchosts] - include: resolvconf.yml + when: dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' tags: [bootstrap-os, resolvconf] - name: Check if we are running inside a Azure VM diff --git a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml index 52f17600a..174ec29a8 100644 --- a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml @@ -67,7 +67,7 @@ - name: pick dnsmasq cluster IP or default resolver set_fact: dnsmasq_server: |- - {%- if skip_dnsmasq|bool and not dns_early|bool -%} + {%- if dns_mode == 'kubedns' and not dns_early|bool -%} {{ [ skydns_server ] + upstream_dns_servers|default([]) }} {%- elif dns_early|bool -%} {{ upstream_dns_servers|default([]) }} diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index 93f13fb5f..4bc2b2372 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: reset | stop services - service: name={{item}} state=stopped + service: name={{ item }} state=stopped with_items: - kubelet - etcd @@ -16,13 +16,26 @@ - etcd register: services_removed +- name: reset | remove docker dropins + file: + path: "/etc/systemd/system/docker.service.d/{{ item }}" + state: absent + with_items: + - docker-dns.conf + - docker-options.conf + register: docker_dropins_removed + - name: reset | systemctl daemon-reload command: systemctl daemon-reload - when: services_removed.changed + when: services_removed.changed or docker_dropins_removed.changed - name: reset | remove all containers shell: "{{ docker_bin_dir }}/docker ps -aq | xargs -r docker rm -fv" +- name: reset | restart docker if needed + service: name=docker state=restarted + when: docker_dropins_removed.changed + - name: reset | gather mounted kubelet dirs shell: mount | grep /var/lib/kubelet | awk '{print $3}' | tac register: mounted_dirs @@ -42,6 +55,40 @@ - /etc/cni - /etc/nginx - /etc/dnsmasq.d + - /etc/dnsmasq.conf + - /etc/dnsmasq.d-available - /etc/etcd.env - /etc/calico - /opt/cni + - /etc/dhcp/dhclient.d/zdnsupdate.sh + - /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate + - "{{ bin_dir }}/kubelet" + +- name: reset | remove dns settings from dhclient.conf + blockinfile: + dest: "{{ item }}" + state: absent + follow: yes + marker: "# Ansible entries {mark}" + failed_when: false + with_items: + - /etc/dhclient.conf + - /etc/dhcp/dhclient.conf + +- name: reset | remove host entries from /etc/hosts + blockinfile: + dest: "/etc/hosts" + state: absent + follow: yes + marker: "# Ansible inventory hosts {mark}" + +- name: reset | Restart network + service: + name: >- + {% if ansible_os_family == "RedHat" -%} + network + {%- elif ansible_os_family == "Debian" -%} + networking + {%- endif %} + state: restarted + when: ansible_os_family != "CoreOS"