From 9468642269ce19ca59128f8db2f847e9ef9edae0 Mon Sep 17 00:00:00 2001 From: Emin AKTAS Date: Fri, 23 Sep 2022 20:28:26 +0300 Subject: [PATCH] feat: allows users to have more control on DNS (#9270) Signed-off-by: eminaktas Signed-off-by: eminaktas --- docs/dns-stack.md | 13 ++++++++++++- docs/vars.md | 2 ++ .../sample/group_vars/k8s_cluster/k8s-cluster.yml | 8 ++++++++ .../kubernetes/preinstall/tasks/0040-set_facts.yml | 13 +++++++++---- .../kubernetes/preinstall/tasks/0060-resolvconf.yml | 4 +--- .../preinstall/tasks/0063-networkmanager-dns.yml | 9 +++++++-- .../preinstall/templates/dhclient_dnsupdate.sh.j2 | 2 +- .../templates/dhclient_dnsupdate_rh.sh.j2 | 2 +- .../preinstall/templates/resolved.conf.j2 | 4 ++++ 9 files changed, 45 insertions(+), 12 deletions(-) diff --git a/docs/dns-stack.md b/docs/dns-stack.md index 44df1785f..9d172b832 100644 --- a/docs/dns-stack.md +++ b/docs/dns-stack.md @@ -19,6 +19,14 @@ ndots value to be used in ``/etc/resolv.conf`` It is important to note that multiple search domains combined with high ``ndots`` values lead to poor performance of DNS stack, so please choose it wisely. +## dns_timeout + +timeout value to be used in ``/etc/resolv.conf`` + +## dns_attempts + +attempts value to be used in ``/etc/resolv.conf`` + ### searchdomains Custom search domains to be added in addition to the cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``). @@ -26,6 +34,8 @@ Custom search domains to be added in addition to the cluster search domains (``d Most Linux systems limit the total number of search domains to 6 and the total length of all search domains to 256 characters. Depending on the length of ``dns_domain``, you're limited to less than the total limit. +`remove_default_searchdomains: true` will remove the default cluster search domains. + Please note that ``resolvconf_mode: docker_dns`` will automatically add your systems search domains as additional search domains. Please take this into the accounts for the limits. @@ -270,7 +280,8 @@ nodelocaldns_secondary_skew_seconds: 5 * the ``searchdomains`` have a limitation of a 6 names and 256 chars length. Due to default ``svc, default.svc`` subdomains, the actual - limits are a 4 names and 239 chars respectively. + limits are a 4 names and 239 chars respectively. If `remove_default_searchdomains: true` + added you are back to 6 names. * the ``nameservers`` have a limitation of a 3 servers, although there is a way to mitigate that with the ``upstream_dns_servers``, diff --git a/docs/vars.md b/docs/vars.md index 1fde812d3..97f0a710a 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -28,6 +28,7 @@ Some variables of note include: * *kube_proxy_mode* - Changes k8s proxy mode to iptables mode * *kube_version* - Specify a given Kubernetes version * *searchdomains* - Array of DNS domains to search when looking up hostnames +* *remove_default_searchdomains* - Boolean that removes the default searchdomain * *nameservers* - Array of nameservers to use for DNS lookup * *preinstall_selinux_state* - Set selinux state, permitted values are permissive, enforcing and disabled. @@ -166,6 +167,7 @@ variables to match your requirements. addition to Kubespray deployed DNS * *nameservers* - Array of DNS servers configured for use by hosts * *searchdomains* - Array of up to 4 search domains +* *remove_default_searchdomains* - Boolean. If enabled, `searchdomains` variable can hold 6 search domains. * *dns_etchosts* - Content of hosts file for coredns and nodelocaldns For more information, see [DNS diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index d5a0c919d..ee26992b6 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -160,6 +160,14 @@ kube_encrypt_secret_data: false cluster_name: cluster.local # Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods ndots: 2 +# dns_timeout: 2 +# dns_attempts: 2 +# Custom search domains to be added in addition to the default cluster search domains +# searchdomains: +# - svc.{{ cluster_name }} +# - default.svc.{{ cluster_name }} +# Remove default cluster search domains (``default.svc.{{ dns_domain }}, svc.{{ dns_domain }}``). +# remove_default_searchdomains: false # Can be coredns, coredns_dual, manual or none dns_mode: coredns # Set manual server if using a custom cluster DNS server diff --git a/roles/kubernetes/preinstall/tasks/0040-set_facts.yml b/roles/kubernetes/preinstall/tasks/0040-set_facts.yml index 10602ef3e..e3a36f010 100644 --- a/roles/kubernetes/preinstall/tasks/0040-set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/0040-set_facts.yml @@ -91,12 +91,17 @@ changed_when: false check_mode: no +- name: set default dns if remove_default_searchdomains is false + set_fact: + default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"] + when: not remove_default_searchdomains|default()|bool or (remove_default_searchdomains|default()|bool and searchdomains|default([])|length==0) + - name: set dns facts set_fact: resolvconf: >- {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%} bogus_domains: |- - {% for d in [ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([]) -%} + {% for d in default_searchdomains|default([]) + searchdomains|default([]) -%} {{ dns_domain }}.{{ d }}./{{ d }}.{{ d }}./com.{{ d }}./ {%- endfor %} cloud_resolver: "{{ ['169.254.169.254'] if cloud_provider is defined and cloud_provider == 'gce' else @@ -169,11 +174,11 @@ - name: generate search domains to resolvconf set_fact: searchentries: - search {{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }} + search {{ (default_searchdomains|default([]) + searchdomains|default([])) | join(' ') }} domainentry: domain {{ dns_domain }} supersede_search: - supersede domain-search "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join('", "') }}"; + supersede domain-search "{{ (default_searchdomains|default([]) + searchdomains|default([])) | join('", "') }}"; supersede_domain: supersede domain-name "{{ dns_domain }}"; @@ -196,7 +201,7 @@ - name: generate nameservers for resolvconf, including cluster DNS set_fact: nameserverentries: |- - {{ ( ( [nodelocaldns_ip] if enable_nodelocaldns else []) + coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([]) + configured_nameservers|d([])) | unique | join(',') }} + {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server|d([]) if not enable_nodelocaldns else []) + nameservers|d([]) + cloud_resolver|d([]) + configured_nameservers|d([])) | unique | join(',') }} supersede_nameserver: supersede domain-name-servers {{ ( coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(', ') }}; when: not dns_early or dns_late diff --git a/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml b/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml index 2759b53e1..0d0b66b00 100644 --- a/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml +++ b/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml @@ -13,9 +13,7 @@ {% for item in nameserverentries.split(',') %} nameserver {{ item }} {% endfor %} - options ndots:{{ ndots }} - options timeout:2 - options attempts:2 + options ndots:{{ ndots }} timeout:{{ dns_timeout|default('2') }} attempts:{{ dns_attempts|default('2') }} state: present insertbefore: BOF create: yes diff --git a/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml b/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml index 851f236ac..f245814ad 100644 --- a/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml +++ b/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml @@ -9,12 +9,17 @@ backup: yes notify: Preinstall | update resolvconf for networkmanager +- name: set default dns if remove_default_searchdomains is false + set_fact: + default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"] + when: not remove_default_searchdomains|default()|bool or (remove_default_searchdomains|default()|bool and searchdomains|default([])|length==0) + - name: NetworkManager | Add DNS search to NM configuration ini_file: path: /etc/NetworkManager/conf.d/dns.conf section: global-dns option: searches - value: "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(',') }}" + value: "{{ (default_searchdomains|default([]) + searchdomains|default([])) | join(',') }}" mode: '0600' backup: yes notify: Preinstall | update resolvconf for networkmanager @@ -24,7 +29,7 @@ path: /etc/NetworkManager/conf.d/dns.conf section: global-dns option: options - value: "ndots:{{ ndots }};timeout:2;attempts:2;" + value: "ndots:{{ ndots }};timeout:{{ dns_timeout|default('2') }};attempts:{{ dns_attempts|default('2') }};" mode: '0600' backup: yes notify: Preinstall | update resolvconf for networkmanager diff --git a/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 index b40a80ae4..8cf8b8192 100644 --- a/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 +++ b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2 @@ -6,7 +6,7 @@ if [ $reason = "BOUND" ]; then if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d') - OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}" + OPTIONS="options timeout:{{ dns_timeout|default('2') }} attempts:{{ dns_attempts|default('2') }} ndots:{{ ndots }}" printf "%b\n" "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf fi diff --git a/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 index 028f0fd89..511839fd5 100644 --- a/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 +++ b/roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2 @@ -6,7 +6,7 @@ zdnsupdate_config() { if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d') - OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}" + OPTIONS="options timeout:{{ dns_timeout|default('2') }} attempts:{{ dns_attempts|default('2') }} ndots:{{ ndots }}" echo -e "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf fi diff --git a/roles/kubernetes/preinstall/templates/resolved.conf.j2 b/roles/kubernetes/preinstall/templates/resolved.conf.j2 index 72d4e3331..901fd2473 100644 --- a/roles/kubernetes/preinstall/templates/resolved.conf.j2 +++ b/roles/kubernetes/preinstall/templates/resolved.conf.j2 @@ -5,7 +5,11 @@ DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server )| list | join(' ') }} {% endif %} FallbackDNS={{ ( upstream_dns_servers|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(' ') }} +{% if remove_default_searchdomains is sameas false or (remove_default_searchdomains is sameas true and searchdomains|default([])|length==0)%} Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }} +{% else %} +Domains={{ searchdomains|default([]) | join(' ') }} +{% endif %} #LLMNR=no #MulticastDNS=no DNSSEC=no