Do not forward private domains for upstream resolvers

Also fix kube log level 4 to log dnsmasq queries.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
Co-authored-by: Matthew Mosesohn <mmosesohn@mirantis.com>

docs/dns-stack.md

@@ -38,6 +38,13 @@ or `8.8.8.8`. And domain is set to the default ``dns_domain`` value as well.
 Later, the nameservers will be reconfigured to the DNS service IP that Kargo
 configures for K8s cluster.
 
+Important to note that multiple search domains combined with high ``ndots``
+values lead to poor performance of DNS stack, so please choose it wise.
+The dnsmasq daemon set can take lower ``ndots`` values and return NXDOMAIN
+replies for [bogus internal FQDNS](https://github.com/kubernetes/kubernetes/issues/19634#issuecomment-253948954)
+before it even hits the kubedns app. Which makes it to be serving as a guarding
+recursive resolver in front of the kubedns running SkyDNS as an authoritative resolver.
+
 DNS configuration details
 -------------------------
 
@@ -93,8 +100,7 @@ Limitations
   [no way to specify a custom value](https://github.com/kubernetes/kubernetes/issues/33554)
   for the SkyDNS ``ndots`` param via an
   [option for KubeDNS](https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-dns/app/options/options.go)
-  add-on, while SkyDNS supports it though. Thus, DNS SRV records may not work
-  as expected as they require the ``ndots:7``.
+  add-on, while SkyDNS supports it though.
 
 * the ``searchdomains`` have a limitation of a 6 names and 256 chars
   length. Due to default ``svc, default.svc`` subdomains, the actual

roles/dnsmasq/templates/01-kube-dns.conf.j2

@@ -7,6 +7,8 @@ addn-hosts=/etc/hosts
 strict-order
 # Forward k8s domain to kube-dns
 server=/{{ dns_domain }}/{{ skydns_server }}
+# Reply NXDOMAIN to private/internal domains requests
+local=/internal./local./lc./{{ private_domains }}
 
 #Set upstream dns servers
 {% if upstream_dns_servers is defined %}
@@ -17,7 +19,7 @@ server={{ srv }}
 server={{ default_resolver }}
 {% endif %}
 
-{% if kube_log_level == 4 %}
+{% if kube_log_level == '4' %}
 log-queries
 {% endif %}
 bogus-priv

roles/kubernetes-apps/ansible/templates/kubedns-rc.yml

@@ -78,8 +78,16 @@ spec:
         - --log-facility=-
         - --cache-size=1000
         - --no-resolv
-        - --server=127.0.0.1#10053
-{% if kube_log_level == 4 %}
+        - --server=/{{ dns_domain }}/127.0.0.1#10053
+        - --local=/internal./local./lc./{{ private_domains }}
+{% if upstream_dns_servers is defined %}
+{% for srv in upstream_dns_servers %}
+        - --server={{ srv }}
+{% endfor %}
+{% else %}
+        - --server={{ default_resolver }}
+{% endif %}
+{% if kube_log_level == '4' %}
         - --log-queries
 {% endif %}
         ports:

roles/kubernetes/preinstall/tasks/set_facts.yml

@@ -50,5 +50,9 @@
 - set_fact:
     etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}"
 - set_fact:
+    private_domains: |-
+      {% for d in [ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([]) -%}
+      {{dns_domain}}.{{d}}./{{d}}.{{d}}./com.{{d}}./
+      {%- endfor %}
     default_resolver: >-
       {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%}