Use dnsmasq inside pods

.travis.yml

@@ -15,15 +15,15 @@ env:
     # Debian Jessie
     - >-
       KUBE_NETWORK_PLUGIN=flannel
-      CLOUD_IMAGE=debian-8
+      CLOUD_IMAGE=debian-8-kubespray
       CLOUD_REGION=europe-west1-b
     - >-
       KUBE_NETWORK_PLUGIN=calico
-      CLOUD_IMAGE=debian-8
+      CLOUD_IMAGE=debian-8-kubespray
       CLOUD_REGION=us-central1-c
     - >-
       KUBE_NETWORK_PLUGIN=weave
-      CLOUD_IMAGE=debian-8
+      CLOUD_IMAGE=debian-8-kubespray
       CLOUD_REGION=us-east1-d
 
     # Centos 7

cluster.yml

@@ -8,8 +8,11 @@
     - { role: docker, tags: docker, when: ansible_os_family != "CoreOS" }
     - { role: kubernetes/node, tags: node }
     - { role: network_plugin, tags: network }
-    - { role: dnsmasq, tags: dnsmasq }
 
 - hosts: kube-master
   roles:
     - { role: kubernetes/master, tags: master }
+
+- hosts: kube-master[0]
+  roles:
+    - { role: dnsmasq, tags: dnsmasq }

inventory/group_vars/all.yml

@@ -97,7 +97,8 @@ upstream_dns_servers:
 dns_setup: true
 dns_domain: "{{ cluster_name }}"
 #
-# # Ip address of the kubernetes dns service
+# # Ip address of the kubernetes skydns service
+skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}"
 dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
 
 # For multi masters architecture:

roles/dnsmasq/library/kube.py

@@ -0,0 +1,318 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+DOCUMENTATION = """
+---
+module: kube
+short_description: Manage Kubernetes Cluster
+description:
+  - Create, replace, remove, and stop resources within a Kubernetes Cluster
+version_added: "2.0"
+options:
+  name:
+    required: false
+    default: null
+    description:
+      - The name associated with resource
+  filename:
+    required: false
+    default: null
+    description:
+      - The path and filename of the resource(s) definition file.
+  kubectl:
+    required: false
+    default: null
+    description:
+      - The path to the kubectl bin
+  namespace:
+    required: false
+    default: null
+    description:
+      - The namespace associated with the resource(s)
+  resource:
+    required: false
+    default: null
+    description:
+      - The resource to perform an action on. pods (po), replicationControllers (rc), services (svc)
+  label:
+    required: false
+    default: null
+    description:
+      - The labels used to filter specific resources.
+  server:
+    required: false
+    default: null
+    description:
+      - The url for the API server that commands are executed against.
+  api_version:
+    required: false
+    choices: ['v1', 'v1beta3']
+    default: v1
+    description:
+      - The API version associated with cluster.
+  force:
+    required: false
+    default: false
+    description:
+      - A flag to indicate to force delete, replace, or stop.
+  all:
+    required: false
+    default: false
+    description:
+      - A flag to indicate delete all, stop all, or all namespaces when checking exists.
+  log_level:
+    required: false
+    default: 0
+    description:
+      - Indicates the level of verbosity of logging by kubectl.
+  state:
+    required: false
+    choices: ['present', 'absent', 'latest', 'reloaded', 'stopped']
+    default: present
+    description:
+      - present handles checking existence or creating if definition file provided,
+        absent handles deleting resource(s) based on other options,
+        latest handles creating ore updating based on existence,
+        reloaded handles updating resource(s) definition using definition file,
+        stopped handles stopping resource(s) based on other options.
+requirements:
+  - kubectl
+author: "Kenny Jones (@kenjones-cisco)"
+"""
+
+EXAMPLES = """
+- name: test nginx is present
+  kube: name=nginx resource=rc state=present
+
+- name: test nginx is stopped
+  kube: name=nginx resource=rc state=stopped
+
+- name: test nginx is absent
+  kube: name=nginx resource=rc state=absent
+
+- name: test nginx is present
+  kube: filename=/tmp/nginx.yml
+"""
+
+
+class KubeManager(object):
+
+    def __init__(self, module):
+
+        self.module = module
+
+        self.kubectl = module.params.get('kubectl')
+        if self.kubectl is None:
+            self.kubectl =  module.get_bin_path('kubectl', True)
+        self.base_cmd = [self.kubectl]
+        self.api_version = module.params.get('api_version')
+
+        if self.api_version:
+            self.base_cmd.append('--api-version=' + self.api_version)
+
+        if module.params.get('server'):
+            self.base_cmd.append('--server=' + module.params.get('server'))
+
+        if module.params.get('log_level'):
+            self.base_cmd.append('--v=' + str(module.params.get('log_level')))
+
+        if module.params.get('namespace'):
+            self.base_cmd.append('--namespace=' + module.params.get('namespace'))
+
+        self.all = module.params.get('all')
+        self.force = module.params.get('force')
+        self.name = module.params.get('name')
+        self.filename = module.params.get('filename')
+        self.resource = module.params.get('resource')
+        self.label = module.params.get('label')
+
+    def _execute(self, cmd):
+        args = self.base_cmd + cmd
+        try:
+            rc, out, err = self.module.run_command(args)
+            if rc != 0:
+                self.module.fail_json(
+                    msg='error running kubectl (%s) command (rc=%d): %s' % (' '.join(args), rc, out or err))
+        except Exception as exc:
+            self.module.fail_json(
+                msg='error running kubectl (%s) command: %s' % (' '.join(args), str(exc)))
+        return out.splitlines()
+
+    def _execute_nofail(self, cmd):
+        args = self.base_cmd + cmd
+        rc, out, err = self.module.run_command(args)
+        if rc != 0:
+            return None
+        return out.splitlines()
+
+    def create(self, check=True):
+        if check and self.exists():
+            return []
+
+        cmd = ['create']
+
+        if not self.filename:
+            self.module.fail_json(msg='filename required to create')
+
+        cmd.append('--filename=' + self.filename)
+
+        return self._execute(cmd)
+
+    def replace(self):
+
+        if not self.force and not self.exists():
+            return []
+
+        cmd = ['replace']
+        if self.api_version != 'v1':
+            cmd = ['update']
+
+        if self.force:
+            cmd.append('--force')
+
+        if not self.filename:
+            self.module.fail_json(msg='filename required to reload')
+
+        cmd.append('--filename=' + self.filename)
+
+        return self._execute(cmd)
+
+    def delete(self):
+
+        if not self.force and not self.exists():
+            return []
+
+        cmd = ['delete']
+
+        if self.filename:
+            cmd.append('--filename=' + self.filename)
+        else:
+            if not self.resource:
+                self.module.fail_json(msg='resource required to delete without filename')
+
+            cmd.append(self.resource)
+
+            if self.name:
+                cmd.append(self.name)
+
+            if self.label:
+                cmd.append('--selector=' + self.label)
+
+            if self.all:
+                cmd.append('--all')
+
+            if self.force:
+                cmd.append('--ignore-not-found')
+
+        return self._execute(cmd)
+
+    def exists(self):
+        cmd = ['get']
+
+        if not self.resource:
+            return False
+
+        cmd.append(self.resource)
+
+        if self.name:
+            cmd.append(self.name)
+
+        cmd.append('--no-headers')
+
+        if self.label:
+            cmd.append('--selector=' + self.label)
+
+        if self.all:
+            cmd.append('--all-namespaces')
+
+        result = self._execute_nofail(cmd)
+        if not result:
+            return False
+        return True
+
+    def stop(self):
+
+        if not self.force and not self.exists():
+            return []
+
+        cmd = ['stop']
+
+        if self.filename:
+            cmd.append('--filename=' + self.filename)
+        else:
+            if not self.resource:
+                self.module.fail_json(msg='resource required to stop without filename')
+
+            cmd.append(self.resource)
+
+            if self.name:
+                cmd.append(self.name)
+
+            if self.label:
+                cmd.append('--selector=' + self.label)
+
+            if self.all:
+                cmd.append('--all')
+
+            if self.force:
+                cmd.append('--ignore-not-found')
+
+        return self._execute(cmd)
+
+
+def main():
+
+    module = AnsibleModule(
+        argument_spec=dict(
+            name=dict(),
+            filename=dict(),
+            namespace=dict(),
+            resource=dict(),
+            label=dict(),
+            server=dict(),
+            kubectl=dict(),
+            api_version=dict(default='v1', choices=['v1', 'v1beta3']),
+            force=dict(default=False, type='bool'),
+            all=dict(default=False, type='bool'),
+            log_level=dict(default=0, type='int'),
+            state=dict(default='present', choices=['present', 'absent', 'latest', 'reloaded', 'stopped']),
+            )
+        )
+
+    changed = False
+
+    manager = KubeManager(module)
+    state = module.params.get('state')
+
+    if state == 'present':
+        result = manager.create()
+
+    elif state == 'absent':
+        result = manager.delete()
+
+    elif state == 'reloaded':
+        result = manager.replace()
+
+    elif state == 'stopped':
+        result = manager.stop()
+
+    elif state == 'latest':
+        if manager.exists():
+            manager.force = True
+            result = manager.replace()
+        else:
+            result = manager.create(check=False)
+
+    else:
+        module.fail_json(msg='Unrecognized state %s.' % state)
+
+    if result:
+        changed = True
+    module.exit_json(changed=changed,
+                     msg='success: %s' % (' '.join(result))
+                     )
+
+
+from ansible.module_utils.basic import *  # noqa
+if __name__ == '__main__':
+    main()

roles/dnsmasq/tasks/main.yml

@@ -31,11 +31,32 @@
     dest: /etc/dnsmasq.d/01-kube-dns.conf
     state: link
 
-- name: Create dnsmasq pod manifest
+- name: Create dnsmasq manifests
-  template: src=dnsmasq-pod.yml dest=/etc/kubernetes/manifests/dnsmasq-pod.manifest
+  template: src={{item.file}} dest=/etc/kubernetes/{{item.file}}
+  with_items:
+    - {file: dnsmasq-ds.yml, type: ds}
+    - {file: dnsmasq-svc.yml, type: svc}
+  register: manifests
+
+# - name: Start resources
+#   command:  create -f /etc/kubernetes/{{item.item.file}} --namespace=kube-system
+#   ignore_errors: yes
+
+- name: Start Resources
+  kube:
+    name: dnsmasq
+    namespace: kube-system
+    kubectl: /usr/local/bin/kubectl
+    resource: "{{item.item.type}}"
+    filename: /etc/kubernetes/{{item.item.file}}
+    state: "{{item.changed | ternary('latest','present') }}"
+  with_items: manifests.results
+
+
 
 - name: Check for dnsmasq port (pulling image and running container)
   wait_for:
+    host: "{{dns_server}}"
     port: 53
     delay: 5
 
@@ -59,7 +80,7 @@
 
 - name: Add local dnsmasq to resolv.conf
   lineinfile:
-    line: "nameserver 127.0.0.1"
+    line: "nameserver {{dns_server}}"
     dest: "{{resolvconffile}}"
     state: present
     insertafter: "^search.*$"

roles/dnsmasq/templates/01-kube-dns.conf.j2

@@ -1,6 +1,6 @@
 #Listen on localhost
 bind-interfaces
-listen-address=127.0.0.1
+listen-address=0.0.0.0
 
 addn-hosts=/etc/hosts
 
@@ -17,4 +17,4 @@ server={{ srv }}
 {% endif %}
 
 # Forward k8s domain to kube-dns
-server=/{{ dns_domain }}/{{ dns_server }}
+server=/{{ dns_domain }}/{{ skydns_server }}

roles/dnsmasq/templates/dnsmasq-ds.yml

@@ -0,0 +1,52 @@
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: dnsmasq
+  namespace: kube-system
+  labels:
+    k8s-app: dnsmasq
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: dnsmasq
+    spec:
+      containers:
+        - name: dnsmasq
+          image: andyshinn/dnsmasq:2.72
+          command:
+            - dnsmasq
+          args:
+            - -k
+            - "-7"
+            - /etc/dnsmasq.d
+          securityContext:
+            capabilities:
+              add:
+                - NET_ADMIN
+          imagePullPolicy: Always
+          resources:
+            limits:
+              cpu: 100m
+              memory: 256M
+          ports:
+            - name: dns
+              containerPort: 53
+              protocol: UDP
+            - name: dns-tcp
+              containerPort: 53
+              protocol: TCP
+          volumeMounts:
+            - name: etcdnsmasqd
+              mountPath: /etc/dnsmasq.d
+            - name: etcdnsmasqdavailable
+              mountPath: /etc/dnsmasq.d-available
+
+      volumes:
+        - name: etcdnsmasqd
+          hostPath:
+            path: /etc/dnsmasq.d
+        - name: etcdnsmasqdavailable
+          hostPath:
+            path: /etc/dnsmasq.d-available

roles/dnsmasq/templates/dnsmasq-pod.yml

@@ -1,49 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: dnsmasq
-  namespace: kube-system
-spec:
-  hostNetwork: true
-  containers:
-    - name: dnsmasq
-      image: andyshinn/dnsmasq:2.72
-      command:
-        - dnsmasq
-      args:
-        - -k
-        - "-7"
-        - /etc/dnsmasq.d
-        - --local-service
-      securityContext:
-        capabilities:
-          add:
-            - NET_ADMIN
-      imagePullPolicy: Always
-      resources:
-        limits:
-          cpu: 100m
-          memory: 256M
-      ports:
-        - name: dns
-          containerPort: 53
-          hostPort: 53
-          protocol: UDP
-        - name: dns-tcp
-          containerPort: 53
-          hostPort: 53
-          protocol: TCP
-      volumeMounts:
-        - name: etcdnsmasqd
-          mountPath: /etc/dnsmasq.d
-        - name: etcdnsmasqdavailable
-          mountPath: /etc/dnsmasq.d-available
-
-  volumes:
-    - name: etcdnsmasqd
-      hostPath:
-        path: /etc/dnsmasq.d
-    - name: etcdnsmasqdavailable
-      hostPath:
-        path: /etc/dnsmasq.d-available

roles/dnsmasq/templates/dnsmasq-svc.yml

@@ -0,0 +1,23 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    kubernetes.io/cluster-service: 'true'
+    k8s-app: dnsmasq
+  name: dnsmasq
+  namespace: kube-system
+spec:
+  ports:
+    - port: 53
+      name: dns-tcp
+      targetPort: 53
+      protocol: TCP
+    - port: 53
+      name: dns
+      targetPort: 53
+      protocol: UDP
+  type: ClusterIP
+  clusterIP: {{dns_server}}
+  selector:
+    k8s-app: dnsmasq