Add federation support

Includes CoreDNS with etcd pod for its backend.

cluster.yml

@@ -89,3 +89,4 @@
   roles:
     - { role: kargo-defaults}
     - { role: kubernetes-apps, tags: apps }
+    - { role: kubernetes/federation, tags: federation }

inventory/group_vars/k8s-cluster.yml

@@ -110,8 +110,9 @@ resolvconf_mode: docker_dns
 # Deploy netchecker app to verify DNS resolve as an HTTP service
 deploy_netchecker: false
 # Ip address of the kubernetes skydns service
-skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}"
 dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
+skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}"
+coredns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}"
 dns_domain: "{{ cluster_name }}"
 
 # Path used to store Docker data

roles/download/defaults/main.yml

@@ -31,10 +31,12 @@ pod_infra_version: 3.0
 
 # Download URL's
 etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd"
+kubernetes_client_download_url: "https://dl.k8s.io/v1.6.0-beta.1/kubernetes-client-linux-amd64.tar.gz"
+#kubernetes_client_download_url: "https://dl.k8s.io/{{kube_version}}/kubernetes-client-linux-amd64.tar.gz"
 
 # Checksums
 etcd_checksum: "385afd518f93e3005510b7aaa04d38ee4a39f06f5152cd33bb86d4f0c94c7485"
-
+kubernetes_client_checksum: "d13f3bede2beb1d7fbca7f01a2c0775938d9127073b0fa1cecba4fd152947eae"
 # Containers
 # Possible values: host, docker
 etcd_deployment_type: "docker"
@@ -123,6 +125,14 @@ downloads:
     repo: "{{ hyperkube_image_repo }}"
     tag: "{{ hyperkube_image_tag }}"
     sha256: "{{ hyperkube_digest_checksum|default(None) }}"
+  kubernetes_client:
+    version: "{{ kube_version}}"
+    dest: "kubernetes_client/kubernetes-client-linux-amd64.tar.gz"
+    sha256: "{{ kubernetes_client_checksum }}"
+    url: "{{ kubernetes_client_download_url }}"
+    unarchive: true
+    owner: "kube"
+    mode: "0755"
   flannel:
     container: true
     repo: "{{ flannel_image_repo }}"
@@ -230,8 +240,8 @@ download:
   dest: "{{ file.dest|default(None) }}"
   version: "{{ file.version|default(None) }}"
   sha256: "{{ file.sha256|default(None) }}"
-  source_url: "{{ file.source_url|default(None) }}"
   url: "{{ file.url|default(None) }}"
   unarchive: "{{ file.unarchive|default('false') }}"
   owner: "{{ file.owner|default('kube') }}"
   mode: "{{ file.mode|default(None) }}"
+

roles/kubernetes/federation/defaults/main.yml

@@ -0,0 +1,11 @@
+# Common in master/child
+federation_name: "federation"
+federation_context: "{{ federation_name }}"
+federation_dns_zone: "federation"
+federation_namespace: "federation-system"
+
+# Federation master
+federation_master: true
+
+# Federation child
+#federation_master: false

roles/kubernetes/federation/meta/main.yml

@@ -0,0 +1,8 @@
+---
+dependencies:
+  - role: download
+    file: "{{ downloads.hyperkube }}"
+    tags: [download, hyperkube]
+  - role: download
+    file: "{{ downloads.kubernetes_client }}"
+    tags: [download, hyperkube]

roles/kubernetes/federation/tasks/coredns.yml

@@ -0,0 +1,29 @@
+- name: Federation | coredns | Create coredns config
+  template:
+    src: federation-coredns.conf.j2
+    dest: "{{ kube_config_dir }}/federation-coredns.conf"
+  tags: coredns
+
+- name: Federation | coredns | Lay Down coredns Template
+  template:
+    src: "{{item.file}}"
+    dest: "{{kube_config_dir}}/{{item.file}}"
+  with_items:
+    - {name: coredns-etcd, file: etcd-pod.yml.j2, type: pod}
+    - {name: coredns-etcd, file: etcd-svc.yml.j2, type: svc}
+    - {name: coredns, file: coredns-deploy.yml.j2, type: deployment}
+    - {name: coredns, file: coredns-svc.yml.j2, type: svc}
+  register: manifests
+  tags: coredns
+
+- name: Federation | coredns | Start Resources
+  kube:
+    name: "{{item.item.name}}"
+    namespace: "{{ system_namespace }}"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "{{item.item.type}}"
+    filename: "{{kube_config_dir}}/{{item.item.file}}"
+    state: "{{item.changed | ternary('latest','present') }}"
+  with_items: "{{ manifests.results }}"
+  tags: coredns
+

roles/kubernetes/federation/tasks/main.yml

@@ -0,0 +1,68 @@
+---
+#TODO - name: See if federation is already started
+- name: fedstart
+  command: /bin/true
+
+- include: coredns.yml
+  when: federation_master|default(false)
+
+- name: Copy kubefed from releases dir
+  copy:
+    src: "{{ local_release_dir }}/kubernetes_client/kubernetes/client/bin/kubefed"
+    dest: "{{ bin_dir }}/kubefed"
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Set up coredns federation config
+  template:
+    src: "federation-coredns.conf.j2"
+    dest: "{{ kube_config_dir }}/federation-coredns.conf"
+    owner: root
+    group: root
+    mode: 0640
+
+
+- name: See if namespace is created
+  command: "{{ bin_dir }}/kubectl get namespaces {{ federation_namespace }}"
+  register: federation_namespace_created
+  failed_when: false
+  when: federation_master|default(false)
+
+- name: Run kubefed init
+  command: >-
+    {{ bin_dir }}/kubefed init {{ federation_name }}
+    --host-cluster-context=kubelet-{{ cluster_name }}
+    --kubeconfig={{ kube_config_dir }}/node-kubeconfig.yaml
+    --federation-system-namespace={{ federation_namespace }}
+    --api-server-service-type=NodePort
+    --etcd-persistent-storage=false
+    --dns-provider=coredns
+    --dns-provider-config={{ kube_config_dir }}/federation-coredns.conf
+    --dns-zone-name={{ federation_dns_zone }}
+    --image={{ hyperkube_image_repo}}:{{ hyperkube_image_tag }}
+  when: federation_master|default(false) and federation_namespace_created.rc != 0
+
+#- name: Create federation context if necessary
+#  command: >-
+#    kubectl create config {{ federation_context }} blah blah
+#  when: not federation_master|default(false)
+
+- name: Run kubefed join
+  command: >-
+    {{ bin_dir }}/kubefed join {{ federation-name }}
+    --kubeconfig={{ kube_config_dir }}/node-kubeconfig.yaml
+    --host-cluster-context=kubelet-{{ cluster_name }}
+    --cluster-context={{ federation_context }}
+    --api-server-service-type=NodePort
+    --dns-zone-name={{ dns_domain }}
+  when: not federation_master|default(false)
+
+
+- name: Verify federation is enabled
+  command: "{{ bin_dir }}/kubectl --context={{ federation_name }} get clusters"
+  environment:
+    KUBECONFIG: "{{ kube_config_dir }}/node-kubeconfig.yaml"
+  retries: 12
+  delay: "{{ retry_stagger | random + 3 }}"
+

roles/kubernetes/federation/templates/coredns-deploy.yml.j2

@@ -0,0 +1,84 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: {{ system_namespace }}
+data:
+  Corefile: |
+    .:53 {
+        etcd {{ federation_name }} {
+          stubzones
+          path /skydns
+          endpoint coredns-etcd.{{ system_namespace }}
+          # FIXME(mattymo): https://github.com/kubernetes/kubernetes/issues/42995
+          #endpoint {{ etcd_access_addresses }}
+          #tls {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem {{ etcd_cert_dir }}/node-{{inventory_hostname }}-key.pem {{ etcd_cert_dir }}/ca.pem
+        }
+        errors
+        log stdout
+        health
+        proxy . /etc/resolv.conf
+        cache 30
+    }
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: coredns
+  namespace: {{ system_namespace }}
+  labels:
+    k8s-app: coredns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "CoreDNS"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: coredns
+  template:
+    metadata:
+      labels:
+        k8s-app: coredns
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
+    spec:
+      containers:
+      - name: coredns
+        image: coredns/coredns:latest
+        imagePullPolicy: Always
+        args: [ "-conf", "/etc/coredns/Corefile" ]
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+        - name: etcd-certs
+          mountPath: {{ etcd_cert_dir }}
+          readOnly: true
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+      dnsPolicy: Default
+      volumes:
+        - name: config-volume
+          configMap:
+            name: coredns
+            items:
+            - key: Corefile
+              path: Corefile
+        - hostPath:
+            path: {{ etcd_cert_dir }}
+          name: etcd-certs
+

roles/kubernetes/federation/templates/coredns-svc.yml.j2

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: coredns
+  namespace: {{ system_namespace }}
+  labels:
+    k8s-app: coredns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "coredns"
+spec:
+  selector:
+    k8s-app: coredns
+  clusterIP: {{ coredns_server }}
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    protocol: TCP

roles/kubernetes/federation/templates/etcd-pod.yml.j2

@@ -0,0 +1,35 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: coredns-etcd
+  name: coredns-etcd
+spec:
+  containers:
+  - command:
+    - /usr/local/bin/etcd
+    - --name
+    - coredns-etcd
+    - --initial-advertise-peer-urls
+    - http://coredns-etcd:2380
+    - --listen-peer-urls
+    - http://0.0.0.0:2380
+    - --listen-client-urls
+    - http://0.0.0.0:2379
+    - --advertise-client-urls
+    - http://coredns-etcd:2379
+    - --initial-cluster
+    - coredns-etcd=http://127.0.0.1:2380
+    - --initial-cluster-state
+    - new
+    image: {{ etcd_image_repo }}:{{ etcd_image_tag }}
+    name: coredns-etcd
+    ports:
+    - containerPort: 2379
+      name: client
+      protocol: TCP
+    - containerPort: 2380
+      name: server
+      protocol: TCP
+  restartPolicy: Always

roles/kubernetes/federation/templates/etcd-svc.yml.j2

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: coredns-etcd
+  namespace: {{ system_namespace }}
+  labels:
+    k8s-app: coredns-etcd
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "coredns-etcd"
+spec:
+  selector:
+    k8s-app: coredns-etcd
+  ports:
+  - name: client
+    port: 2379
+    protocol: TCP
+    targetPort: 2379
+  - name: server
+    port: 2380
+    protocol: TCP
+    targetPort: 2380

roles/kubernetes/federation/templates/federation-coredns.conf.j2

@@ -0,0 +1,3 @@
+[Global]
+etcd-endpoints = http://coredns-etcd.{{ system_namespace }}:2379
+zones = {{ federation_name }}

roles/kubernetes/node/tasks/main.yml

@@ -24,6 +24,7 @@
     src: node-kubeconfig.yaml.j2
     dest: "{{ kube_config_dir }}/node-kubeconfig.yaml"
     backup: yes
+    force: "{% if federation_master|default(false) %}no{% else %}yes{% endif %}"
   notify: restart kubelet
   tags: kubelet
 

upgrade-cluster.yml

@@ -99,3 +99,4 @@
   roles:
     - { role: kargo-defaults}
     - { role: kubernetes-apps, tags: apps }
+    - { role: kubernetes/federation, tags: federation }