From cff4c5db4ac9f6688cf3af0ee043e99d9fb7cdc3 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Mon, 6 Mar 2017 19:39:34 +0300 Subject: [PATCH] Add federation support Includes CoreDNS with etcd pod for its backend. --- cluster.yml | 1 + inventory/group_vars/k8s-cluster.yml | 3 +- roles/download/defaults/main.yml | 14 +++- roles/kubernetes/federation/defaults/main.yml | 11 +++ roles/kubernetes/federation/meta/main.yml | 8 ++ roles/kubernetes/federation/tasks/coredns.yml | 29 +++++++ roles/kubernetes/federation/tasks/main.yml | 68 +++++++++++++++ .../templates/coredns-deploy.yml.j2 | 84 +++++++++++++++++++ .../federation/templates/coredns-svc.yml.j2 | 20 +++++ .../federation/templates/etcd-pod.yml.j2 | 35 ++++++++ .../federation/templates/etcd-svc.yml.j2 | 21 +++++ .../templates/federation-coredns.conf.j2 | 3 + roles/kubernetes/node/tasks/main.yml | 1 + upgrade-cluster.yml | 1 + 14 files changed, 296 insertions(+), 3 deletions(-) create mode 100644 roles/kubernetes/federation/defaults/main.yml create mode 100644 roles/kubernetes/federation/meta/main.yml create mode 100644 roles/kubernetes/federation/tasks/coredns.yml create mode 100644 roles/kubernetes/federation/tasks/main.yml create mode 100644 roles/kubernetes/federation/templates/coredns-deploy.yml.j2 create mode 100644 roles/kubernetes/federation/templates/coredns-svc.yml.j2 create mode 100644 roles/kubernetes/federation/templates/etcd-pod.yml.j2 create mode 100644 roles/kubernetes/federation/templates/etcd-svc.yml.j2 create mode 100644 roles/kubernetes/federation/templates/federation-coredns.conf.j2 diff --git a/cluster.yml b/cluster.yml index 01b033b2f..f45c3ac09 100644 --- a/cluster.yml +++ b/cluster.yml @@ -89,3 +89,4 @@ roles: - { role: kargo-defaults} - { role: kubernetes-apps, tags: apps } + - { role: kubernetes/federation, tags: federation } diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index cbd922c63..f5c513867 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -110,8 +110,9 @@ resolvconf_mode: docker_dns # Deploy netchecker app to verify DNS resolve as an HTTP service deploy_netchecker: false # Ip address of the kubernetes skydns service -skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}" dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}" +skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}" +coredns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}" dns_domain: "{{ cluster_name }}" # Path used to store Docker data diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 23e87bbbd..add3b7eed 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -31,10 +31,12 @@ pod_infra_version: 3.0 # Download URL's etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd" +kubernetes_client_download_url: "https://dl.k8s.io/v1.6.0-beta.1/kubernetes-client-linux-amd64.tar.gz" +#kubernetes_client_download_url: "https://dl.k8s.io/{{kube_version}}/kubernetes-client-linux-amd64.tar.gz" # Checksums etcd_checksum: "385afd518f93e3005510b7aaa04d38ee4a39f06f5152cd33bb86d4f0c94c7485" - +kubernetes_client_checksum: "d13f3bede2beb1d7fbca7f01a2c0775938d9127073b0fa1cecba4fd152947eae" # Containers # Possible values: host, docker etcd_deployment_type: "docker" @@ -123,6 +125,14 @@ downloads: repo: "{{ hyperkube_image_repo }}" tag: "{{ hyperkube_image_tag }}" sha256: "{{ hyperkube_digest_checksum|default(None) }}" + kubernetes_client: + version: "{{ kube_version}}" + dest: "kubernetes_client/kubernetes-client-linux-amd64.tar.gz" + sha256: "{{ kubernetes_client_checksum }}" + url: "{{ kubernetes_client_download_url }}" + unarchive: true + owner: "kube" + mode: "0755" flannel: container: true repo: "{{ flannel_image_repo }}" @@ -230,8 +240,8 @@ download: dest: "{{ file.dest|default(None) }}" version: "{{ file.version|default(None) }}" sha256: "{{ file.sha256|default(None) }}" - source_url: "{{ file.source_url|default(None) }}" url: "{{ file.url|default(None) }}" unarchive: "{{ file.unarchive|default('false') }}" owner: "{{ file.owner|default('kube') }}" mode: "{{ file.mode|default(None) }}" + diff --git a/roles/kubernetes/federation/defaults/main.yml b/roles/kubernetes/federation/defaults/main.yml new file mode 100644 index 000000000..7106b6148 --- /dev/null +++ b/roles/kubernetes/federation/defaults/main.yml @@ -0,0 +1,11 @@ +# Common in master/child +federation_name: "federation" +federation_context: "{{ federation_name }}" +federation_dns_zone: "federation" +federation_namespace: "federation-system" + +# Federation master +federation_master: true + +# Federation child +#federation_master: false diff --git a/roles/kubernetes/federation/meta/main.yml b/roles/kubernetes/federation/meta/main.yml new file mode 100644 index 000000000..00fa2f6e6 --- /dev/null +++ b/roles/kubernetes/federation/meta/main.yml @@ -0,0 +1,8 @@ +--- +dependencies: + - role: download + file: "{{ downloads.hyperkube }}" + tags: [download, hyperkube] + - role: download + file: "{{ downloads.kubernetes_client }}" + tags: [download, hyperkube] diff --git a/roles/kubernetes/federation/tasks/coredns.yml b/roles/kubernetes/federation/tasks/coredns.yml new file mode 100644 index 000000000..6e660a7c7 --- /dev/null +++ b/roles/kubernetes/federation/tasks/coredns.yml @@ -0,0 +1,29 @@ +- name: Federation | coredns | Create coredns config + template: + src: federation-coredns.conf.j2 + dest: "{{ kube_config_dir }}/federation-coredns.conf" + tags: coredns + +- name: Federation | coredns | Lay Down coredns Template + template: + src: "{{item.file}}" + dest: "{{kube_config_dir}}/{{item.file}}" + with_items: + - {name: coredns-etcd, file: etcd-pod.yml.j2, type: pod} + - {name: coredns-etcd, file: etcd-svc.yml.j2, type: svc} + - {name: coredns, file: coredns-deploy.yml.j2, type: deployment} + - {name: coredns, file: coredns-svc.yml.j2, type: svc} + register: manifests + tags: coredns + +- name: Federation | coredns | Start Resources + kube: + name: "{{item.item.name}}" + namespace: "{{ system_namespace }}" + kubectl: "{{bin_dir}}/kubectl" + resource: "{{item.item.type}}" + filename: "{{kube_config_dir}}/{{item.item.file}}" + state: "{{item.changed | ternary('latest','present') }}" + with_items: "{{ manifests.results }}" + tags: coredns + diff --git a/roles/kubernetes/federation/tasks/main.yml b/roles/kubernetes/federation/tasks/main.yml new file mode 100644 index 000000000..cf01da9b3 --- /dev/null +++ b/roles/kubernetes/federation/tasks/main.yml @@ -0,0 +1,68 @@ +--- +#TODO - name: See if federation is already started +- name: fedstart + command: /bin/true + +- include: coredns.yml + when: federation_master|default(false) + +- name: Copy kubefed from releases dir + copy: + src: "{{ local_release_dir }}/kubernetes_client/kubernetes/client/bin/kubefed" + dest: "{{ bin_dir }}/kubefed" + owner: root + group: root + mode: 0755 + +- name: Set up coredns federation config + template: + src: "federation-coredns.conf.j2" + dest: "{{ kube_config_dir }}/federation-coredns.conf" + owner: root + group: root + mode: 0640 + + +- name: See if namespace is created + command: "{{ bin_dir }}/kubectl get namespaces {{ federation_namespace }}" + register: federation_namespace_created + failed_when: false + when: federation_master|default(false) + +- name: Run kubefed init + command: >- + {{ bin_dir }}/kubefed init {{ federation_name }} + --host-cluster-context=kubelet-{{ cluster_name }} + --kubeconfig={{ kube_config_dir }}/node-kubeconfig.yaml + --federation-system-namespace={{ federation_namespace }} + --api-server-service-type=NodePort + --etcd-persistent-storage=false + --dns-provider=coredns + --dns-provider-config={{ kube_config_dir }}/federation-coredns.conf + --dns-zone-name={{ federation_dns_zone }} + --image={{ hyperkube_image_repo}}:{{ hyperkube_image_tag }} + when: federation_master|default(false) and federation_namespace_created.rc != 0 + +#- name: Create federation context if necessary +# command: >- +# kubectl create config {{ federation_context }} blah blah +# when: not federation_master|default(false) + +- name: Run kubefed join + command: >- + {{ bin_dir }}/kubefed join {{ federation-name }} + --kubeconfig={{ kube_config_dir }}/node-kubeconfig.yaml + --host-cluster-context=kubelet-{{ cluster_name }} + --cluster-context={{ federation_context }} + --api-server-service-type=NodePort + --dns-zone-name={{ dns_domain }} + when: not federation_master|default(false) + + +- name: Verify federation is enabled + command: "{{ bin_dir }}/kubectl --context={{ federation_name }} get clusters" + environment: + KUBECONFIG: "{{ kube_config_dir }}/node-kubeconfig.yaml" + retries: 12 + delay: "{{ retry_stagger | random + 3 }}" + diff --git a/roles/kubernetes/federation/templates/coredns-deploy.yml.j2 b/roles/kubernetes/federation/templates/coredns-deploy.yml.j2 new file mode 100644 index 000000000..e3afdcfa1 --- /dev/null +++ b/roles/kubernetes/federation/templates/coredns-deploy.yml.j2 @@ -0,0 +1,84 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: {{ system_namespace }} +data: + Corefile: | + .:53 { + etcd {{ federation_name }} { + stubzones + path /skydns + endpoint coredns-etcd.{{ system_namespace }} + # FIXME(mattymo): https://github.com/kubernetes/kubernetes/issues/42995 + #endpoint {{ etcd_access_addresses }} + #tls {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem {{ etcd_cert_dir }}/node-{{inventory_hostname }}-key.pem {{ etcd_cert_dir }}/ca.pem + } + errors + log stdout + health + proxy . /etc/resolv.conf + cache 30 + } +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: coredns + namespace: {{ system_namespace }} + labels: + k8s-app: coredns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: coredns + template: + metadata: + labels: + k8s-app: coredns + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' + spec: + containers: + - name: coredns + image: coredns/coredns:latest + imagePullPolicy: Always + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + - name: etcd-certs + mountPath: {{ etcd_cert_dir }} + readOnly: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile + - hostPath: + path: {{ etcd_cert_dir }} + name: etcd-certs + diff --git a/roles/kubernetes/federation/templates/coredns-svc.yml.j2 b/roles/kubernetes/federation/templates/coredns-svc.yml.j2 new file mode 100644 index 000000000..4bd6a981b --- /dev/null +++ b/roles/kubernetes/federation/templates/coredns-svc.yml.j2 @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: coredns + namespace: {{ system_namespace }} + labels: + k8s-app: coredns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "coredns" +spec: + selector: + k8s-app: coredns + clusterIP: {{ coredns_server }} + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/roles/kubernetes/federation/templates/etcd-pod.yml.j2 b/roles/kubernetes/federation/templates/etcd-pod.yml.j2 new file mode 100644 index 000000000..262b42f96 --- /dev/null +++ b/roles/kubernetes/federation/templates/etcd-pod.yml.j2 @@ -0,0 +1,35 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + app: coredns-etcd + name: coredns-etcd +spec: + containers: + - command: + - /usr/local/bin/etcd + - --name + - coredns-etcd + - --initial-advertise-peer-urls + - http://coredns-etcd:2380 + - --listen-peer-urls + - http://0.0.0.0:2380 + - --listen-client-urls + - http://0.0.0.0:2379 + - --advertise-client-urls + - http://coredns-etcd:2379 + - --initial-cluster + - coredns-etcd=http://127.0.0.1:2380 + - --initial-cluster-state + - new + image: {{ etcd_image_repo }}:{{ etcd_image_tag }} + name: coredns-etcd + ports: + - containerPort: 2379 + name: client + protocol: TCP + - containerPort: 2380 + name: server + protocol: TCP + restartPolicy: Always diff --git a/roles/kubernetes/federation/templates/etcd-svc.yml.j2 b/roles/kubernetes/federation/templates/etcd-svc.yml.j2 new file mode 100644 index 000000000..e52dd30bc --- /dev/null +++ b/roles/kubernetes/federation/templates/etcd-svc.yml.j2 @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: coredns-etcd + namespace: {{ system_namespace }} + labels: + k8s-app: coredns-etcd + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "coredns-etcd" +spec: + selector: + k8s-app: coredns-etcd + ports: + - name: client + port: 2379 + protocol: TCP + targetPort: 2379 + - name: server + port: 2380 + protocol: TCP + targetPort: 2380 diff --git a/roles/kubernetes/federation/templates/federation-coredns.conf.j2 b/roles/kubernetes/federation/templates/federation-coredns.conf.j2 new file mode 100644 index 000000000..a7212d048 --- /dev/null +++ b/roles/kubernetes/federation/templates/federation-coredns.conf.j2 @@ -0,0 +1,3 @@ +[Global] +etcd-endpoints = http://coredns-etcd.{{ system_namespace }}:2379 +zones = {{ federation_name }} diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index a6a9c16f2..4435fa4bc 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -24,6 +24,7 @@ src: node-kubeconfig.yaml.j2 dest: "{{ kube_config_dir }}/node-kubeconfig.yaml" backup: yes + force: "{% if federation_master|default(false) %}no{% else %}yes{% endif %}" notify: restart kubelet tags: kubelet diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml index f4f48d543..b3b36e8b3 100644 --- a/upgrade-cluster.yml +++ b/upgrade-cluster.yml @@ -99,3 +99,4 @@ roles: - { role: kargo-defaults} - { role: kubernetes-apps, tags: apps } + - { role: kubernetes/federation, tags: federation }