From 5881ba43f8ccf35bd7aaa2e650d7255c09f9984d Mon Sep 17 00:00:00 2001 From: Wei Tie Date: Tue, 5 Dec 2017 17:06:52 -0800 Subject: [PATCH 1/4] Split contiv etcd and etcd-proxy into two daemonsets Putting contiv etcd and etcd-proxy into the same daemonset and manage the difference by a env file is not good for scaling (adding nodes). This commit split them into two daemonsets so that when adding nodes, k8s could automatically starting a etcd-proxy on new nodes without need to run related play that putting env file. --- roles/network_plugin/contiv/defaults/main.yml | 10 ++++++ roles/network_plugin/contiv/tasks/main.yml | 2 ++ .../contiv/templates/contiv-etcd-proxy.yml.j2 | 31 +++++++++++++++++++ .../contiv/templates/contiv-etcd.env.j2 | 18 ++--------- .../contiv/templates/contiv-etcd.yml.j2 | 22 ++++++------- 5 files changed, 56 insertions(+), 27 deletions(-) create mode 100644 roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 diff --git a/roles/network_plugin/contiv/defaults/main.yml b/roles/network_plugin/contiv/defaults/main.yml index 6d1299c0b..83d940173 100644 --- a/roles/network_plugin/contiv/defaults/main.yml +++ b/roles/network_plugin/contiv/defaults/main.yml @@ -6,6 +6,8 @@ contiv_etcd_data_dir: "/var/lib/etcd/contiv-data" contiv_netmaster_port: 9999 contiv_cni_version: 0.1.0 +contiv_etcd_image_repo: "{{ etcd_image_repo }}" +contiv_etcd_image_tag: "{{ etcd_image_tag }}" contiv_etcd_listen_ip: "{{ ip | default(ansible_default_ipv4['address']) }}" contiv_etcd_listen_port: 6666 contiv_etcd_peer_port: 6667 @@ -14,6 +16,14 @@ contiv_etcd_peer_urls: http://{{ contiv_etcd_listen_ip }}:{{ contiv_etcd_peer_po contiv_etcd_listen_urls: - http://{{ contiv_etcd_listen_ip }}:{{ contiv_etcd_listen_port }} - http://127.0.0.1:{{ contiv_etcd_listen_port }} +contiv_etcd_endpoints: |- + {% for host in groups['kube-master'] -%} + contiv_etcd{{ loop.index }}=http://{{ hostvars[host]['ip'] | default(hostvars[host].ansible_default_ipv4['address']) }}:{{ contiv_etcd_peer_port }}{% if not loop.last %},{% endif %} + {%- endfor %} +contiv_etcd_name: |- + {% for host in groups['kube-master'] %} + {% if host == inventory_hostname -%}contiv_etcd{{ loop.index }}{%- endif %} + {% endfor %} # Parameters for Contiv api-proxy contiv_enable_api_proxy: true diff --git a/roles/network_plugin/contiv/tasks/main.yml b/roles/network_plugin/contiv/tasks/main.yml index b1ed41c24..f05090ffb 100644 --- a/roles/network_plugin/contiv/tasks/main.yml +++ b/roles/network_plugin/contiv/tasks/main.yml @@ -21,6 +21,7 @@ template: src: contiv-etcd.env.j2 dest: "{{ contiv_etcd_conf_dir }}/contiv-etcd.env" + when: inventory_hostname in groups['kube-master'] - set_fact: contiv_config_dir: "{{ contiv_config_dir }}" @@ -38,6 +39,7 @@ - {name: contiv-netplugin, file: contiv-netplugin-clusterrole.yml, type: clusterrole} - {name: contiv-netplugin, file: contiv-netplugin-serviceaccount.yml, type: serviceaccount} - {name: contiv-etcd, file: contiv-etcd.yml, type: daemonset} + - {name: contiv-etcd-proxy, file: contiv-etcd-proxy.yml, type: daemonset} - {name: contiv-netplugin, file: contiv-netplugin.yml, type: daemonset} - {name: contiv-netmaster, file: contiv-netmaster.yml, type: daemonset} diff --git a/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 b/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 new file mode 100644 index 000000000..5b4b643c2 --- /dev/null +++ b/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 @@ -0,0 +1,31 @@ +--- +kind: DaemonSet +apiVersion: extensions/v1beta1 +metadata: + name: contiv-etcd-proxy + namespace: {{ system_namespace }} + labels: + k8s-app: contiv-etcd-proxy +spec: + selector: + matchLabels: + k8s-app: contiv-etcd-proxy + template: + metadata: + labels: + k8s-app: contiv-etcd-proxy + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + hostNetwork: true + hostPID: true + containers: + - name: contiv-etcd-proxy + image: {{ contiv_etcd_image_repo }}:{{ contiv_etcd_image_tag }} + env: + - name: ETCD_LISTEN_CLIENT_URLS + value: http://127.0.0.1:{{ contiv_etcd_listen_port }} + - name: ETCD_PROXY + value: "on" + - name: ETCD_INITIAL_CLUSTER + value: {{ contiv_etcd_endpoints }} diff --git a/roles/network_plugin/contiv/templates/contiv-etcd.env.j2 b/roles/network_plugin/contiv/templates/contiv-etcd.env.j2 index 1a4efb466..e5f920af1 100644 --- a/roles/network_plugin/contiv/templates/contiv-etcd.env.j2 +++ b/roles/network_plugin/contiv/templates/contiv-etcd.env.j2 @@ -1,22 +1,8 @@ # contiv etcd config -{% if inventory_hostname in groups['kube-master'] %} export ETCD_DATA_DIR=/var/lib/etcd/contiv-data export ETCD_ADVERTISE_CLIENT_URLS={{ contiv_etcd_ad_urls }} export ETCD_INITIAL_ADVERTISE_PEER_URLS={{ contiv_etcd_peer_urls }} export ETCD_LISTEN_PEER_URLS={{ contiv_etcd_peer_urls }} export ETCD_LISTEN_CLIENT_URLS={{ contiv_etcd_listen_urls | join(",") }} -export ETCD_NAME= -{%- for host in groups['kube-master'] -%} -{%- if host == inventory_hostname -%} -contiv_etcd{{ loop.index }} -{%- endif %} -{%- endfor %} - -{% else %} -export ETCD_LISTEN_CLIENT_URLS=http://127.0.0.1:{{ contiv_etcd_listen_port }} -export ETCD_PROXY=on -{% endif %} -export ETCD_INITIAL_CLUSTER= -{%- for host in groups['kube-master'] -%} -contiv_etcd{{ loop.index }}=http://{{ hostvars[host]['ip'] | default(hostvars[host].ansible_default_ipv4['address']) }}:{{ contiv_etcd_peer_port }}, -{%- endfor -%} +export ETCD_NAME={{ contiv_etcd_name }} +export ETCD_INITIAL_CLUSTER={{ contiv_etcd_endpoints }} diff --git a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 index b5519ed45..5e2327a3c 100644 --- a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 @@ -19,26 +19,26 @@ spec: spec: hostNetwork: true hostPID: true + nodeSelector: + node-role.kubernetes.io/master: "true" tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule containers: - name: contiv-etcd - image: {{ etcd_image_repo }}:{{ etcd_image_tag }} + image: {{ contiv_etcd_image_repo }}:{{ contiv_etcd_image_tag }} command: ["sh","-c"] args: - '. {{ contiv_etcd_conf_dir }}/contiv-etcd.env && /usr/local/bin/etcd' volumeMounts: - - name: etc-contiv-etcd + - name: contiv-etcd-conf-dir mountPath: {{ contiv_etcd_conf_dir }} - - name: var-lib-etcd-contiv-data + - name: contiv-etcd-data-dir mountPath: {{ contiv_etcd_data_dir }} - securityContext: - privileged: true volumes: - - name: etc-contiv-etcd - hostPath: - path: {{ contiv_etcd_conf_dir }} - - name: var-lib-etcd-contiv-data + - name: contiv-etcd-data-dir hostPath: path: {{ contiv_etcd_data_dir }} + - name: contiv-etcd-conf-dir + hostPath: + path: {{ contiv_etcd_conf_dir }} From dad95c873bcae975e3e5addb162e83f83c0bc34b Mon Sep 17 00:00:00 2001 From: Wei Tie Date: Thu, 7 Dec 2017 23:33:29 -0800 Subject: [PATCH 2/4] Remove templating for etcd members Use a etcd-initer init container to generate etcd args, it determines etcd name by comparing its ip and etcd cluster ips. This way will make etcd configuration independent to the ansible templating so that could be easier on adding master nodes. --- roles/network_plugin/contiv/defaults/main.yml | 12 +------- roles/network_plugin/contiv/tasks/main.yml | 6 ---- .../contiv/templates/contiv-etcd.env.j2 | 8 ------ .../contiv/templates/contiv-etcd.yml.j2 | 28 +++++++++++++++++-- 4 files changed, 26 insertions(+), 28 deletions(-) delete mode 100644 roles/network_plugin/contiv/templates/contiv-etcd.env.j2 diff --git a/roles/network_plugin/contiv/defaults/main.yml b/roles/network_plugin/contiv/defaults/main.yml index 83d940173..b6e237df5 100644 --- a/roles/network_plugin/contiv/defaults/main.yml +++ b/roles/network_plugin/contiv/defaults/main.yml @@ -1,29 +1,19 @@ --- contiv_config_dir: "{{ kube_config_dir }}/contiv" -contiv_etcd_conf_dir: "/etc/contiv/etcd/" +contiv_etcd_conf_dir: "/etc/contiv/etcd" contiv_etcd_data_dir: "/var/lib/etcd/contiv-data" contiv_netmaster_port: 9999 contiv_cni_version: 0.1.0 contiv_etcd_image_repo: "{{ etcd_image_repo }}" contiv_etcd_image_tag: "{{ etcd_image_tag }}" -contiv_etcd_listen_ip: "{{ ip | default(ansible_default_ipv4['address']) }}" contiv_etcd_listen_port: 6666 contiv_etcd_peer_port: 6667 -contiv_etcd_ad_urls: http://{{ contiv_etcd_listen_ip }}:{{ contiv_etcd_listen_port }} -contiv_etcd_peer_urls: http://{{ contiv_etcd_listen_ip }}:{{ contiv_etcd_peer_port }} -contiv_etcd_listen_urls: - - http://{{ contiv_etcd_listen_ip }}:{{ contiv_etcd_listen_port }} - - http://127.0.0.1:{{ contiv_etcd_listen_port }} contiv_etcd_endpoints: |- {% for host in groups['kube-master'] -%} contiv_etcd{{ loop.index }}=http://{{ hostvars[host]['ip'] | default(hostvars[host].ansible_default_ipv4['address']) }}:{{ contiv_etcd_peer_port }}{% if not loop.last %},{% endif %} {%- endfor %} -contiv_etcd_name: |- - {% for host in groups['kube-master'] %} - {% if host == inventory_hostname -%}contiv_etcd{{ loop.index }}{%- endif %} - {% endfor %} # Parameters for Contiv api-proxy contiv_enable_api_proxy: true diff --git a/roles/network_plugin/contiv/tasks/main.yml b/roles/network_plugin/contiv/tasks/main.yml index f05090ffb..d9b372480 100644 --- a/roles/network_plugin/contiv/tasks/main.yml +++ b/roles/network_plugin/contiv/tasks/main.yml @@ -17,12 +17,6 @@ - "{{ contiv_etcd_conf_dir }}" - "{{ contiv_etcd_data_dir }}" -- name: Contiv | Create contiv etcd config env - template: - src: contiv-etcd.env.j2 - dest: "{{ contiv_etcd_conf_dir }}/contiv-etcd.env" - when: inventory_hostname in groups['kube-master'] - - set_fact: contiv_config_dir: "{{ contiv_config_dir }}" contiv_enable_api_proxy: "{{ contiv_enable_api_proxy }}" diff --git a/roles/network_plugin/contiv/templates/contiv-etcd.env.j2 b/roles/network_plugin/contiv/templates/contiv-etcd.env.j2 deleted file mode 100644 index e5f920af1..000000000 --- a/roles/network_plugin/contiv/templates/contiv-etcd.env.j2 +++ /dev/null @@ -1,8 +0,0 @@ -# contiv etcd config -export ETCD_DATA_DIR=/var/lib/etcd/contiv-data -export ETCD_ADVERTISE_CLIENT_URLS={{ contiv_etcd_ad_urls }} -export ETCD_INITIAL_ADVERTISE_PEER_URLS={{ contiv_etcd_peer_urls }} -export ETCD_LISTEN_PEER_URLS={{ contiv_etcd_peer_urls }} -export ETCD_LISTEN_CLIENT_URLS={{ contiv_etcd_listen_urls | join(",") }} -export ETCD_NAME={{ contiv_etcd_name }} -export ETCD_INITIAL_CLUSTER={{ contiv_etcd_endpoints }} diff --git a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 index 5e2327a3c..8060f4c01 100644 --- a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 @@ -24,12 +24,34 @@ spec: tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule + initContainers: + - name: contiv-etcd-init + image: ferest/etcd-initer:latest + imagePullPolicy: Always + env: + - name: ETCD_INIT_ARGSFILE + value: '{{ contiv_etcd_conf_dir }}/contiv-etcd-args' + - name: ETCD_INIT_LISTEN_PORT + value: '{{ contiv_etcd_listen_port }}' + - name: ETCD_INIT_PEER_PORT + value: '{{ contiv_etcd_peer_port }}' + - name: ETCD_INIT_CLUSTER + value: '{{ contiv_etcd_endpoints }}' + - name: ETCD_INIT_DATA_DIR + value: '{{ contiv_etcd_data_dir }}' + volumeMounts: + - name: contiv-etcd-conf-dir + mountPath: {{ contiv_etcd_conf_dir }} containers: - name: contiv-etcd image: {{ contiv_etcd_image_repo }}:{{ contiv_etcd_image_tag }} - command: ["sh","-c"] - args: - - '. {{ contiv_etcd_conf_dir }}/contiv-etcd.env && /usr/local/bin/etcd' + command: + - sh + - -c + - "/usr/local/bin/etcd $(cat $ETCD_INIT_ARGSFILE)" + env: + - name: ETCD_INIT_ARGSFILE + value: {{ contiv_etcd_conf_dir }}/contiv-etcd-args volumeMounts: - name: contiv-etcd-conf-dir mountPath: {{ contiv_etcd_conf_dir }} From 4e97225424d95359a8089fbe0585519455657532 Mon Sep 17 00:00:00 2001 From: Wei Tie Date: Wed, 13 Dec 2017 18:35:12 -0800 Subject: [PATCH 3/4] Add quote for etcd endpoints --- .../network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 b/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 index 5b4b643c2..a9690cc2f 100644 --- a/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 @@ -24,8 +24,8 @@ spec: image: {{ contiv_etcd_image_repo }}:{{ contiv_etcd_image_tag }} env: - name: ETCD_LISTEN_CLIENT_URLS - value: http://127.0.0.1:{{ contiv_etcd_listen_port }} + value: 'http://127.0.0.1:{{ contiv_etcd_listen_port }}' - name: ETCD_PROXY value: "on" - name: ETCD_INITIAL_CLUSTER - value: {{ contiv_etcd_endpoints }} + value: '{{ contiv_etcd_endpoints }}' From 3bb505d43fd09b045866ee0a6a6094d5dc59a322 Mon Sep 17 00:00:00 2001 From: Wei Tie Date: Thu, 14 Dec 2017 14:59:40 -0800 Subject: [PATCH 4/4] Remove unrequired mounts --- .../contiv/templates/contiv-netmaster.yml.j2 | 30 ------------------- .../contiv/templates/contiv-netplugin.yml.j2 | 12 -------- 2 files changed, 42 deletions(-) diff --git a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 index 3129674a8..56be2d93d 100644 --- a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 @@ -50,41 +50,11 @@ spec: securityContext: privileged: true volumeMounts: - - mountPath: /etc/openvswitch - name: etc-openvswitch - readOnly: false - - mountPath: /lib/modules - name: lib-modules - readOnly: false - - mountPath: /var/run - name: var-run - readOnly: false - mountPath: /var/contiv name: var-contiv readOnly: false - - mountPath: /etc/kubernetes/ssl - name: etc-kubernetes-ssl - readOnly: false - - mountPath: /opt/cni/bin - name: cni-bin-dir - readOnly: false volumes: # Used by contiv-netmaster - - name: etc-openvswitch - hostPath: - path: /etc/openvswitch - - name: lib-modules - hostPath: - path: /lib/modules - - name: var-run - hostPath: - path: /var/run - name: var-contiv hostPath: path: /var/contiv - - name: etc-kubernetes-ssl - hostPath: - path: /etc/kubernetes/ssl - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin diff --git a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 index b49a2e928..9c2c0a036 100644 --- a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 @@ -75,12 +75,6 @@ spec: - mountPath: /var/contiv name: var-contiv readOnly: false - - mountPath: /etc/kubernetes/pki - name: etc-kubernetes-pki - readOnly: false - - mountPath: /etc/kubernetes/ssl - name: etc-kubernetes-ssl - readOnly: false - mountPath: /opt/cni/bin name: cni-bin-dir readOnly: false @@ -101,12 +95,6 @@ spec: - name: var-contiv hostPath: path: /var/contiv - - name: etc-kubernetes-pki - hostPath: - path: /etc/kubernetes/pki - - name: etc-kubernetes-ssl - hostPath: - path: /etc/kubernetes/ssl # Used to install CNI. - name: cni-bin-dir hostPath: