Merge pull request #2037 from tiewei/contiv-etcd-split

Split contiv etcd and etcd-proxy into two daemonsets

roles/network_plugin/contiv/defaults/main.yml

@@ -1,19 +1,19 @@
 ---
 
 contiv_config_dir: "{{ kube_config_dir }}/contiv"
-contiv_etcd_conf_dir: "/etc/contiv/etcd/"
+contiv_etcd_conf_dir: "/etc/contiv/etcd"
 contiv_etcd_data_dir: "/var/lib/etcd/contiv-data"
 contiv_netmaster_port: 9999
 contiv_cni_version: 0.1.0
 
-contiv_etcd_listen_ip: "{{ ip | default(ansible_default_ipv4['address']) }}"
+contiv_etcd_image_repo: "{{ etcd_image_repo }}"
+contiv_etcd_image_tag: "{{ etcd_image_tag }}"
 contiv_etcd_listen_port: 6666
 contiv_etcd_peer_port: 6667
-contiv_etcd_ad_urls: http://{{ contiv_etcd_listen_ip }}:{{ contiv_etcd_listen_port }}
-contiv_etcd_peer_urls: http://{{ contiv_etcd_listen_ip }}:{{ contiv_etcd_peer_port }}
-contiv_etcd_listen_urls:
-  - http://{{ contiv_etcd_listen_ip }}:{{ contiv_etcd_listen_port }}
-  - http://127.0.0.1:{{ contiv_etcd_listen_port }}
+contiv_etcd_endpoints: |-
+  {% for host in groups['kube-master'] -%}
+    contiv_etcd{{ loop.index }}=http://{{ hostvars[host]['ip'] | default(hostvars[host].ansible_default_ipv4['address']) }}:{{ contiv_etcd_peer_port }}{% if not loop.last %},{% endif %}
+  {%- endfor %}
 
 # Parameters for Contiv api-proxy
 contiv_enable_api_proxy: true

roles/network_plugin/contiv/tasks/main.yml

@@ -17,11 +17,6 @@
     - "{{ contiv_etcd_conf_dir }}"
     - "{{ contiv_etcd_data_dir }}"
 
-- name: Contiv | Create contiv etcd config env
-  template:
-    src: contiv-etcd.env.j2
-    dest: "{{ contiv_etcd_conf_dir }}/contiv-etcd.env"
-
 - set_fact:
     contiv_config_dir: "{{ contiv_config_dir }}"
     contiv_enable_api_proxy: "{{ contiv_enable_api_proxy }}"
@@ -38,6 +33,7 @@
       - {name: contiv-netplugin, file: contiv-netplugin-clusterrole.yml, type: clusterrole}
       - {name: contiv-netplugin, file: contiv-netplugin-serviceaccount.yml, type: serviceaccount}
       - {name: contiv-etcd, file: contiv-etcd.yml, type: daemonset}
+      - {name: contiv-etcd-proxy, file: contiv-etcd-proxy.yml, type: daemonset}
       - {name: contiv-netplugin, file: contiv-netplugin.yml, type: daemonset}
       - {name: contiv-netmaster, file: contiv-netmaster.yml, type: daemonset}
 

roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2

@@ -0,0 +1,31 @@
+---
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: contiv-etcd-proxy
+  namespace: {{ system_namespace }}
+  labels:
+    k8s-app: contiv-etcd-proxy
+spec:
+  selector:
+    matchLabels:
+      k8s-app: contiv-etcd-proxy
+  template:
+    metadata:
+      labels:
+        k8s-app: contiv-etcd-proxy
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      hostNetwork: true
+      hostPID: true
+      containers:
+        - name: contiv-etcd-proxy
+          image: {{ contiv_etcd_image_repo }}:{{ contiv_etcd_image_tag }}
+          env:
+            - name: ETCD_LISTEN_CLIENT_URLS
+              value: 'http://127.0.0.1:{{ contiv_etcd_listen_port }}'
+            - name: ETCD_PROXY
+              value: "on"
+            - name: ETCD_INITIAL_CLUSTER
+              value: '{{ contiv_etcd_endpoints }}'

roles/network_plugin/contiv/templates/contiv-etcd.env.j2

@@ -1,22 +0,0 @@
-# contiv etcd config
-{% if inventory_hostname in groups['kube-master'] %}
-export ETCD_DATA_DIR=/var/lib/etcd/contiv-data
-export ETCD_ADVERTISE_CLIENT_URLS={{ contiv_etcd_ad_urls }}
-export ETCD_INITIAL_ADVERTISE_PEER_URLS={{ contiv_etcd_peer_urls }}
-export ETCD_LISTEN_PEER_URLS={{ contiv_etcd_peer_urls }}
-export ETCD_LISTEN_CLIENT_URLS={{ contiv_etcd_listen_urls | join(",") }}
-export ETCD_NAME=
-{%- for host in groups['kube-master'] -%}
-{%- if host == inventory_hostname -%}
-contiv_etcd{{ loop.index }}
-{%- endif %}
-{%- endfor %}
-
-{% else %}
-export ETCD_LISTEN_CLIENT_URLS=http://127.0.0.1:{{ contiv_etcd_listen_port }}
-export ETCD_PROXY=on
-{% endif %}
-export ETCD_INITIAL_CLUSTER=
-{%- for host in groups['kube-master'] -%}
-contiv_etcd{{ loop.index }}=http://{{ hostvars[host]['ip'] | default(hostvars[host].ansible_default_ipv4['address']) }}:{{ contiv_etcd_peer_port }},
-{%- endfor -%}

roles/network_plugin/contiv/templates/contiv-etcd.yml.j2

@@ -19,26 +19,48 @@ spec:
     spec:
       hostNetwork: true
       hostPID: true
+      nodeSelector:
+        node-role.kubernetes.io/master: "true"
       tolerations:
         - key: node-role.kubernetes.io/master
           effect: NoSchedule
+      initContainers:
+        - name: contiv-etcd-init
+          image: ferest/etcd-initer:latest
+          imagePullPolicy: Always
+          env:
+            - name: ETCD_INIT_ARGSFILE
+              value: '{{ contiv_etcd_conf_dir }}/contiv-etcd-args'
+            - name: ETCD_INIT_LISTEN_PORT
+              value: '{{ contiv_etcd_listen_port }}'
+            - name: ETCD_INIT_PEER_PORT
+              value: '{{ contiv_etcd_peer_port }}'
+            - name: ETCD_INIT_CLUSTER
+              value: '{{ contiv_etcd_endpoints }}'
+            - name: ETCD_INIT_DATA_DIR
+              value: '{{ contiv_etcd_data_dir }}'
+          volumeMounts:
+            - name: contiv-etcd-conf-dir
+              mountPath: {{ contiv_etcd_conf_dir }}
       containers:
         - name: contiv-etcd
-          image: {{ etcd_image_repo }}:{{ etcd_image_tag }}
-          command: ["sh","-c"]
-          args:
-            - '. {{ contiv_etcd_conf_dir }}/contiv-etcd.env && /usr/local/bin/etcd'
+          image: {{ contiv_etcd_image_repo }}:{{ contiv_etcd_image_tag }}
+          command:
+            - sh
+            - -c
+            - "/usr/local/bin/etcd $(cat $ETCD_INIT_ARGSFILE)"
+          env:
+            - name: ETCD_INIT_ARGSFILE
+              value: {{ contiv_etcd_conf_dir }}/contiv-etcd-args
           volumeMounts:
-            - name: etc-contiv-etcd
+            - name: contiv-etcd-conf-dir
               mountPath: {{ contiv_etcd_conf_dir }}
-            - name: var-lib-etcd-contiv-data
+            - name: contiv-etcd-data-dir
               mountPath: {{ contiv_etcd_data_dir }}
-          securityContext:
-            privileged: true
       volumes:
-        - name: etc-contiv-etcd
-          hostPath:
-            path: {{ contiv_etcd_conf_dir }}
-        - name: var-lib-etcd-contiv-data
+        - name: contiv-etcd-data-dir
           hostPath:
             path: {{ contiv_etcd_data_dir }}
+        - name: contiv-etcd-conf-dir
+          hostPath:
+            path: {{ contiv_etcd_conf_dir }}

roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2

@@ -50,41 +50,11 @@ spec:
           securityContext:
             privileged: true
           volumeMounts:
-            - mountPath: /etc/openvswitch
-              name: etc-openvswitch
-              readOnly: false
-            - mountPath: /lib/modules
-              name: lib-modules
-              readOnly: false
-            - mountPath: /var/run
-              name: var-run
-              readOnly: false
             - mountPath: /var/contiv
               name: var-contiv
               readOnly: false
-            - mountPath: /etc/kubernetes/ssl
-              name: etc-kubernetes-ssl
-              readOnly: false
-            - mountPath: /opt/cni/bin
-              name: cni-bin-dir
-              readOnly: false
       volumes:
         # Used by contiv-netmaster
-        - name: etc-openvswitch
-          hostPath:
-            path: /etc/openvswitch
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-        - name: var-run
-          hostPath:
-            path: /var/run
         - name: var-contiv
           hostPath:
             path: /var/contiv
-        - name: etc-kubernetes-ssl
-          hostPath:
-            path: /etc/kubernetes/ssl
-        - name: cni-bin-dir
-          hostPath:
-            path: /opt/cni/bin

roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2

@@ -75,12 +75,6 @@ spec:
             - mountPath: /var/contiv
               name: var-contiv
               readOnly: false
-            - mountPath: /etc/kubernetes/pki
-              name: etc-kubernetes-pki
-              readOnly: false
-            - mountPath: /etc/kubernetes/ssl
-              name: etc-kubernetes-ssl
-              readOnly: false
             - mountPath: /opt/cni/bin
               name: cni-bin-dir
               readOnly: false
@@ -101,12 +95,6 @@ spec:
         - name: var-contiv
           hostPath:
             path: /var/contiv
-        - name: etc-kubernetes-pki
-          hostPath:
-            path: /etc/kubernetes/pki
-        - name: etc-kubernetes-ssl
-          hostPath:
-            path: /etc/kubernetes/ssl
         # Used to install CNI.
         - name: cni-bin-dir
           hostPath: