Add external etcd support for network plugins

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

docs/network_plugins.md

@@ -0,0 +1,36 @@
+Networking plugins
+==================
+
+Kargo supports weave, flannel, canal and calico plugins. By default,
+the plugins that require a etcd cluster, will share it with kubernetes
+components.
+
+Separate etcd cluster for networking plugins
+--------------------------------------------
+
+Kargo allows users to define an external etcd cluster endpoint and
+certificates/keys location for networking plugins. This isolates plugins' data from
+kube components' data that lives in the internal etcd cluster.
+
+There are ``network_plugin_etcd_access_endpoint`` and ``network_plugin_etcd_cert_dir``
+vars to define the secure endpoint and certificates/keys location (
+defaults to ``/etc/ssl/etcd/ssl/networking_plugins``).
+
+It is expected the following files to be provided by a user in the given certificates
+directory of the first internal (for kube components) `etcd` cluster node:
+
+* For calico node/cni `unprivileged` etcd access:
+  * ca.pem
+  * node.pem
+  * node-key.pem
+* For `admin` etcd access:
+  * ca-key.pem
+  * admin.pem
+  * admin-key.pem
+
+Note, when configuring the networking plugins with ansible playbooks, that etcd node
+distributes these files across all of the k8s-cluster nodes (but the internal etcd
+cluster). The files are stored at the same ``network_plugin_etcd_cert_dir`` path.
+
+The first kube-master node must be able to reach the given external etcd endpoint via
+HTTPS protocol as well. It is required for the networking plugins configuration stage.

roles/etcd/defaults/main.yml

@@ -4,5 +4,6 @@ etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/
 etcd_config_dir: /etc/ssl/etcd
 etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
 etcd_cert_group: root
+network_plugin_etcd_cert_dir: "{{ etcd_cert_dir }}/networking_plugins"
 
 etcd_script_dir: "{{ bin_dir }}/etcd-scripts"

roles/etcd/handlers/main.yml

@@ -27,3 +27,6 @@
   set_fact:
     etcd_secret_changed: true
 
+- name: set ext_etcd_secret_changed
+  set_fact:
+    ext_etcd_secret_changed: true

roles/etcd/tasks/ext_check_certs.yml

@@ -0,0 +1,30 @@
+---
+- name: "Check_certs | check if expected external etcd certs exist on first etcd master"
+  stat:
+    path: "{{ network_plugin_etcd_cert_dir }}/ca.pem"
+  delegate_to: "{{groups['etcd'][0]}}"
+  register: ext_etcdcert_master
+  failed_when: not ext_etcdcert_master.stat.exists
+  run_once: true
+
+- name: "Check_certs | Set default value for 'ext_sync_certs' to false"
+  set_fact:
+    ext_sync_certs: false
+
+- name: "Check certs | check if a cert already exists"
+  stat:
+    path: "{{ network_plugin_etcd_cert_dir }}/ca.pem"
+  register: ext_etcdcert
+
+- name: "Check_certs | Set 'sync_certs' to true"
+  set_fact:
+    sync_certs: true
+  when: >-
+      {%- set ext_certs = {'sync': False} -%}
+      {%- for server in play_hosts
+         if (not hostvars[server].ext_etcdcert.stat.exists|default(False)) or
+         (hostvars[server].ext_etcdcert.stat.checksum|default('') != ext_etcdcert_master.stat.checksum|default('')) -%}
+         {%- set _ = ext_certs.update({'sync': True}) -%}
+      {%- endfor -%}
+      {{ ext_certs.sync }}
+  run_once: true

roles/etcd/tasks/gen_certs.yml

@@ -15,6 +15,15 @@
     owner=root
     recurse=yes
 
+- name: Gen_certs | create ext etcd cert dir
+  file:
+    path={{ network_plugin_etcd_cert_dir }}
+    group={{ etcd_cert_group }}
+    state=directory
+    owner=root
+    recurse=yes
+  when: "{{ network_plugin_etcd_access_endpoint.defined }}"
+
 - name: Gen_certs | write openssl config
   template:
     src: "openssl.conf.j2"
@@ -42,6 +51,7 @@
 - set_fact:
     master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'member.pem', 'member-key.pem']
     node_certs: ['ca.pem', 'node.pem', 'node-key.pem']
+    network_plugin_etcd_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'ca.pem', 'node.pem', 'node-key.pem']
   tags: facts
 
 - name: Gen_certs | Gather etcd master certs
@@ -60,6 +70,14 @@
   when: sync_certs|default(false)
   notify: set etcd_secret_changed
 
+- name: Gen_certs | Gather ext etcd node certs
+  shell: "tar cfz - -C {{ network_plugin_etcd_cert_dir }} {{ network_plugin_etcd_certs|join(' ') }} | base64 --wrap=0"
+  register: ext_etcd_node_cert_data
+  delegate_to: "{{groups['etcd'][0]}}"
+  run_once: true
+  when: "{{ ext_sync_certs|default(false) and network_plugin_etcd_access_endpoint.defined }}"
+  notify: set ext_etcd_secret_changed
+
 - name: Gen_certs | Copy certs on masters
   shell: "echo '{{etcd_master_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ etcd_cert_dir }}"
   changed_when: false
@@ -72,6 +90,12 @@
   when: inventory_hostname in groups['k8s-cluster'] and sync_certs|default(false) and
         inventory_hostname not in groups['etcd']
 
+- name: Gen_certs | Copy ext etcd certs on nodes
+  shell: "echo '{{ext_etcd_node_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ network_plugin_etcd_cert_dir }}"
+  changed_when: false
+  when: "{{ inventory_hostname in groups['k8s-cluster'] and  ext_sync_certs|default(false) and
+        inventory_hostname != groups['etcd'] and network_plugin_etcd_access_endpoint.defined }}"
+
 - name: Gen_certs | check certificate permissions
   file:
     path={{ etcd_cert_dir }}
@@ -81,11 +105,27 @@
     recurse=yes
   tags: facts
 
+- name: Gen_certs | check ext etcd certificate permissions
+  file:
+    path={{ network_plugin_etcd_cert_dir }}
+    group={{ etcd_cert_group }}
+    state=directory
+    owner=kube
+    recurse=yes
+  when: "{{ network_plugin_etcd_access_endpoint.defined }}"
+  tags: facts
+
 - name: Gen_certs | set permissions on keys
   shell: chmod 0600 {{ etcd_cert_dir}}/*key.pem
   when: inventory_hostname in groups['etcd']
   changed_when: false
 
+- name: Gen_certs | set permissions on ext etcd keys
+  shell: chmod 0600 {{ network_plugin_etcd_cert_dir}}/*key.pem
+  when: inventory_hostname in groups['k8s-cluster']
+  changed_when: false
+  when: "{{ network_plugin_etcd_access_endpoint.defined }}"
+
 - name: Gen_certs | target ca-certificate store file
   set_fact:
     ca_cert_path: |-
@@ -105,11 +145,18 @@
     remote_src: true
   register: etcd_ca_cert
 
+- name: Gen_certs | add ext etcd CA to trusted CA dir
+  copy:
+    src: "{{ network_plugin_etcd_cert_dir }}/ca.pem"
+    dest: "{{ ca_cert_path }}"
+    remote_src: true
+  register: ext_etcd_ca_cert
+  when: "{{ network_plugin_etcd_access_endpoint.defined }}"
+
 - name: Gen_certs | update ca-certificates (Debian/Ubuntu/CoreOS)
   command: update-ca-certificates
-  when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"]
+  when: (etcd_ca_cert.changed or ext_etcd_ca_cert.changed) and ansible_os_family in ["Debian", "CoreOS"]
 
 - name: Gen_certs | update ca-certificates (RedHat)
   command: update-ca-trust extract
-  when: etcd_ca_cert.changed and ansible_os_family == "RedHat"
-
+  when: (etcd_ca_cert.changed or ext_etcd_ca_cert.changed) and ansible_os_family == "RedHat"

roles/etcd/tasks/main.yml

@@ -3,6 +3,9 @@
   tags: etcd-pre-upgrade
 - include: check_certs.yml
   tags: [etcd-secrets, facts]
+- include: ext_check_certs.yml
+  when: "{{ network_plugin_etcd_access_endpoint.defined }}"
+  tags: [network, etcd-secrets, facts]
 - include: gen_certs.yml
   tags: etcd-secrets
 - include: install.yml

roles/kubernetes/preinstall/defaults/main.yml

@@ -29,6 +29,11 @@ openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')  }}"
 # All clients access each node individually, instead of using a load balancer.
 etcd_multiaccess: true
 
+# External etcd cluster certs for network plugins, if network_plugin_etcd_access_endpoint is defined
+etcd_config_dir: /etc/ssl/etcd
+etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
+network_plugin_etcd_cert_dir: "{{ etcd_cert_dir }}/networking_plugins"
+
 # CoreOS cloud init config file to define /etc/resolv.conf content
 # for hostnet pods and infra needs
 resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf

roles/kubernetes/preinstall/tasks/set_facts.yml

@@ -50,6 +50,9 @@
 - set_fact:
     etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}"
 - set_fact:
+    network_plugins_etcd_cert_dir_real: "{% if network_plugin_etcd_access_endpoint.defined %}{{network_plugin_etcd_cert_dir}}{% else %}{{etcd_cert_dir}}{% endif %}"
+    network_plugins_etcd_access_endpoint: "{{ network_plugin_etcd_access_endpoint|default(etcd_access_endpoint) }}"
+    network_plugins_etcd_access_delagate: "{% if network_plugin_etcd_access_endpoint.defined %}{{groups['kube-master'][0]}}{% else %}{{groups['etcd'][0]}}{% endif %}"
     peer_with_calico_rr: "{{ 'calico-rr' in groups and groups['calico-rr']|length > 0 }}"
 
 - include: set_resolv_facts.yml

roles/network_plugin/calico/defaults/main.yml

@@ -11,6 +11,7 @@ overwrite_hyperkube_cni: true
 
 calico_cert_dir: /etc/calico/certs
 etcd_cert_dir: /etc/ssl/etcd/ssl
+network_plugin_etcd_cert_dir: "{{ etcd_cert_dir }}/networking_plugins"
 
 # Global as_num (/calico/bgp/v1/global/as_num)
 global_as_num: "64512"

roles/network_plugin/calico/tasks/main.yml

@@ -21,7 +21,7 @@
 
 - name: Calico | Link etcd certificates for calico-node
   file:
-    src: "{{ etcd_cert_dir }}/{{ item.s }}"
+    src: "{{ network_plugin_etcd_cert_dir_real }}/{{ item.s }}"
     dest: "{{ calico_cert_dir }}/{{ item.d }}"
     state: hard
     force: yes
@@ -66,17 +66,18 @@
   retries: 10
   delay: 5
   delegate_to: "{{groups['etcd'][0]}}"
+  when: not network_plugin_etcd_access_endpoint.defined
   run_once: true
 
 - name: Calico | Check if calico network pool has already been configured
   command: |-
     curl \
-      --cacert {{ etcd_cert_dir }}/ca.pem \
-      --cert {{ etcd_cert_dir}}/admin.pem \
-      --key {{ etcd_cert_dir }}/admin-key.pem \
-      https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool
+      --cacert {{ network_plugin_etcd_cert_dir_real }}/ca.pem \
+      --cert {{ network_plugin_etcd_cert_dir_real }}/admin.pem \
+      --key {{ network_plugin_etcd_cert_dir_real }}/admin-key.pem \
+      https://{%- if etcd_multiaccess -%}{{network_plugin_etcd_access_endpoint[0]}}{%- else -%}{{network_plugin_etcd_access_endpoint}}{%- endif -%}:2379/v2/keys/calico/v1/ipam/v4/pool
   register: calico_conf
-  delegate_to: "{{groups['etcd'][0]}}"
+  delegate_to: "{{network_plugins_etcd_access_delagate}}"
   run_once: true
   tags: facts
 
@@ -130,12 +131,12 @@
 - name: Calico | Get calico configuration from etcd
   command: |-
     curl \
-      --cacert {{ etcd_cert_dir }}/ca.pem \
-      --cert {{ etcd_cert_dir}}/admin.pem \
-      --key {{ etcd_cert_dir }}/admin-key.pem \
-      https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool
+      --cacert {{ network_plugins_etcd_cert_dir_real }}/ca.pem \
+      --cert {{ network_plugins_etcd_cert_dir_real }}/admin.pem \
+      --key {{ network_plugins_etcd_cert_dir_real }}/admin-key.pem \
+      https://{%- if etcd_multiaccess -%}{{network_plugin_etcd_access_endpoint[0]}}{%- else -%}{{network_plugin_etcd_access_endpoint}}{%- endif -%}:2379/v2/keys/calico/v1/ipam/v4/pool
   register: calico_pools_raw
-  delegate_to: "{{groups['etcd'][0]}}"
+  delegate_to: "{{network_plugins_etcd_access_delegate}}"
   run_once: true
 
 - set_fact:

roles/network_plugin/calico/templates/calicoctl-container.j2

@@ -1,7 +1,7 @@
 #!/bin/bash
 /usr/bin/docker run -i --privileged --rm \
 --net=host --pid=host \
--e ETCD_ENDPOINTS={{ etcd_access_endpoint }} \
+-e ETCD_ENDPOINTS={{ network_plugins_etcd_access_endpoint }} \
 -e ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt \
 -e ETCD_CERT_FILE=/etc/calico/certs/cert.crt \
 -e ETCD_KEY_FILE=/etc/calico/certs/key.pem \

roles/network_plugin/calico/templates/cni-calico.conf.j2

@@ -4,10 +4,10 @@
   "hostname": "{{ inventory_hostname }}",
 {% endif %}
   "type": "calico",
-  "etcd_endpoints": "{{ etcd_access_endpoint }}",
-  "etcd_cert_file": "{{ etcd_cert_dir }}/node.pem",
-  "etcd_key_file": "{{ etcd_cert_dir }}/node-key.pem",
-  "etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem",
+  "etcd_endpoints": "{{ network_plugin_etcd_access_endpoint }}",
+  "etcd_cert_file": "{{ network_plugin_etcd_cert_dir_real }}/node.pem",
+  "etcd_key_file": "{{ network_plugin_etcd_cert_dir_real }}/node-key.pem",
+  "etcd_ca_cert_file": "{{ network_plugin_etcd_cert_dir_real }}/ca.pem",
   "log_level": "info",
   "ipam": {
     "type": "calico-ipam"

roles/network_plugin/calico/templates/network-environment.j2

@@ -6,7 +6,7 @@ DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }}
 KUBERNETES_MASTER={{ kube_apiserver_endpoint }}
 
 # IP and port of etcd instance used by Calico
-ETCD_ENDPOINTS={{ etcd_access_endpoint }}
+ETCD_ENDPOINTS={{ network_plugin_etcd_access_endpoint }}
 ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt
 ETCD_CERT_FILE=/etc/calico/certs/cert.crt
 ETCD_KEY_FILE=/etc/calico/certs/key.pem

roles/network_plugin/canal/defaults/main.yml

@@ -13,3 +13,4 @@ canal_log_level: "info"
 # Etcd SSL dirs
 canal_cert_dir: /etc/canal/certs
 etcd_cert_dir: /etc/ssl/etcd/ssl
+network_plugin_etcd_cert_dir: "{{ etcd_cert_dir }}/networking_plugins"

roles/network_plugin/canal/tasks/main.yml

@@ -15,7 +15,7 @@
 
 - name: Canal | Link etcd certificates for canal-node
   file:
-    src: "{{ etcd_cert_dir }}/{{ item.s }}"
+    src: "{{ network_plugin_etcd_cert_dir_real }}/{{ item.s }}"
     dest: "{{ canal_cert_dir }}/{{ item.d }}"
     state: hard
     force: yes
@@ -26,10 +26,10 @@
 
 - name: Canal | Set Flannel etcd configuration
   command: |-
-    {{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \
+    {{ bin_dir }}/etcdctl --peers={{ network_plugin_etcd_access_endpoint }} \
     set /{{ cluster_name }}/network/config \
     '{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }'
-  delegate_to: "{{groups['etcd'][0]}}"
+  delegate_to: "{{network_plugin_etcd_access_delagate}}"
   run_once: true
 
 - name: Canal | Write canal configmap

roles/network_plugin/canal/templates/canal-config.yml.j2

@@ -7,7 +7,7 @@ metadata:
   name: canal-config
 data:
   # Configure this with the location of your etcd cluster.
-  etcd_endpoints: "{{ etcd_access_endpoint }}"
+  etcd_endpoints: "{{ network_plugin_etcd_access_endpoint }}"
 
   # The interface used by canal for host <-> host communication.
   # If left blank, then the interface is chosing using the node's

roles/network_plugin/canal/templates/cni-canal.conf.j2

@@ -3,7 +3,7 @@
   "type": "flannel",
   "delegate": {
     "type": "calico",
-    "etcd_endpoints": "{{ etcd_access_endpoint }}",
+    "etcd_endpoints": "{{ network_plugin_etcd_access_endpoint }}",
     "log_level": "info",
     "policy": {
       "type": "k8s"

roles/network_plugin/flannel/tasks/main.yml

@@ -1,10 +1,10 @@
 ---
 - name: Flannel | Set Flannel etcd configuration
   command: |-
-    {{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \
+    {{ bin_dir }}/etcdctl --peers={{ network_plugin_etcd_access_endpoint }} \
     set /{{ cluster_name }}/network/config \
     '{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }'
-  delegate_to: "{{groups['etcd'][0]}}"
+  delegate_to: "{{network_plugins_etcd_access_delagate}}"
   run_once: true
 
 - name: Flannel | Create flannel pod manifest

roles/network_plugin/flannel/templates/flannel-pod.yml

@@ -14,7 +14,7 @@
           path: "/run/flannel"
       - name: "etcd-certs"
         hostPath:
-          path: "{{ etcd_cert_dir }}"
+          path: "{{ network_plugin_etcd_cert_dir_real }}"
     containers:
       - name: "flannel-container"
         image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}"
@@ -22,7 +22,7 @@
         command:
           - "/bin/sh"
           - "-c"
-          - "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ etcd_cert_dir }}/ca.pem -etcd-certfile {{ etcd_cert_dir }}/node.pem -etcd-keyfile {{ etcd_cert_dir }}/node-key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}"
+          - "/opt/bin/flanneld -etcd-endpoints {{ network_plugin_etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ network_plugin_etcd_cert_dir_real }}/ca.pem -etcd-certfile {{ network_plugin_etcd_cert_dir_real }}/node.pem -etcd-keyfile {{ network_plugin_etcd_cert_dir_real }}/node-key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}"
         ports:
           - hostPort: 10253
             containerPort: 10253
@@ -33,7 +33,7 @@
           - name: "subnetenv"
             mountPath: "/run/flannel"
           - name: "etcd-certs"
-            mountPath: "{{ etcd_cert_dir }}"
+            mountPath: "{{ network_plugin_etcd_cert_dir_real }}"
             readOnly: true
         securityContext:
           privileged: true