diff --git a/docs/network_plugins.md b/docs/network_plugins.md new file mode 100644 index 000000000..98c0632d0 --- /dev/null +++ b/docs/network_plugins.md @@ -0,0 +1,36 @@ +Networking plugins +================== + +Kargo supports weave, flannel, canal and calico plugins. By default, +the plugins that require a etcd cluster, will share it with kubernetes +components. + +Separate etcd cluster for networking plugins +-------------------------------------------- + +Kargo allows users to define an external etcd cluster endpoint and +certificates/keys location for networking plugins. This isolates plugins' data from +kube components' data that lives in the internal etcd cluster. + +There are ``network_plugin_etcd_access_endpoint`` and ``network_plugin_etcd_cert_dir`` +vars to define the secure endpoint and certificates/keys location ( +defaults to ``/etc/ssl/etcd/ssl/networking_plugins``). + +It is expected the following files to be provided by a user in the given certificates +directory of the first internal (for kube components) `etcd` cluster node: + +* For calico node/cni `unprivileged` etcd access: + * ca.pem + * node.pem + * node-key.pem +* For `admin` etcd access: + * ca-key.pem + * admin.pem + * admin-key.pem + +Note, when configuring the networking plugins with ansible playbooks, that etcd node +distributes these files across all of the k8s-cluster nodes (but the internal etcd +cluster). The files are stored at the same ``network_plugin_etcd_cert_dir`` path. + +The first kube-master node must be able to reach the given external etcd endpoint via +HTTPS protocol as well. It is required for the networking plugins configuration stage. diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index 2df4ba165..f0103fd2b 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -4,5 +4,6 @@ etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/ etcd_config_dir: /etc/ssl/etcd etcd_cert_dir: "{{ etcd_config_dir }}/ssl" etcd_cert_group: root +network_plugin_etcd_cert_dir: "{{ etcd_cert_dir }}/networking_plugins" etcd_script_dir: "{{ bin_dir }}/etcd-scripts" diff --git a/roles/etcd/handlers/main.yml b/roles/etcd/handlers/main.yml index 4256490c9..014087d4b 100644 --- a/roles/etcd/handlers/main.yml +++ b/roles/etcd/handlers/main.yml @@ -27,3 +27,6 @@ set_fact: etcd_secret_changed: true +- name: set ext_etcd_secret_changed + set_fact: + ext_etcd_secret_changed: true diff --git a/roles/etcd/tasks/ext_check_certs.yml b/roles/etcd/tasks/ext_check_certs.yml new file mode 100644 index 000000000..9ab4823fe --- /dev/null +++ b/roles/etcd/tasks/ext_check_certs.yml @@ -0,0 +1,30 @@ +--- +- name: "Check_certs | check if expected external etcd certs exist on first etcd master" + stat: + path: "{{ network_plugin_etcd_cert_dir }}/ca.pem" + delegate_to: "{{groups['etcd'][0]}}" + register: ext_etcdcert_master + failed_when: not ext_etcdcert_master.stat.exists + run_once: true + +- name: "Check_certs | Set default value for 'ext_sync_certs' to false" + set_fact: + ext_sync_certs: false + +- name: "Check certs | check if a cert already exists" + stat: + path: "{{ network_plugin_etcd_cert_dir }}/ca.pem" + register: ext_etcdcert + +- name: "Check_certs | Set 'sync_certs' to true" + set_fact: + sync_certs: true + when: >- + {%- set ext_certs = {'sync': False} -%} + {%- for server in play_hosts + if (not hostvars[server].ext_etcdcert.stat.exists|default(False)) or + (hostvars[server].ext_etcdcert.stat.checksum|default('') != ext_etcdcert_master.stat.checksum|default('')) -%} + {%- set _ = ext_certs.update({'sync': True}) -%} + {%- endfor -%} + {{ ext_certs.sync }} + run_once: true diff --git a/roles/etcd/tasks/gen_certs.yml b/roles/etcd/tasks/gen_certs.yml index a12c63ac8..785db3165 100644 --- a/roles/etcd/tasks/gen_certs.yml +++ b/roles/etcd/tasks/gen_certs.yml @@ -15,6 +15,15 @@ owner=root recurse=yes +- name: Gen_certs | create ext etcd cert dir + file: + path={{ network_plugin_etcd_cert_dir }} + group={{ etcd_cert_group }} + state=directory + owner=root + recurse=yes + when: "{{ network_plugin_etcd_access_endpoint.defined }}" + - name: Gen_certs | write openssl config template: src: "openssl.conf.j2" @@ -42,6 +51,7 @@ - set_fact: master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'member.pem', 'member-key.pem'] node_certs: ['ca.pem', 'node.pem', 'node-key.pem'] + network_plugin_etcd_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'ca.pem', 'node.pem', 'node-key.pem'] tags: facts - name: Gen_certs | Gather etcd master certs @@ -60,6 +70,14 @@ when: sync_certs|default(false) notify: set etcd_secret_changed +- name: Gen_certs | Gather ext etcd node certs + shell: "tar cfz - -C {{ network_plugin_etcd_cert_dir }} {{ network_plugin_etcd_certs|join(' ') }} | base64 --wrap=0" + register: ext_etcd_node_cert_data + delegate_to: "{{groups['etcd'][0]}}" + run_once: true + when: "{{ ext_sync_certs|default(false) and network_plugin_etcd_access_endpoint.defined }}" + notify: set ext_etcd_secret_changed + - name: Gen_certs | Copy certs on masters shell: "echo '{{etcd_master_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ etcd_cert_dir }}" changed_when: false @@ -72,6 +90,12 @@ when: inventory_hostname in groups['k8s-cluster'] and sync_certs|default(false) and inventory_hostname not in groups['etcd'] +- name: Gen_certs | Copy ext etcd certs on nodes + shell: "echo '{{ext_etcd_node_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ network_plugin_etcd_cert_dir }}" + changed_when: false + when: "{{ inventory_hostname in groups['k8s-cluster'] and ext_sync_certs|default(false) and + inventory_hostname != groups['etcd'] and network_plugin_etcd_access_endpoint.defined }}" + - name: Gen_certs | check certificate permissions file: path={{ etcd_cert_dir }} @@ -81,11 +105,27 @@ recurse=yes tags: facts +- name: Gen_certs | check ext etcd certificate permissions + file: + path={{ network_plugin_etcd_cert_dir }} + group={{ etcd_cert_group }} + state=directory + owner=kube + recurse=yes + when: "{{ network_plugin_etcd_access_endpoint.defined }}" + tags: facts + - name: Gen_certs | set permissions on keys shell: chmod 0600 {{ etcd_cert_dir}}/*key.pem when: inventory_hostname in groups['etcd'] changed_when: false +- name: Gen_certs | set permissions on ext etcd keys + shell: chmod 0600 {{ network_plugin_etcd_cert_dir}}/*key.pem + when: inventory_hostname in groups['k8s-cluster'] + changed_when: false + when: "{{ network_plugin_etcd_access_endpoint.defined }}" + - name: Gen_certs | target ca-certificate store file set_fact: ca_cert_path: |- @@ -105,11 +145,18 @@ remote_src: true register: etcd_ca_cert +- name: Gen_certs | add ext etcd CA to trusted CA dir + copy: + src: "{{ network_plugin_etcd_cert_dir }}/ca.pem" + dest: "{{ ca_cert_path }}" + remote_src: true + register: ext_etcd_ca_cert + when: "{{ network_plugin_etcd_access_endpoint.defined }}" + - name: Gen_certs | update ca-certificates (Debian/Ubuntu/CoreOS) command: update-ca-certificates - when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"] + when: (etcd_ca_cert.changed or ext_etcd_ca_cert.changed) and ansible_os_family in ["Debian", "CoreOS"] - name: Gen_certs | update ca-certificates (RedHat) command: update-ca-trust extract - when: etcd_ca_cert.changed and ansible_os_family == "RedHat" - + when: (etcd_ca_cert.changed or ext_etcd_ca_cert.changed) and ansible_os_family == "RedHat" diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index 5b25a8392..3bd4f3371 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -3,6 +3,9 @@ tags: etcd-pre-upgrade - include: check_certs.yml tags: [etcd-secrets, facts] +- include: ext_check_certs.yml + when: "{{ network_plugin_etcd_access_endpoint.defined }}" + tags: [network, etcd-secrets, facts] - include: gen_certs.yml tags: etcd-secrets - include: install.yml diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index 35ad8abea..f5fa16f94 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -29,6 +29,11 @@ openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID') }}" # All clients access each node individually, instead of using a load balancer. etcd_multiaccess: true +# External etcd cluster certs for network plugins, if network_plugin_etcd_access_endpoint is defined +etcd_config_dir: /etc/ssl/etcd +etcd_cert_dir: "{{ etcd_config_dir }}/ssl" +network_plugin_etcd_cert_dir: "{{ etcd_cert_dir }}/networking_plugins" + # CoreOS cloud init config file to define /etc/resolv.conf content # for hostnet pods and infra needs resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index f57cd702e..eec5f843b 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -50,6 +50,9 @@ - set_fact: etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}" - set_fact: + network_plugins_etcd_cert_dir_real: "{% if network_plugin_etcd_access_endpoint.defined %}{{network_plugin_etcd_cert_dir}}{% else %}{{etcd_cert_dir}}{% endif %}" + network_plugins_etcd_access_endpoint: "{{ network_plugin_etcd_access_endpoint|default(etcd_access_endpoint) }}" + network_plugins_etcd_access_delagate: "{% if network_plugin_etcd_access_endpoint.defined %}{{groups['kube-master'][0]}}{% else %}{{groups['etcd'][0]}}{% endif %}" peer_with_calico_rr: "{{ 'calico-rr' in groups and groups['calico-rr']|length > 0 }}" - include: set_resolv_facts.yml diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml index 391e7c53e..c68950619 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico/defaults/main.yml @@ -11,6 +11,7 @@ overwrite_hyperkube_cni: true calico_cert_dir: /etc/calico/certs etcd_cert_dir: /etc/ssl/etcd/ssl +network_plugin_etcd_cert_dir: "{{ etcd_cert_dir }}/networking_plugins" # Global as_num (/calico/bgp/v1/global/as_num) global_as_num: "64512" diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index 19d74759c..212e6230b 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -21,7 +21,7 @@ - name: Calico | Link etcd certificates for calico-node file: - src: "{{ etcd_cert_dir }}/{{ item.s }}" + src: "{{ network_plugin_etcd_cert_dir_real }}/{{ item.s }}" dest: "{{ calico_cert_dir }}/{{ item.d }}" state: hard force: yes @@ -66,17 +66,18 @@ retries: 10 delay: 5 delegate_to: "{{groups['etcd'][0]}}" + when: not network_plugin_etcd_access_endpoint.defined run_once: true - name: Calico | Check if calico network pool has already been configured command: |- curl \ - --cacert {{ etcd_cert_dir }}/ca.pem \ - --cert {{ etcd_cert_dir}}/admin.pem \ - --key {{ etcd_cert_dir }}/admin-key.pem \ - https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool + --cacert {{ network_plugin_etcd_cert_dir_real }}/ca.pem \ + --cert {{ network_plugin_etcd_cert_dir_real }}/admin.pem \ + --key {{ network_plugin_etcd_cert_dir_real }}/admin-key.pem \ + https://{%- if etcd_multiaccess -%}{{network_plugin_etcd_access_endpoint[0]}}{%- else -%}{{network_plugin_etcd_access_endpoint}}{%- endif -%}:2379/v2/keys/calico/v1/ipam/v4/pool register: calico_conf - delegate_to: "{{groups['etcd'][0]}}" + delegate_to: "{{network_plugins_etcd_access_delagate}}" run_once: true tags: facts @@ -130,12 +131,12 @@ - name: Calico | Get calico configuration from etcd command: |- curl \ - --cacert {{ etcd_cert_dir }}/ca.pem \ - --cert {{ etcd_cert_dir}}/admin.pem \ - --key {{ etcd_cert_dir }}/admin-key.pem \ - https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool + --cacert {{ network_plugins_etcd_cert_dir_real }}/ca.pem \ + --cert {{ network_plugins_etcd_cert_dir_real }}/admin.pem \ + --key {{ network_plugins_etcd_cert_dir_real }}/admin-key.pem \ + https://{%- if etcd_multiaccess -%}{{network_plugin_etcd_access_endpoint[0]}}{%- else -%}{{network_plugin_etcd_access_endpoint}}{%- endif -%}:2379/v2/keys/calico/v1/ipam/v4/pool register: calico_pools_raw - delegate_to: "{{groups['etcd'][0]}}" + delegate_to: "{{network_plugins_etcd_access_delegate}}" run_once: true - set_fact: diff --git a/roles/network_plugin/calico/templates/calicoctl-container.j2 b/roles/network_plugin/calico/templates/calicoctl-container.j2 index 7be30928a..5d26d7d7f 100644 --- a/roles/network_plugin/calico/templates/calicoctl-container.j2 +++ b/roles/network_plugin/calico/templates/calicoctl-container.j2 @@ -1,7 +1,7 @@ #!/bin/bash /usr/bin/docker run -i --privileged --rm \ --net=host --pid=host \ --e ETCD_ENDPOINTS={{ etcd_access_endpoint }} \ +-e ETCD_ENDPOINTS={{ network_plugins_etcd_access_endpoint }} \ -e ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt \ -e ETCD_CERT_FILE=/etc/calico/certs/cert.crt \ -e ETCD_KEY_FILE=/etc/calico/certs/key.pem \ diff --git a/roles/network_plugin/calico/templates/cni-calico.conf.j2 b/roles/network_plugin/calico/templates/cni-calico.conf.j2 index 8a3016324..6a837adc5 100644 --- a/roles/network_plugin/calico/templates/cni-calico.conf.j2 +++ b/roles/network_plugin/calico/templates/cni-calico.conf.j2 @@ -4,10 +4,10 @@ "hostname": "{{ inventory_hostname }}", {% endif %} "type": "calico", - "etcd_endpoints": "{{ etcd_access_endpoint }}", - "etcd_cert_file": "{{ etcd_cert_dir }}/node.pem", - "etcd_key_file": "{{ etcd_cert_dir }}/node-key.pem", - "etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem", + "etcd_endpoints": "{{ network_plugin_etcd_access_endpoint }}", + "etcd_cert_file": "{{ network_plugin_etcd_cert_dir_real }}/node.pem", + "etcd_key_file": "{{ network_plugin_etcd_cert_dir_real }}/node-key.pem", + "etcd_ca_cert_file": "{{ network_plugin_etcd_cert_dir_real }}/ca.pem", "log_level": "info", "ipam": { "type": "calico-ipam" diff --git a/roles/network_plugin/calico/templates/network-environment.j2 b/roles/network_plugin/calico/templates/network-environment.j2 index 8fd13d36c..2c52a8b5c 100644 --- a/roles/network_plugin/calico/templates/network-environment.j2 +++ b/roles/network_plugin/calico/templates/network-environment.j2 @@ -6,7 +6,7 @@ DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }} KUBERNETES_MASTER={{ kube_apiserver_endpoint }} # IP and port of etcd instance used by Calico -ETCD_ENDPOINTS={{ etcd_access_endpoint }} +ETCD_ENDPOINTS={{ network_plugin_etcd_access_endpoint }} ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt ETCD_CERT_FILE=/etc/calico/certs/cert.crt ETCD_KEY_FILE=/etc/calico/certs/key.pem diff --git a/roles/network_plugin/canal/defaults/main.yml b/roles/network_plugin/canal/defaults/main.yml index d67d593f5..fbcc975fe 100644 --- a/roles/network_plugin/canal/defaults/main.yml +++ b/roles/network_plugin/canal/defaults/main.yml @@ -13,3 +13,4 @@ canal_log_level: "info" # Etcd SSL dirs canal_cert_dir: /etc/canal/certs etcd_cert_dir: /etc/ssl/etcd/ssl +network_plugin_etcd_cert_dir: "{{ etcd_cert_dir }}/networking_plugins" diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml index d968e9e46..82d9aa2cc 100644 --- a/roles/network_plugin/canal/tasks/main.yml +++ b/roles/network_plugin/canal/tasks/main.yml @@ -15,7 +15,7 @@ - name: Canal | Link etcd certificates for canal-node file: - src: "{{ etcd_cert_dir }}/{{ item.s }}" + src: "{{ network_plugin_etcd_cert_dir_real }}/{{ item.s }}" dest: "{{ canal_cert_dir }}/{{ item.d }}" state: hard force: yes @@ -26,10 +26,10 @@ - name: Canal | Set Flannel etcd configuration command: |- - {{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \ + {{ bin_dir }}/etcdctl --peers={{ network_plugin_etcd_access_endpoint }} \ set /{{ cluster_name }}/network/config \ '{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }' - delegate_to: "{{groups['etcd'][0]}}" + delegate_to: "{{network_plugin_etcd_access_delagate}}" run_once: true - name: Canal | Write canal configmap diff --git a/roles/network_plugin/canal/templates/canal-config.yml.j2 b/roles/network_plugin/canal/templates/canal-config.yml.j2 index 1d0d3002a..fea77a38f 100644 --- a/roles/network_plugin/canal/templates/canal-config.yml.j2 +++ b/roles/network_plugin/canal/templates/canal-config.yml.j2 @@ -7,7 +7,7 @@ metadata: name: canal-config data: # Configure this with the location of your etcd cluster. - etcd_endpoints: "{{ etcd_access_endpoint }}" + etcd_endpoints: "{{ network_plugin_etcd_access_endpoint }}" # The interface used by canal for host <-> host communication. # If left blank, then the interface is chosing using the node's diff --git a/roles/network_plugin/canal/templates/cni-canal.conf.j2 b/roles/network_plugin/canal/templates/cni-canal.conf.j2 index b835443c7..da27a3dc9 100644 --- a/roles/network_plugin/canal/templates/cni-canal.conf.j2 +++ b/roles/network_plugin/canal/templates/cni-canal.conf.j2 @@ -3,7 +3,7 @@ "type": "flannel", "delegate": { "type": "calico", - "etcd_endpoints": "{{ etcd_access_endpoint }}", + "etcd_endpoints": "{{ network_plugin_etcd_access_endpoint }}", "log_level": "info", "policy": { "type": "k8s" diff --git a/roles/network_plugin/flannel/tasks/main.yml b/roles/network_plugin/flannel/tasks/main.yml index 4dde123ae..f99dd5d9e 100644 --- a/roles/network_plugin/flannel/tasks/main.yml +++ b/roles/network_plugin/flannel/tasks/main.yml @@ -1,10 +1,10 @@ --- - name: Flannel | Set Flannel etcd configuration command: |- - {{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \ + {{ bin_dir }}/etcdctl --peers={{ network_plugin_etcd_access_endpoint }} \ set /{{ cluster_name }}/network/config \ '{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }' - delegate_to: "{{groups['etcd'][0]}}" + delegate_to: "{{network_plugins_etcd_access_delagate}}" run_once: true - name: Flannel | Create flannel pod manifest diff --git a/roles/network_plugin/flannel/templates/flannel-pod.yml b/roles/network_plugin/flannel/templates/flannel-pod.yml index 1af2152ea..08168dbfb 100644 --- a/roles/network_plugin/flannel/templates/flannel-pod.yml +++ b/roles/network_plugin/flannel/templates/flannel-pod.yml @@ -14,7 +14,7 @@ path: "/run/flannel" - name: "etcd-certs" hostPath: - path: "{{ etcd_cert_dir }}" + path: "{{ network_plugin_etcd_cert_dir_real }}" containers: - name: "flannel-container" image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}" @@ -22,7 +22,7 @@ command: - "/bin/sh" - "-c" - - "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ etcd_cert_dir }}/ca.pem -etcd-certfile {{ etcd_cert_dir }}/node.pem -etcd-keyfile {{ etcd_cert_dir }}/node-key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}" + - "/opt/bin/flanneld -etcd-endpoints {{ network_plugin_etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ network_plugin_etcd_cert_dir_real }}/ca.pem -etcd-certfile {{ network_plugin_etcd_cert_dir_real }}/node.pem -etcd-keyfile {{ network_plugin_etcd_cert_dir_real }}/node-key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}" ports: - hostPort: 10253 containerPort: 10253 @@ -33,7 +33,7 @@ - name: "subnetenv" mountPath: "/run/flannel" - name: "etcd-certs" - mountPath: "{{ etcd_cert_dir }}" + mountPath: "{{ network_plugin_etcd_cert_dir_real }}" readOnly: true securityContext: privileged: true