diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index 8da2df988..6014a9ea0 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -12,6 +12,7 @@ etcd_data_dir: "/var/lib/etcd" # Number of etcd backups to retain. Set to a value < 0 to retain all backups etcd_backup_retention_count: -1 +force_etcd_cert_refresh: true etcd_config_dir: /etc/ssl/etcd etcd_cert_dir: "{{ etcd_config_dir }}/ssl" etcd_cert_dir_mode: "0700" diff --git a/roles/etcd/tasks/check_certs.yml b/roles/etcd/tasks/check_certs.yml index d3aaa9c23..21e79015a 100644 --- a/roles/etcd/tasks/check_certs.yml +++ b/roles/etcd/tasks/check_certs.yml @@ -1,8 +1,8 @@ --- -- name: "Check_certs | check if all certs have already been generated on first master" +- name: "Check_certs | Register certs that have already been generated on first etcd node" find: paths: "{{ etcd_cert_dir }}" - patterns: "ca.pem,node*.pem" + patterns: "ca.pem,node*.pem,member*.pem,admin*.pem" get_checksum: true delegate_to: "{{ groups['etcd'][0] }}" register: etcdcert_master @@ -14,40 +14,66 @@ gen_certs: false etcd_secret_changed: false -- name: "Check certs | check if a cert already exists on node" +- name: "Check certs | Register ca and etcd admin/member certs on etcd hosts" stat: path: "{{ etcd_cert_dir }}/{{ item }}" - register: etcdcert_node + register: etcd_member_certs + when: inventory_hostname in groups['etcd'] with_items: - ca.pem + - member-{{ inventory_hostname }}.pem + - member-{{ inventory_hostname }}-key.pem + - admin-{{ inventory_hostname }}.pem + - admin-{{ inventory_hostname }}-key.pem + +- name: "Check certs | Register ca and etcd node certs on kubernetes hosts" + stat: + path: "{{ etcd_cert_dir }}/{{ item }}" + register: etcd_node_certs + when: (('calico-rr' in groups and inventory_hostname in groups['calico-rr']) or + inventory_hostname in groups['k8s-cluster']) + with_items: + - ca.pem + - node-{{ inventory_hostname }}.pem - node-{{ inventory_hostname }}-key.pem -- name: "Check_certs | Set 'gen_certs' to true" +- name: "Check_certs | Set 'gen_certs' to true if expected certificates are not on the first etcd node" set_fact: gen_certs: true - when: not item in etcdcert_master.files|map(attribute='path') | list + when: force_etcd_cert_refresh or not item in etcdcert_master.files|map(attribute='path') | list run_once: true with_items: "{{ expected_files }}" vars: expected_files: >- ['{{ etcd_cert_dir }}/ca.pem', - {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort %} - {% for host in all_etcd_hosts %} - '{{ etcd_cert_dir }}/node-{{ host }}-key.pem', + {% set etcd_members = groups['etcd'] %} + {% for host in etcd_members %} + '{{ etcd_cert_dir }}/admin-{{ host }}.pem', '{{ etcd_cert_dir }}/admin-{{ host }}-key.pem', - '{{ etcd_cert_dir }}/member-{{ host }}-key.pem' + '{{ etcd_cert_dir }}/member-{{ host }}.pem', + '{{ etcd_cert_dir }}/member-{{ host }}-key.pem', + {% endfor %} + {% set k8s_nodes = groups['k8s-cluster']|union(groups['calico-rr']|default([]))|unique|sort %} + {% for host in k8s_nodes %} + '{{ etcd_cert_dir }}/node-{{ host }}.pem', + '{{ etcd_cert_dir }}/node-{{ host }}-key.pem' {% if not loop.last %}{{','}}{% endif %} {% endfor %}] -- name: "Check_certs | Set 'gen_master_certs' to true" +- name: "Check_certs | Set 'gen_master_certs' object to track whether member and admin certs exist on first etcd node" set_fact: gen_master_certs: |- { - {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort -%} + {% set etcd_members = groups['etcd'] -%} {% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %} - {% for host in all_etcd_hosts -%} - {% set host_cert = "%s/member-%s-key.pem"|format(etcd_cert_dir, host) %} - {% if host_cert in existing_certs -%} + {% for host in etcd_members -%} + {% set member_cert = "%s/member-%s.pem"|format(etcd_cert_dir, host) %} + {% set member_key = "%s/member-%s-key.pem"|format(etcd_cert_dir, host) %} + {% set admin_cert = "%s/admin-%s.pem"|format(etcd_cert_dir, host) %} + {% set admin_key = "%s/admin-%s-key.pem"|format(etcd_cert_dir, host) %} + {% if force_etcd_cert_refresh -%} + "{{ host }}": True, + {% elif member_cert in existing_certs and member_key in existing_certs and admin_cert in existing_certs and admin_key in existing_certs -%} "{{ host }}": False, {% else -%} "{{ host }}": True, @@ -56,15 +82,18 @@ } run_once: true -- name: "Check_certs | Set 'gen_node_certs' to true" +- name: "Check_certs | Set 'gen_node_certs' object to track whether node certs exist on first etcd node" set_fact: gen_node_certs: |- { - {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort -%} + {% set k8s_nodes = groups['k8s-cluster']|union(groups['calico-rr']|default([]))|unique|sort -%} {% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %} - {% for host in all_etcd_hosts -%} - {% set host_cert = "%s/node-%s-key.pem"|format(etcd_cert_dir, host) %} - {% if host_cert in existing_certs -%} + {% for host in k8s_nodes -%} + {% set host_cert = "%s/node-%s.pem"|format(etcd_cert_dir, host) %} + {% set host_key = "%s/node-%s-key.pem"|format(etcd_cert_dir, host) %} + {% if force_etcd_cert_refresh -%} + "{{ host }}": True, + {% elif host_cert in existing_certs and host_key in existing_certs -%} "{{ host }}": False, {% else -%} "{{ host }}": True, @@ -73,12 +102,41 @@ } run_once: true +- name: "Check_certs | Set 'etcd_member_requires_sync' to true if ca or member/admin cert and key don't exist on etcd member or checksum doesn't match" + set_fact: + etcd_member_requires_sync: true + when: + - inventory_hostname in groups['etcd'] + - (not etcd_member_certs.results[0].stat.exists|default(false)) or + (not etcd_member_certs.results[1].stat.exists|default(false)) or + (not etcd_member_certs.results[2].stat.exists|default(false)) or + (not etcd_member_certs.results[3].stat.exists|default(false)) or + (not etcd_member_certs.results[4].stat.exists|default(false)) or + (etcd_member_certs.results[0].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[0].stat.path)|map(attribute="checksum")|first|default('')) or + (etcd_member_certs.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[1].stat.path)|map(attribute="checksum")|first|default('')) or + (etcd_member_certs.results[2].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[2].stat.path)|map(attribute="checksum")|first|default('')) or + (etcd_member_certs.results[3].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[3].stat.path)|map(attribute="checksum")|first|default('')) or + (etcd_member_certs.results[4].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[4].stat.path)|map(attribute="checksum")|first|default('')) + +- name: "Check_certs | Set 'kubernetes_host_requires_sync' to true if ca or node cert and key don't exist on kubernetes host or checksum doesn't match" + set_fact: + kubernetes_host_requires_sync: true + when: + - (('calico-rr' in groups and inventory_hostname in groups['calico-rr']) or + inventory_hostname in groups['k8s-cluster']) and + inventory_hostname not in groups['etcd'] + - (not etcd_node_certs.results[0].stat.exists|default(false)) or + (not etcd_node_certs.results[1].stat.exists|default(false)) or + (not etcd_node_certs.results[2].stat.exists|default(false)) or + (etcd_node_certs.results[0].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[0].stat.path)|map(attribute="checksum")|first|default('')) or + (etcd_node_certs.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[1].stat.path)|map(attribute="checksum")|first|default('')) or + (etcd_node_certs.results[2].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[2].stat.path)|map(attribute="checksum")|first|default('')) + - name: "Check_certs | Set 'sync_certs' to true" set_fact: sync_certs: true when: - - gen_node_certs[inventory_hostname] or - gen_master_certs[inventory_hostname] or - (not etcdcert_node.results[0].stat.exists|default(false)) or - (not etcdcert_node.results[1].stat.exists|default(false)) or - (etcdcert_node.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcdcert_node.results[1].stat.path)|map(attribute="checksum")|first|default('')) + - etcd_member_requires_sync|default(false) or + kubernetes_host_requires_sync|default(false) or + (inventory_hostname in gen_master_certs and gen_master_certs[inventory_hostname]) or + (inventory_hostname in gen_node_certs and gen_node_certs[inventory_hostname]) diff --git a/roles/etcd/tasks/gen_certs_script.yml b/roles/etcd/tasks/gen_certs_script.yml index 36a8e2fc6..893e61c19 100644 --- a/roles/etcd/tasks/gen_certs_script.yml +++ b/roles/etcd/tasks/gen_certs_script.yml @@ -66,11 +66,10 @@ {% endfor %}" run_once: yes delegate_to: "{{ groups['etcd'][0] }}" - when: - - gen_certs|default(false) + when: gen_certs|default(false) notify: set etcd_secret_changed -- name: Gen_certs | Gather etcd master certs +- name: Gen_certs | Gather etcd member and admin certs from first etcd node slurp: src: "{{ item }}" register: etcd_master_certs @@ -83,10 +82,6 @@ '{{ etcd_cert_dir }}/member-{{ node }}.pem', '{{ etcd_cert_dir }}/member-{{ node }}-key.pem', {% endfor %}]" - - "[{% for node in (groups['k8s-cluster'] + groups['calico-rr']|default([]))|unique %} - '{{ etcd_cert_dir }}/node-{{ node }}.pem', - '{{ etcd_cert_dir }}/node-{{ node }}-key.pem', - {% endfor %}]" delegate_to: "{{ groups['etcd'][0] }}" when: - inventory_hostname in groups['etcd'] @@ -94,7 +89,7 @@ - inventory_hostname != groups['etcd'][0] notify: set etcd_secret_changed -- name: Gen_certs | Write etcd master certs +- name: Gen_certs | Write etcd member and admin certs to other etcd nodes copy: dest: "{{ item.item }}" content: "{{ item.content | b64decode }}" @@ -109,6 +104,35 @@ loop_control: label: "{{ item.item }}" +- name: Gen_certs | Gather node certs from first etcd node + slurp: + src: "{{ item }}" + register: etcd_master_node_certs + with_items: + - "[{% for node in (groups['k8s-cluster'] + groups['calico-rr']|default([]))|unique %} + '{{ etcd_cert_dir }}/node-{{ node }}.pem', + '{{ etcd_cert_dir }}/node-{{ node }}-key.pem', + {% endfor %}]" + delegate_to: "{{ groups['etcd'][0] }}" + when: + - inventory_hostname in groups['etcd'] + - inventory_hostname != groups['etcd'][0] + notify: set etcd_secret_changed + +- name: Gen_certs | Write node certs to other etcd nodes + copy: + dest: "{{ item.item }}" + content: "{{ item.content | b64decode }}" + group: "{{ etcd_cert_group }}" + owner: kube + mode: 0640 + with_items: "{{ etcd_master_node_certs.results }}" + when: + - inventory_hostname in groups['etcd'] + - inventory_hostname != groups['etcd'][0] + loop_control: + label: "{{ item.item }}" + - name: Gen_certs | Set cert names per node set_fact: my_etcd_node_certs: [ 'ca.pem',