Updated etcd cert check tasks to detect when new cert gen is required (#7219)

* Added force_etcd_cert_refresh var to maintain existing functionality. Broke out etcd node cert syncing from member and admin cert sync logic. Now first etcd will sync node certs to other etcd members on every run to keep all etcds up to date after adding additional worker nodes to the cluster

* Updated etcd cert check tasks to better detect when new certificates need to be generated

* Move usage of force_etcd_cert_refresh var to gen_certs fact set

* Force etcd cert generation per server if force_etcd_cert_refresh is set to true

* Include gathering of node certs even if k8s-cluster member and in etcd group.

* Removed run_once due to when statement

roles/etcd/defaults/main.yml

@@ -12,6 +12,7 @@ etcd_data_dir: "/var/lib/etcd"
 # Number of etcd backups to retain. Set to a value < 0 to retain all backups
 etcd_backup_retention_count: -1
 
+force_etcd_cert_refresh: true
 etcd_config_dir: /etc/ssl/etcd
 etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
 etcd_cert_dir_mode: "0700"

roles/etcd/tasks/check_certs.yml

@@ -1,8 +1,8 @@
 ---
-- name: "Check_certs | check if all certs have already been generated on first master"
+- name: "Check_certs | Register certs that have already been generated on first etcd node"
   find:
     paths: "{{ etcd_cert_dir }}"
-    patterns: "ca.pem,node*.pem"
+    patterns: "ca.pem,node*.pem,member*.pem,admin*.pem"
     get_checksum: true
   delegate_to: "{{ groups['etcd'][0] }}"
   register: etcdcert_master
@@ -14,40 +14,66 @@
     gen_certs: false
     etcd_secret_changed: false
 
-- name: "Check certs | check if a cert already exists on node"
+- name: "Check certs | Register ca and etcd admin/member certs on etcd hosts"
   stat:
     path: "{{ etcd_cert_dir }}/{{ item }}"
-  register: etcdcert_node
+  register: etcd_member_certs
+  when: inventory_hostname in groups['etcd']
   with_items:
     - ca.pem
+    - member-{{ inventory_hostname }}.pem
+    - member-{{ inventory_hostname }}-key.pem
+    - admin-{{ inventory_hostname }}.pem
+    - admin-{{ inventory_hostname }}-key.pem
+
+- name: "Check certs | Register ca and etcd node certs on kubernetes hosts"
+  stat:
+    path: "{{ etcd_cert_dir }}/{{ item }}"
+  register: etcd_node_certs
+  when: (('calico-rr' in groups and inventory_hostname in groups['calico-rr']) or
+        inventory_hostname in groups['k8s-cluster'])
+  with_items:
+    - ca.pem
+    - node-{{ inventory_hostname }}.pem
     - node-{{ inventory_hostname }}-key.pem
 
-- name: "Check_certs | Set 'gen_certs' to true"
+- name: "Check_certs | Set 'gen_certs' to true if expected certificates are not on the first etcd node"
   set_fact:
     gen_certs: true
-  when: not item in etcdcert_master.files|map(attribute='path') | list
+  when: force_etcd_cert_refresh or not item in etcdcert_master.files|map(attribute='path') | list
   run_once: true
   with_items: "{{ expected_files }}"
   vars:
     expected_files: >-
       ['{{ etcd_cert_dir }}/ca.pem',
-      {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort %}
-      {% for host in all_etcd_hosts %}
-        '{{ etcd_cert_dir }}/node-{{ host }}-key.pem',
+      {% set etcd_members = groups['etcd'] %}
+      {% for host in etcd_members %}
+        '{{ etcd_cert_dir }}/admin-{{ host }}.pem',
         '{{ etcd_cert_dir }}/admin-{{ host }}-key.pem',
-        '{{ etcd_cert_dir }}/member-{{ host }}-key.pem'
+        '{{ etcd_cert_dir }}/member-{{ host }}.pem',
+        '{{ etcd_cert_dir }}/member-{{ host }}-key.pem',
+      {% endfor %}
+      {% set k8s_nodes = groups['k8s-cluster']|union(groups['calico-rr']|default([]))|unique|sort %}
+      {% for host in k8s_nodes %}
+        '{{ etcd_cert_dir }}/node-{{ host }}.pem',
+        '{{ etcd_cert_dir }}/node-{{ host }}-key.pem'
         {% if not loop.last %}{{','}}{% endif %}
       {% endfor %}]
 
-- name: "Check_certs | Set 'gen_master_certs' to true"
+- name: "Check_certs | Set 'gen_master_certs' object to track whether member and admin certs exist on first etcd node"
   set_fact:
     gen_master_certs: |-
       {
-      {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort -%}
+      {% set etcd_members = groups['etcd'] -%}
       {% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %}
-      {% for host in all_etcd_hosts -%}
-        {% set host_cert = "%s/member-%s-key.pem"|format(etcd_cert_dir, host) %}
-        {% if host_cert in existing_certs -%}
+      {% for host in etcd_members -%}
+        {% set member_cert = "%s/member-%s.pem"|format(etcd_cert_dir, host) %}
+        {% set member_key = "%s/member-%s-key.pem"|format(etcd_cert_dir, host) %}
+        {% set admin_cert = "%s/admin-%s.pem"|format(etcd_cert_dir, host) %}
+        {% set admin_key = "%s/admin-%s-key.pem"|format(etcd_cert_dir, host) %}
+        {% if force_etcd_cert_refresh -%}
+        "{{ host }}": True,
+        {% elif member_cert in existing_certs and member_key in existing_certs and admin_cert in existing_certs and admin_key in existing_certs  -%}
         "{{ host }}": False,
         {% else -%}
         "{{ host }}": True,
@@ -56,15 +82,18 @@
       }
   run_once: true
 
-- name: "Check_certs | Set 'gen_node_certs' to true"
+- name: "Check_certs | Set 'gen_node_certs' object to track whether node certs exist on first etcd node"
   set_fact:
     gen_node_certs: |-
       {
-      {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort -%}
+      {% set k8s_nodes = groups['k8s-cluster']|union(groups['calico-rr']|default([]))|unique|sort -%}
       {% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %}
-      {% for host in all_etcd_hosts -%}
-        {% set host_cert = "%s/node-%s-key.pem"|format(etcd_cert_dir, host) %}
-        {% if host_cert in existing_certs -%}
+      {% for host in k8s_nodes -%}
+        {% set host_cert = "%s/node-%s.pem"|format(etcd_cert_dir, host) %}
+        {% set host_key = "%s/node-%s-key.pem"|format(etcd_cert_dir, host) %}
+        {% if force_etcd_cert_refresh -%}
+        "{{ host }}": True,
+        {% elif host_cert in existing_certs and host_key in existing_certs -%}
         "{{ host }}": False,
         {% else -%}
         "{{ host }}": True,
@@ -73,12 +102,41 @@
       }
   run_once: true
 
+- name: "Check_certs | Set 'etcd_member_requires_sync' to true if ca or member/admin cert and key don't exist on etcd member or checksum doesn't match"
+  set_fact:
+    etcd_member_requires_sync: true
+  when:
+    - inventory_hostname in groups['etcd']
+    - (not etcd_member_certs.results[0].stat.exists|default(false)) or
+      (not etcd_member_certs.results[1].stat.exists|default(false)) or
+      (not etcd_member_certs.results[2].stat.exists|default(false)) or
+      (not etcd_member_certs.results[3].stat.exists|default(false)) or
+      (not etcd_member_certs.results[4].stat.exists|default(false)) or
+      (etcd_member_certs.results[0].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[0].stat.path)|map(attribute="checksum")|first|default('')) or
+      (etcd_member_certs.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[1].stat.path)|map(attribute="checksum")|first|default('')) or
+      (etcd_member_certs.results[2].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[2].stat.path)|map(attribute="checksum")|first|default('')) or
+      (etcd_member_certs.results[3].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[3].stat.path)|map(attribute="checksum")|first|default('')) or
+      (etcd_member_certs.results[4].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[4].stat.path)|map(attribute="checksum")|first|default(''))
+
+- name: "Check_certs | Set 'kubernetes_host_requires_sync' to true if ca or node cert and key don't exist on kubernetes host or checksum doesn't match"
+  set_fact:
+    kubernetes_host_requires_sync: true
+  when:
+    - (('calico-rr' in groups and inventory_hostname in groups['calico-rr']) or
+      inventory_hostname in groups['k8s-cluster']) and
+      inventory_hostname not in groups['etcd']
+    - (not etcd_node_certs.results[0].stat.exists|default(false)) or
+      (not etcd_node_certs.results[1].stat.exists|default(false)) or
+      (not etcd_node_certs.results[2].stat.exists|default(false)) or
+      (etcd_node_certs.results[0].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[0].stat.path)|map(attribute="checksum")|first|default('')) or
+      (etcd_node_certs.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[1].stat.path)|map(attribute="checksum")|first|default('')) or
+      (etcd_node_certs.results[2].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[2].stat.path)|map(attribute="checksum")|first|default(''))
+
 - name: "Check_certs | Set 'sync_certs' to true"
   set_fact:
     sync_certs: true
   when:
-    - gen_node_certs[inventory_hostname] or
-      gen_master_certs[inventory_hostname] or
-      (not etcdcert_node.results[0].stat.exists|default(false)) or
-      (not etcdcert_node.results[1].stat.exists|default(false)) or
-      (etcdcert_node.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcdcert_node.results[1].stat.path)|map(attribute="checksum")|first|default(''))
+    - etcd_member_requires_sync|default(false) or
+      kubernetes_host_requires_sync|default(false) or
+      (inventory_hostname in gen_master_certs and gen_master_certs[inventory_hostname]) or
+      (inventory_hostname in gen_node_certs and gen_node_certs[inventory_hostname])

roles/etcd/tasks/gen_certs_script.yml

@@ -66,11 +66,10 @@
               {% endfor %}"
   run_once: yes
   delegate_to: "{{ groups['etcd'][0] }}"
-  when:
-    - gen_certs|default(false)
+  when: gen_certs|default(false)
   notify: set etcd_secret_changed
 
-- name: Gen_certs | Gather etcd master certs
+- name: Gen_certs | Gather etcd member and admin certs from first etcd node
   slurp:
     src: "{{ item }}"
   register: etcd_master_certs
@@ -83,10 +82,6 @@
         '{{ etcd_cert_dir }}/member-{{ node }}.pem',
         '{{ etcd_cert_dir }}/member-{{ node }}-key.pem',
         {% endfor %}]"
-    - "[{% for node in (groups['k8s-cluster'] + groups['calico-rr']|default([]))|unique %}
-        '{{ etcd_cert_dir }}/node-{{ node }}.pem',
-        '{{ etcd_cert_dir }}/node-{{ node }}-key.pem',
-        {% endfor %}]"
   delegate_to: "{{ groups['etcd'][0] }}"
   when:
     - inventory_hostname in groups['etcd']
@@ -94,7 +89,7 @@
     - inventory_hostname != groups['etcd'][0]
   notify: set etcd_secret_changed
 
-- name: Gen_certs | Write etcd master certs
+- name: Gen_certs | Write etcd member and admin certs to other etcd nodes
   copy:
     dest: "{{ item.item }}"
     content: "{{ item.content | b64decode }}"
@@ -109,6 +104,35 @@
   loop_control:
     label: "{{ item.item }}"
 
+- name: Gen_certs | Gather node certs from first etcd node
+  slurp:
+    src: "{{ item }}"
+  register: etcd_master_node_certs
+  with_items:
+    - "[{% for node in (groups['k8s-cluster'] + groups['calico-rr']|default([]))|unique %}
+        '{{ etcd_cert_dir }}/node-{{ node }}.pem',
+        '{{ etcd_cert_dir }}/node-{{ node }}-key.pem',
+        {% endfor %}]"
+  delegate_to: "{{ groups['etcd'][0] }}"
+  when:
+    - inventory_hostname in groups['etcd']
+    - inventory_hostname != groups['etcd'][0]
+  notify: set etcd_secret_changed
+
+- name: Gen_certs | Write node certs to other etcd nodes
+  copy:
+    dest: "{{ item.item }}"
+    content: "{{ item.content | b64decode }}"
+    group: "{{ etcd_cert_group }}"
+    owner: kube
+    mode: 0640
+  with_items: "{{ etcd_master_node_certs.results }}"
+  when:
+    - inventory_hostname in groups['etcd']
+    - inventory_hostname != groups['etcd'][0]
+  loop_control:
+    label: "{{ item.item }}"
+
 - name: Gen_certs | Set cert names per node
   set_fact:
     my_etcd_node_certs: [ 'ca.pem',