Updated etcd cert check tasks to detect when new cert gen is required (#7219)

* Added force_etcd_cert_refresh var to maintain existing functionality. Broke out etcd node cert syncing from member and admin cert sync logic. Now first etcd will sync node certs to other etcd members on every run to keep all etcds up to date after adding additional worker nodes to the cluster

* Updated etcd cert check tasks to better detect when new certificates need to be generated

* Move usage of force_etcd_cert_refresh var to gen_certs fact set

* Force etcd cert generation per server if force_etcd_cert_refresh is set to true

* Include gathering of node certs even if k8s-cluster member and in etcd group.

* Removed run_once due to when statement
This commit is contained in:
David Louks 2021-02-09 03:53:22 -06:00 committed by GitHub
parent e3ab665e90
commit aad78840a0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 116 additions and 33 deletions

View file

@ -12,6 +12,7 @@ etcd_data_dir: "/var/lib/etcd"
# Number of etcd backups to retain. Set to a value < 0 to retain all backups # Number of etcd backups to retain. Set to a value < 0 to retain all backups
etcd_backup_retention_count: -1 etcd_backup_retention_count: -1
force_etcd_cert_refresh: true
etcd_config_dir: /etc/ssl/etcd etcd_config_dir: /etc/ssl/etcd
etcd_cert_dir: "{{ etcd_config_dir }}/ssl" etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
etcd_cert_dir_mode: "0700" etcd_cert_dir_mode: "0700"

View file

@ -1,8 +1,8 @@
--- ---
- name: "Check_certs | check if all certs have already been generated on first master" - name: "Check_certs | Register certs that have already been generated on first etcd node"
find: find:
paths: "{{ etcd_cert_dir }}" paths: "{{ etcd_cert_dir }}"
patterns: "ca.pem,node*.pem" patterns: "ca.pem,node*.pem,member*.pem,admin*.pem"
get_checksum: true get_checksum: true
delegate_to: "{{ groups['etcd'][0] }}" delegate_to: "{{ groups['etcd'][0] }}"
register: etcdcert_master register: etcdcert_master
@ -14,40 +14,66 @@
gen_certs: false gen_certs: false
etcd_secret_changed: false etcd_secret_changed: false
- name: "Check certs | check if a cert already exists on node" - name: "Check certs | Register ca and etcd admin/member certs on etcd hosts"
stat: stat:
path: "{{ etcd_cert_dir }}/{{ item }}" path: "{{ etcd_cert_dir }}/{{ item }}"
register: etcdcert_node register: etcd_member_certs
when: inventory_hostname in groups['etcd']
with_items: with_items:
- ca.pem - ca.pem
- member-{{ inventory_hostname }}.pem
- member-{{ inventory_hostname }}-key.pem
- admin-{{ inventory_hostname }}.pem
- admin-{{ inventory_hostname }}-key.pem
- name: "Check certs | Register ca and etcd node certs on kubernetes hosts"
stat:
path: "{{ etcd_cert_dir }}/{{ item }}"
register: etcd_node_certs
when: (('calico-rr' in groups and inventory_hostname in groups['calico-rr']) or
inventory_hostname in groups['k8s-cluster'])
with_items:
- ca.pem
- node-{{ inventory_hostname }}.pem
- node-{{ inventory_hostname }}-key.pem - node-{{ inventory_hostname }}-key.pem
- name: "Check_certs | Set 'gen_certs' to true" - name: "Check_certs | Set 'gen_certs' to true if expected certificates are not on the first etcd node"
set_fact: set_fact:
gen_certs: true gen_certs: true
when: not item in etcdcert_master.files|map(attribute='path') | list when: force_etcd_cert_refresh or not item in etcdcert_master.files|map(attribute='path') | list
run_once: true run_once: true
with_items: "{{ expected_files }}" with_items: "{{ expected_files }}"
vars: vars:
expected_files: >- expected_files: >-
['{{ etcd_cert_dir }}/ca.pem', ['{{ etcd_cert_dir }}/ca.pem',
{% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort %} {% set etcd_members = groups['etcd'] %}
{% for host in all_etcd_hosts %} {% for host in etcd_members %}
'{{ etcd_cert_dir }}/node-{{ host }}-key.pem', '{{ etcd_cert_dir }}/admin-{{ host }}.pem',
'{{ etcd_cert_dir }}/admin-{{ host }}-key.pem', '{{ etcd_cert_dir }}/admin-{{ host }}-key.pem',
'{{ etcd_cert_dir }}/member-{{ host }}-key.pem' '{{ etcd_cert_dir }}/member-{{ host }}.pem',
'{{ etcd_cert_dir }}/member-{{ host }}-key.pem',
{% endfor %}
{% set k8s_nodes = groups['k8s-cluster']|union(groups['calico-rr']|default([]))|unique|sort %}
{% for host in k8s_nodes %}
'{{ etcd_cert_dir }}/node-{{ host }}.pem',
'{{ etcd_cert_dir }}/node-{{ host }}-key.pem'
{% if not loop.last %}{{','}}{% endif %} {% if not loop.last %}{{','}}{% endif %}
{% endfor %}] {% endfor %}]
- name: "Check_certs | Set 'gen_master_certs' to true" - name: "Check_certs | Set 'gen_master_certs' object to track whether member and admin certs exist on first etcd node"
set_fact: set_fact:
gen_master_certs: |- gen_master_certs: |-
{ {
{% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort -%} {% set etcd_members = groups['etcd'] -%}
{% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %} {% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %}
{% for host in all_etcd_hosts -%} {% for host in etcd_members -%}
{% set host_cert = "%s/member-%s-key.pem"|format(etcd_cert_dir, host) %} {% set member_cert = "%s/member-%s.pem"|format(etcd_cert_dir, host) %}
{% if host_cert in existing_certs -%} {% set member_key = "%s/member-%s-key.pem"|format(etcd_cert_dir, host) %}
{% set admin_cert = "%s/admin-%s.pem"|format(etcd_cert_dir, host) %}
{% set admin_key = "%s/admin-%s-key.pem"|format(etcd_cert_dir, host) %}
{% if force_etcd_cert_refresh -%}
"{{ host }}": True,
{% elif member_cert in existing_certs and member_key in existing_certs and admin_cert in existing_certs and admin_key in existing_certs -%}
"{{ host }}": False, "{{ host }}": False,
{% else -%} {% else -%}
"{{ host }}": True, "{{ host }}": True,
@ -56,15 +82,18 @@
} }
run_once: true run_once: true
- name: "Check_certs | Set 'gen_node_certs' to true" - name: "Check_certs | Set 'gen_node_certs' object to track whether node certs exist on first etcd node"
set_fact: set_fact:
gen_node_certs: |- gen_node_certs: |-
{ {
{% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort -%} {% set k8s_nodes = groups['k8s-cluster']|union(groups['calico-rr']|default([]))|unique|sort -%}
{% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %} {% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %}
{% for host in all_etcd_hosts -%} {% for host in k8s_nodes -%}
{% set host_cert = "%s/node-%s-key.pem"|format(etcd_cert_dir, host) %} {% set host_cert = "%s/node-%s.pem"|format(etcd_cert_dir, host) %}
{% if host_cert in existing_certs -%} {% set host_key = "%s/node-%s-key.pem"|format(etcd_cert_dir, host) %}
{% if force_etcd_cert_refresh -%}
"{{ host }}": True,
{% elif host_cert in existing_certs and host_key in existing_certs -%}
"{{ host }}": False, "{{ host }}": False,
{% else -%} {% else -%}
"{{ host }}": True, "{{ host }}": True,
@ -73,12 +102,41 @@
} }
run_once: true run_once: true
- name: "Check_certs | Set 'etcd_member_requires_sync' to true if ca or member/admin cert and key don't exist on etcd member or checksum doesn't match"
set_fact:
etcd_member_requires_sync: true
when:
- inventory_hostname in groups['etcd']
- (not etcd_member_certs.results[0].stat.exists|default(false)) or
(not etcd_member_certs.results[1].stat.exists|default(false)) or
(not etcd_member_certs.results[2].stat.exists|default(false)) or
(not etcd_member_certs.results[3].stat.exists|default(false)) or
(not etcd_member_certs.results[4].stat.exists|default(false)) or
(etcd_member_certs.results[0].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[0].stat.path)|map(attribute="checksum")|first|default('')) or
(etcd_member_certs.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[1].stat.path)|map(attribute="checksum")|first|default('')) or
(etcd_member_certs.results[2].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[2].stat.path)|map(attribute="checksum")|first|default('')) or
(etcd_member_certs.results[3].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[3].stat.path)|map(attribute="checksum")|first|default('')) or
(etcd_member_certs.results[4].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_member_certs.results[4].stat.path)|map(attribute="checksum")|first|default(''))
- name: "Check_certs | Set 'kubernetes_host_requires_sync' to true if ca or node cert and key don't exist on kubernetes host or checksum doesn't match"
set_fact:
kubernetes_host_requires_sync: true
when:
- (('calico-rr' in groups and inventory_hostname in groups['calico-rr']) or
inventory_hostname in groups['k8s-cluster']) and
inventory_hostname not in groups['etcd']
- (not etcd_node_certs.results[0].stat.exists|default(false)) or
(not etcd_node_certs.results[1].stat.exists|default(false)) or
(not etcd_node_certs.results[2].stat.exists|default(false)) or
(etcd_node_certs.results[0].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[0].stat.path)|map(attribute="checksum")|first|default('')) or
(etcd_node_certs.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[1].stat.path)|map(attribute="checksum")|first|default('')) or
(etcd_node_certs.results[2].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcd_node_certs.results[2].stat.path)|map(attribute="checksum")|first|default(''))
- name: "Check_certs | Set 'sync_certs' to true" - name: "Check_certs | Set 'sync_certs' to true"
set_fact: set_fact:
sync_certs: true sync_certs: true
when: when:
- gen_node_certs[inventory_hostname] or - etcd_member_requires_sync|default(false) or
gen_master_certs[inventory_hostname] or kubernetes_host_requires_sync|default(false) or
(not etcdcert_node.results[0].stat.exists|default(false)) or (inventory_hostname in gen_master_certs and gen_master_certs[inventory_hostname]) or
(not etcdcert_node.results[1].stat.exists|default(false)) or (inventory_hostname in gen_node_certs and gen_node_certs[inventory_hostname])
(etcdcert_node.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcdcert_node.results[1].stat.path)|map(attribute="checksum")|first|default(''))

View file

@ -66,11 +66,10 @@
{% endfor %}" {% endfor %}"
run_once: yes run_once: yes
delegate_to: "{{ groups['etcd'][0] }}" delegate_to: "{{ groups['etcd'][0] }}"
when: when: gen_certs|default(false)
- gen_certs|default(false)
notify: set etcd_secret_changed notify: set etcd_secret_changed
- name: Gen_certs | Gather etcd master certs - name: Gen_certs | Gather etcd member and admin certs from first etcd node
slurp: slurp:
src: "{{ item }}" src: "{{ item }}"
register: etcd_master_certs register: etcd_master_certs
@ -83,10 +82,6 @@
'{{ etcd_cert_dir }}/member-{{ node }}.pem', '{{ etcd_cert_dir }}/member-{{ node }}.pem',
'{{ etcd_cert_dir }}/member-{{ node }}-key.pem', '{{ etcd_cert_dir }}/member-{{ node }}-key.pem',
{% endfor %}]" {% endfor %}]"
- "[{% for node in (groups['k8s-cluster'] + groups['calico-rr']|default([]))|unique %}
'{{ etcd_cert_dir }}/node-{{ node }}.pem',
'{{ etcd_cert_dir }}/node-{{ node }}-key.pem',
{% endfor %}]"
delegate_to: "{{ groups['etcd'][0] }}" delegate_to: "{{ groups['etcd'][0] }}"
when: when:
- inventory_hostname in groups['etcd'] - inventory_hostname in groups['etcd']
@ -94,7 +89,7 @@
- inventory_hostname != groups['etcd'][0] - inventory_hostname != groups['etcd'][0]
notify: set etcd_secret_changed notify: set etcd_secret_changed
- name: Gen_certs | Write etcd master certs - name: Gen_certs | Write etcd member and admin certs to other etcd nodes
copy: copy:
dest: "{{ item.item }}" dest: "{{ item.item }}"
content: "{{ item.content | b64decode }}" content: "{{ item.content | b64decode }}"
@ -109,6 +104,35 @@
loop_control: loop_control:
label: "{{ item.item }}" label: "{{ item.item }}"
- name: Gen_certs | Gather node certs from first etcd node
slurp:
src: "{{ item }}"
register: etcd_master_node_certs
with_items:
- "[{% for node in (groups['k8s-cluster'] + groups['calico-rr']|default([]))|unique %}
'{{ etcd_cert_dir }}/node-{{ node }}.pem',
'{{ etcd_cert_dir }}/node-{{ node }}-key.pem',
{% endfor %}]"
delegate_to: "{{ groups['etcd'][0] }}"
when:
- inventory_hostname in groups['etcd']
- inventory_hostname != groups['etcd'][0]
notify: set etcd_secret_changed
- name: Gen_certs | Write node certs to other etcd nodes
copy:
dest: "{{ item.item }}"
content: "{{ item.content | b64decode }}"
group: "{{ etcd_cert_group }}"
owner: kube
mode: 0640
with_items: "{{ etcd_master_node_certs.results }}"
when:
- inventory_hostname in groups['etcd']
- inventory_hostname != groups['etcd'][0]
loop_control:
label: "{{ item.item }}"
- name: Gen_certs | Set cert names per node - name: Gen_certs | Set cert names per node
set_fact: set_fact:
my_etcd_node_certs: [ 'ca.pem', my_etcd_node_certs: [ 'ca.pem',