ac2135e450
* Fix recover-control-plane to work with etcd 3.3.x and add CI * Set default values for testcase * Add actual test jobs * Attempt to satisty gitlab ci linter * Fix ansible targets * Set etcd_member_name as stated in the docs... * Recovering from 0 masters is not supported yet * Add other master to broken_kube-master group as well * Increase number of retries to see if etcd needs more time to heal * Make number of retries for ETCD loops configurable, increase it for recovery CI and document it
149 lines
5.4 KiB
YAML
149 lines
5.4 KiB
YAML
---
|
|
- name: Configure | Check if etcd cluster is healthy
|
|
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
|
|
register: etcd_cluster_is_healthy
|
|
ignore_errors: true
|
|
changed_when: false
|
|
check_mode: no
|
|
when: is_etcd_master and etcd_cluster_setup
|
|
tags:
|
|
- facts
|
|
environment:
|
|
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
|
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
|
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
|
|
|
- name: Configure | Check if etcd-events cluster is healthy
|
|
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
|
|
register: etcd_events_cluster_is_healthy
|
|
ignore_errors: true
|
|
changed_when: false
|
|
check_mode: no
|
|
when: is_etcd_master and etcd_events_cluster_setup
|
|
tags:
|
|
- facts
|
|
environment:
|
|
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
|
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
|
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
|
|
|
- include_tasks: refresh_config.yml
|
|
when: is_etcd_master
|
|
|
|
- name: Configure | Copy etcd.service systemd file
|
|
template:
|
|
src: "etcd-{{ etcd_deployment_type }}.service.j2"
|
|
dest: /etc/systemd/system/etcd.service
|
|
backup: yes
|
|
when: is_etcd_master and etcd_cluster_setup
|
|
|
|
- name: Configure | Copy etcd-events.service systemd file
|
|
template:
|
|
src: "etcd-events-{{ etcd_deployment_type }}.service.j2"
|
|
dest: /etc/systemd/system/etcd-events.service
|
|
backup: yes
|
|
when: is_etcd_master and etcd_events_cluster_setup
|
|
|
|
- name: Configure | reload systemd
|
|
systemd:
|
|
daemon_reload: true
|
|
when: is_etcd_master
|
|
|
|
- name: Configure | Ensure etcd is running
|
|
service:
|
|
name: etcd
|
|
state: started
|
|
enabled: yes
|
|
when: is_etcd_master and etcd_cluster_setup
|
|
|
|
- name: Configure | Ensure etcd-events is running
|
|
service:
|
|
name: etcd-events
|
|
state: started
|
|
enabled: yes
|
|
when: is_etcd_master and etcd_events_cluster_setup
|
|
|
|
- name: Configure | Check if etcd cluster is healthy
|
|
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_client_url }} cluster-health | grep -q 'cluster is healthy'"
|
|
register: etcd_cluster_is_healthy
|
|
until: etcd_cluster_is_healthy.rc == 0
|
|
retries: "{{ etcd_retries }}"
|
|
delay: "{{ retry_stagger | random + 3 }}"
|
|
ignore_errors: false
|
|
changed_when: false
|
|
check_mode: no
|
|
delegate_to: "{{ groups['etcd'][0] }}"
|
|
run_once: yes
|
|
when:
|
|
- is_etcd_master
|
|
- etcd_cluster_setup
|
|
tags:
|
|
- facts
|
|
environment:
|
|
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
|
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
|
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
|
|
|
- name: Configure | Check if etcd-events cluster is healthy
|
|
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_client_url }} cluster-health | grep -q 'cluster is healthy'"
|
|
register: etcd_events_cluster_is_healthy
|
|
until: etcd_events_cluster_is_healthy.rc == 0
|
|
retries: "{{ etcd_retries }}"
|
|
delay: "{{ retry_stagger | random + 3 }}"
|
|
ignore_errors: false
|
|
changed_when: false
|
|
check_mode: no
|
|
delegate_to: "{{ groups['etcd'][0] }}"
|
|
run_once: yes
|
|
when:
|
|
- is_etcd_master
|
|
- etcd_events_cluster_setup
|
|
- etcd_cluster_setup
|
|
tags:
|
|
- facts
|
|
environment:
|
|
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
|
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
|
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
|
|
|
- name: Configure | Check if member is in etcd cluster
|
|
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
|
|
register: etcd_member_in_cluster
|
|
ignore_errors: true
|
|
changed_when: false
|
|
check_mode: no
|
|
when: is_etcd_master and etcd_cluster_setup
|
|
tags:
|
|
- facts
|
|
environment:
|
|
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
|
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
|
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
|
|
|
- name: Configure | Check if member is in etcd-events cluster
|
|
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_access_address }}"
|
|
register: etcd_events_member_in_cluster
|
|
ignore_errors: true
|
|
changed_when: false
|
|
check_mode: no
|
|
when: is_etcd_master and etcd_events_cluster_setup
|
|
tags:
|
|
- facts
|
|
environment:
|
|
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
|
|
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
|
|
ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
|
|
|
|
- name: Configure | Join member(s) to etcd cluster one at a time
|
|
include_tasks: join_etcd_member.yml
|
|
vars:
|
|
target_node: "{{ item }}"
|
|
with_items: "{{ groups['etcd'] }}"
|
|
when: inventory_hostname == item and etcd_cluster_setup and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0
|
|
|
|
- name: Configure | Join member(s) to etcd-events cluster one at a time
|
|
include_tasks: join_etcd-events_member.yml
|
|
vars:
|
|
target_node: "{{ item }}"
|
|
with_items: "{{ groups['etcd'] }}"
|
|
when: inventory_hostname == item and etcd_events_cluster_setup and etcd_events_member_in_cluster.rc != 0 and etcd_events_cluster_is_healthy.rc == 0
|