diff --git a/roles/etcd/tasks/configure.yml b/roles/etcd/tasks/configure.yml index e3f9c31dd..ae51eb1a7 100644 --- a/roles/etcd/tasks/configure.yml +++ b/roles/etcd/tasks/configure.yml @@ -2,9 +2,10 @@ - name: Configure | Check if etcd cluster is healthy shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'" register: etcd_cluster_is_healthy - ignore_errors: true + failed_when: false changed_when: false check_mode: no + run_once: yes when: is_etcd_master and etcd_cluster_setup tags: - facts @@ -16,9 +17,10 @@ - name: Configure | Check if etcd-events cluster is healthy shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'" register: etcd_events_cluster_is_healthy - ignore_errors: true + failed_when: false changed_when: false check_mode: no + run_once: yes when: is_etcd_master and etcd_events_cluster_setup tags: - facts @@ -49,22 +51,26 @@ daemon_reload: true when: is_etcd_master +# when scaling new etcd will fail to start - name: Configure | Ensure etcd is running service: name: etcd state: started enabled: yes + ignore_errors: "{{ etcd_cluster_is_healthy.rc == 0 }}" when: is_etcd_master and etcd_cluster_setup +# when scaling new etcd will fail to start - name: Configure | Ensure etcd-events is running service: name: etcd-events state: started enabled: yes + ignore_errors: "{{ etcd_events_cluster_is_healthy.rc == 0 }}" when: is_etcd_master and etcd_events_cluster_setup -- name: Configure | Check if etcd cluster is healthy - shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_client_url }} cluster-health | grep -q 'cluster is healthy'" +- name: Configure | Wait for etcd cluster to be healthy + shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'" register: etcd_cluster_is_healthy until: etcd_cluster_is_healthy.rc == 0 retries: 4 @@ -72,7 +78,6 @@ ignore_errors: false changed_when: false check_mode: no - delegate_to: "{{ groups['etcd'][0] }}" run_once: yes when: - is_etcd_master @@ -84,8 +89,8 @@ ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" -- name: Configure | Check if etcd-events cluster is healthy - shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_client_url }} cluster-health | grep -q 'cluster is healthy'" +- name: Configure | Wait for etcd-events cluster to be healthy + shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'" register: etcd_events_cluster_is_healthy until: etcd_events_cluster_is_healthy.rc == 0 retries: 4 @@ -93,12 +98,10 @@ ignore_errors: false changed_when: false check_mode: no - delegate_to: "{{ groups['etcd'][0] }}" run_once: yes when: - is_etcd_master - etcd_events_cluster_setup - - etcd_cluster_setup tags: - facts environment: @@ -136,14 +139,10 @@ - name: Configure | Join member(s) to etcd cluster one at a time include_tasks: join_etcd_member.yml - vars: - target_node: "{{ item }}" with_items: "{{ groups['etcd'] }}" when: inventory_hostname == item and etcd_cluster_setup and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0 - name: Configure | Join member(s) to etcd-events cluster one at a time include_tasks: join_etcd-events_member.yml - vars: - target_node: "{{ item }}" with_items: "{{ groups['etcd'] }}" when: inventory_hostname == item and etcd_events_cluster_setup and etcd_events_member_in_cluster.rc != 0 and etcd_events_cluster_is_healthy.rc == 0 diff --git a/roles/etcd/tasks/join_etcd-events_member.yml b/roles/etcd/tasks/join_etcd-events_member.yml index b75460c41..a4110bffc 100644 --- a/roles/etcd/tasks/join_etcd-events_member.yml +++ b/roles/etcd/tasks/join_etcd-events_member.yml @@ -5,7 +5,6 @@ until: member_add_result.rc == 0 retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: target_node == inventory_hostname environment: ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" @@ -21,7 +20,6 @@ {{ etcd_member_name }}={{ etcd_events_peer_url }} {%- endif -%} {%- endfor -%} - when: target_node == inventory_hostname - name: Join Member | Ensure member is in etcd-events cluster shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_events_access_address }}" @@ -30,7 +28,12 @@ check_mode: no tags: - facts - when: target_node == inventory_hostname environment: ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + +- name: Configure | Ensure etcd-events is running + service: + name: etcd-events + state: started + enabled: yes diff --git a/roles/etcd/tasks/join_etcd_member.yml b/roles/etcd/tasks/join_etcd_member.yml index d512eb78a..9ad8eec21 100644 --- a/roles/etcd/tasks/join_etcd_member.yml +++ b/roles/etcd/tasks/join_etcd_member.yml @@ -5,7 +5,6 @@ until: member_add_result.rc == 0 retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: target_node == inventory_hostname environment: ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" @@ -22,7 +21,6 @@ {{ etcd_member_name }}={{ etcd_peer_url }} {%- endif -%} {%- endfor -%} - when: target_node == inventory_hostname - name: Join Member | Ensure member is in etcd cluster shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}" @@ -31,8 +29,13 @@ check_mode: no tags: - facts - when: target_node == inventory_hostname environment: ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" + +- name: Configure | Ensure etcd is running + service: + name: etcd + state: started + enabled: yes diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index 549a34a1f..d34128553 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -103,6 +103,37 @@ - not upgrade_cluster_setup - kubeadm_already_run.stat.exists +- name: kubeadm | Check if apiserver.crt contains all needed SANs + command: openssl x509 -noout -in "{{ kube_cert_dir }}/apiserver.crt" -checkip "{{ item }}" + with_items: "{{ apiserver_sans }}" + register: apiserver_sans_check + changed_when: "'does match certificate' not in apiserver_sans_check.stdout" + when: + - inventory_hostname == groups['kube-master']|first + - kubeadm_already_run.stat.exists + +- name: kubeadm | regenerate apiserver cert 1/2 + file: + state: absent + path: "{{ kube_cert_dir }}/{{ item }}" + with_items: + - apiserver.crt + - apiserver.key + when: + - inventory_hostname == groups['kube-master']|first + - kubeadm_already_run.stat.exists + - apiserver_sans_check.changed + +- name: kubeadm | regenerate apiserver cert 2/2 + command: >- + {{ bin_dir }}/kubeadm + init phase certs apiserver + --config={{ kube_config_dir }}/kubeadm-config.yaml + when: + - inventory_hostname == groups['kube-master']|first + - kubeadm_already_run.stat.exists + - apiserver_sans_check.changed + - name: kubeadm | Initialize first master command: >- timeout -k 300s 300s diff --git a/roles/kubernetes/preinstall/tasks/0090-etchosts.yml b/roles/kubernetes/preinstall/tasks/0090-etchosts.yml index 0cf2fa2fe..d7b18012f 100644 --- a/roles/kubernetes/preinstall/tasks/0090-etchosts.yml +++ b/roles/kubernetes/preinstall/tasks/0090-etchosts.yml @@ -59,3 +59,8 @@ backup: yes unsafe_writes: yes with_dict: "{{ etc_hosts_localhosts_dict_target }}" + +# gather facts to update ansible_fqdn +- name: Update facts + setup: + gather_subset: min