From 27ab364df59a0b5d01b89671179a9407d5edc6a1 Mon Sep 17 00:00:00 2001 From: Alvaro Campesino <57360851+Alvaro-Campesino@users.noreply.github.com> Date: Mon, 6 Dec 2021 09:16:32 +0100 Subject: [PATCH] Improve control plane scale flow (#13) (#7989) * Improve control plane scale flow (#13) * Added version 1.20.10 of K8s * Setting first_kube_control_plane to a existing one * Setting first_kube_control_plane to a existing one * change first_kube_master for first_kube_control_plane * Ansible-lint changes --- .../tasks/define-first-kube-control.yml | 19 +++++++++++++++++++ .../control-plane/tasks/kubeadm-secondary.yml | 8 ++++---- .../control-plane/tasks/kubeadm-setup.yml | 8 ++++---- .../control-plane/tasks/kubeadm-upgrade.yml | 4 ++-- roles/kubernetes/control-plane/tasks/main.yml | 3 +++ roles/kubernetes/kubeadm/tasks/main.yml | 2 +- roles/kubespray-defaults/defaults/main.yaml | 6 +++--- .../kubernetes-services-endpoint.yml.j2 | 2 +- 8 files changed, 37 insertions(+), 15 deletions(-) create mode 100644 roles/kubernetes/control-plane/tasks/define-first-kube-control.yml diff --git a/roles/kubernetes/control-plane/tasks/define-first-kube-control.yml b/roles/kubernetes/control-plane/tasks/define-first-kube-control.yml new file mode 100644 index 000000000..d01f511bd --- /dev/null +++ b/roles/kubernetes/control-plane/tasks/define-first-kube-control.yml @@ -0,0 +1,19 @@ +--- + +- name: Check which kube-control nodes are already members of the cluster + command: "{{ bin_dir }}/kubectl get nodes --selector=node-role.kubernetes.io/control-plane -o json" + register: kube_control_planes_raw + ignore_errors: yes + changed_when: false + +- name: Set fact joined_control_panes + set_fact: + joined_control_planes: "{{ ((kube_control_planes_raw.stdout| from_json)['items'])| default([]) | map (attribute='metadata') | map (attribute='name') | list }}" + delegate_to: item + loop: "{{ groups['kube_control_plane'] }}" + when: kube_control_planes_raw is succeeded + run_once: yes + +- name: Set fact first_kube_control_plane + set_fact: + first_kube_control_plane: "{{ joined_control_planes|default([]) | first | default(groups['kube_control_plane']|first) }}" diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml b/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml index 3a4ee8aa6..b9c1180ac 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml @@ -3,7 +3,7 @@ set_fact: kubeadm_discovery_address: >- {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%} - {{ first_kube_master }}:{{ kube_apiserver_port }} + {{ first_kube_control_plane }}:{{ kube_apiserver_port }} {%- else -%} {{ kube_apiserver_endpoint | regex_replace('https://', '') }} {%- endif %} @@ -18,7 +18,7 @@ --upload-certs register: kubeadm_upload_cert when: - - inventory_hostname == groups['kube_control_plane']|first + - inventory_hostname == first_kube_control_plane - name: Parse certificate key if not set set_fact: @@ -35,7 +35,7 @@ mode: 0640 backup: yes when: - - inventory_hostname != groups['kube_control_plane']|first + - inventory_hostname != first_kube_control_plane - not kubeadm_already_run.stat.exists - name: Wait for k8s apiserver @@ -65,5 +65,5 @@ throttle: 1 until: kubeadm_join_control_plane is succeeded when: - - inventory_hostname != groups['kube_control_plane']|first + - inventory_hostname != first_kube_control_plane - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml index 48732c3d6..ef6a7ac73 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml @@ -134,7 +134,7 @@ # Retry is because upload config sometimes fails retries: 3 until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr - when: inventory_hostname == groups['kube_control_plane']|first and not kubeadm_already_run.stat.exists + when: inventory_hostname == first_kube_control_plane and not kubeadm_already_run.stat.exists failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr environment: PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" @@ -154,7 +154,7 @@ {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create {{ kubeadm_token }} changed_when: false when: - - inventory_hostname == groups['kube_control_plane']|first + - inventory_hostname == first_kube_control_plane - kubeadm_token is defined - kubeadm_refresh_token tags: @@ -167,7 +167,7 @@ retries: 5 delay: 5 until: temp_token is succeeded - delegate_to: "{{ groups['kube_control_plane'] | first }}" + delegate_to: "{{ first_kube_control_plane }}" when: kubeadm_token is not defined tags: - kubeadm_token @@ -191,7 +191,7 @@ # FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file. - name: kubeadm | Remove taint for master with node role command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} {{ item }}" - delegate_to: "{{ groups['kube_control_plane'] | first }}" + delegate_to: "{{ first_kube_control_plane }}" with_items: - "node-role.kubernetes.io/master:NoSchedule-" - "node-role.kubernetes.io/control-plane:NoSchedule-" diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml index c027a1f8d..fe690fc3f 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml @@ -24,7 +24,7 @@ # Retry is because upload config sometimes fails retries: 3 until: kubeadm_upgrade.rc == 0 - when: inventory_hostname == groups['kube_control_plane']|first + when: inventory_hostname == first_kube_control_plane failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr environment: PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" @@ -42,7 +42,7 @@ --etcd-upgrade={{ etcd_kubeadm_enabled | bool | lower }} --force register: kubeadm_upgrade - when: inventory_hostname != groups['kube_control_plane']|first + when: inventory_hostname != first_kube_control_plane failed_when: - kubeadm_upgrade.rc != 0 - '"field is immutable" not in kubeadm_upgrade.stderr' diff --git a/roles/kubernetes/control-plane/tasks/main.yml b/roles/kubernetes/control-plane/tasks/main.yml index ea2dd2d02..3f2f3b403 100644 --- a/roles/kubernetes/control-plane/tasks/main.yml +++ b/roles/kubernetes/control-plane/tasks/main.yml @@ -3,6 +3,9 @@ tags: - k8s-pre-upgrade +- name: Define nodes already joined to existing cluster and first_kube_control_plane + import_tasks: define-first-kube-control.yml + - name: Create webhook token auth config template: src: webhook-token-auth-config.yaml.j2 diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 420ea0bb6..8db58d34f 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -3,7 +3,7 @@ set_fact: kubeadm_discovery_address: >- {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%} - {{ first_kube_master }}:{{ kube_apiserver_port }} + {{ first_kube_control_plane }}:{{ kube_apiserver_port }} {%- else -%} {{ kube_apiserver_endpoint | replace("https://", "") }} {%- endif %} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 5fa3ff33d..9bd98a863 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -491,7 +491,7 @@ is_kube_master: "{{ inventory_hostname in groups['kube_control_plane'] }}" kube_apiserver_count: "{{ groups['kube_control_plane'] | length }}" kube_apiserver_address: "{{ ip | default(fallback_ips[inventory_hostname]) }}" kube_apiserver_access_address: "{{ access_ip | default(kube_apiserver_address) }}" -first_kube_master: "{{ hostvars[groups['kube_control_plane'][0]]['access_ip'] | default(hostvars[groups['kube_control_plane'][0]]['ip'] | default(fallback_ips[groups['kube_control_plane'][0]])) }}" +first_kube_control_plane: "{{ hostvars[groups['kube_control_plane'][0]]['access_ip'] | default(hostvars[groups['kube_control_plane'][0]]['ip'] | default(fallback_ips[groups['kube_control_plane'][0]])) }}" loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}" loadbalancer_apiserver_type: "nginx" # applied if only external loadbalancer_apiserver is defined, otherwise ignored @@ -502,7 +502,7 @@ kube_apiserver_global_endpoint: |- {%- elif use_localhost_as_kubeapi_loadbalancer|default(False)|bool -%} https://127.0.0.1:{{ kube_apiserver_port }} {%- else -%} - https://{{ first_kube_master }}:{{ kube_apiserver_port }} + https://{{ first_kube_control_plane }}:{{ kube_apiserver_port }} {%- endif %} kube_apiserver_endpoint: |- {% if loadbalancer_apiserver is defined -%} @@ -512,7 +512,7 @@ kube_apiserver_endpoint: |- {%- elif is_kube_master -%} https://{{ kube_apiserver_bind_address | regex_replace('0\.0\.0\.0','127.0.0.1') }}:{{ kube_apiserver_port }} {%- else -%} - https://{{ first_kube_master }}:{{ kube_apiserver_port }} + https://{{ first_kube_control_plane }}:{{ kube_apiserver_port }} {%- endif %} kube_apiserver_insecure_endpoint: >- http://{{ kube_apiserver_insecure_bind_address | regex_replace('0\.0\.0\.0','127.0.0.1') }}:{{ kube_apiserver_insecure_port }} diff --git a/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 b/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 index b2d1f4073..025aaa6d4 100644 --- a/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 +++ b/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 @@ -13,7 +13,7 @@ data: KUBERNETES_SERVICE_HOST: "127.0.0.1" KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}" {%- else %} - KUBERNETES_SERVICE_HOST: "{{ first_kube_master }}" + KUBERNETES_SERVICE_HOST: "{{ first_kube_control_plane }}" KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}" {%- endif %} {% endif %}