5cba8b1614
* Wait for kube-apiserver availability before starting upgrade I am experiencing a timing issue when upgrading from kubespray 2.11.0(k8s 1.15.3) to kubespray 2.12.6(k8s 1.16.9). The certificates get replaced in `kubeadm-secondary-legacy.yml` and kube-apiserver notices a mismatch (for a fraction of a second) between `apiserver.crt` and `apiserver.key` which causes it to restart. And sometimes ( ~ 1 out of 5 upgrades) the kube-apiserver isn't back on time for the start of the upgrade task. It fails when kubeadm checks with the kube-apiserver to start the upgrade. The kube-apiserver returns a `connect: connection refused`. I have created this small task to check the availability of the kube-apiserver before starting the upgrade, so that the upgrade will run without an issue. Signed-off-by: Rick Haan <rickhaan94@gmail.com> * Fix markdownlint * Remove old CI Co-authored-by: Maxime Guyot <maxime@root314.com>
68 lines
2.2 KiB
YAML
68 lines
2.2 KiB
YAML
---
|
|
- name: "kubeadm | Wait for master kube-apiserver"
|
|
uri:
|
|
url: "https://{{ kube_apiserver_access_address }}:{{ kube_apiserver_port }}/version"
|
|
status_code: 200
|
|
validate_certs: false
|
|
register: kube_api_server_available
|
|
until: kube_api_server_available.status == 200
|
|
retries: 180
|
|
delay: 1
|
|
when: inventory_hostname == groups['kube-master']
|
|
|
|
- name: kubeadm | Upgrade first master
|
|
command: >-
|
|
timeout -k 600s 600s
|
|
{{ bin_dir }}/kubeadm
|
|
upgrade apply -y {{ kube_version }}
|
|
--config={{ kube_config_dir }}/kubeadm-config.yaml
|
|
--ignore-preflight-errors=all
|
|
--allow-experimental-upgrades
|
|
--allow-release-candidate-upgrades
|
|
--etcd-upgrade=false
|
|
--force
|
|
register: kubeadm_upgrade
|
|
# Retry is because upload config sometimes fails
|
|
retries: 3
|
|
when: inventory_hostname == groups['kube-master']|first
|
|
failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
|
|
notify: Master | restart kubelet
|
|
|
|
# FIXME: https://github.com/kubernetes/kubeadm/issues/1498 remove stdout_lines
|
|
# check after issue is fixed
|
|
- name: kubeadm | Upgrade other masters
|
|
command: >-
|
|
timeout -k 600s 600s
|
|
{{ bin_dir }}/kubeadm
|
|
upgrade apply -y {{ kube_version }}
|
|
--config={{ kube_config_dir }}/kubeadm-config.yaml
|
|
--ignore-preflight-errors=all
|
|
--allow-experimental-upgrades
|
|
--allow-release-candidate-upgrades
|
|
--etcd-upgrade=false
|
|
--force
|
|
register: kubeadm_upgrade
|
|
when: inventory_hostname != groups['kube-master']|first
|
|
failed_when:
|
|
- kubeadm_upgrade.rc != 0
|
|
- '"field is immutable" not in kubeadm_upgrade.stderr'
|
|
- kubeadm_upgrade.stdout_lines | length > 1
|
|
notify: Master | restart kubelet
|
|
|
|
# FIXME: https://github.com/kubernetes/kubeadm/issues/1318
|
|
- name: kubeadm | scale down coredns replicas to 0 if not using coredns dns_mode
|
|
command: >-
|
|
{{ bin_dir }}/kubectl
|
|
--kubeconfig /etc/kubernetes/admin.conf
|
|
-n kube-system
|
|
scale deployment/coredns --replicas 0
|
|
register: scale_down_coredns
|
|
retries: 6
|
|
delay: 5
|
|
until: scale_down_coredns is succeeded
|
|
run_once: yes
|
|
when:
|
|
- kubeadm_scale_down_coredns_enabled
|
|
- dns_mode not in ['coredns', 'coredns_dual']
|
|
changed_when: false
|