Fix reconfigure and upgrade cluster (#3938)

extra_playbooks/upgrade-only-k8s.yml

@@ -46,7 +46,7 @@
     - { role: kubespray-defaults}
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: kubernetes/node, tags: node }
-    - { role: kubernetes/master, tags: master }
+    - { role: kubernetes/master, tags: master, upgrade_cluster_setup: true }
     - { role: kubernetes/client, tags: client }
     - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
     - { role: upgrade/post-upgrade, tags: post-upgrade }

roles/kubernetes/master/defaults/main.yml

@@ -1,4 +1,7 @@
 ---
+# disable upgrade cluster
+upgrade_cluster_setup: false
+
 # An experimental dev/test only dynamic volumes provisioner,
 # for PetSets. Works for kube>=v1.3 only.
 kube_hostpath_dynamic_provisioner: "false"

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -86,45 +86,14 @@
     dest: "{{ audit_policy_file }}"
   when: kubernetes_audit|default(false)
 
-- name: gets the kubeadm version
-  command: "{{ bin_dir }}/kubeadm version -o short"
-  register: kubeadm_output
-
-- name: sets kubeadm api version to v1alpha1
-  set_fact:
-    kubeadmConfig_api_version: v1alpha1
-  when: kubeadm_output.stdout is version('v1.11.0', '<')
-
-- name: sets kubeadm api version to v1alpha2
-  set_fact:
-    kubeadmConfig_api_version: v1alpha2
-  when:
-    - kubeadm_output.stdout is version('v1.11.0', '>=')
-    - kubeadm_output.stdout is version('v1.12.0', '<')
-
-- name: sets kubeadm api version to v1alpha3
-  set_fact:
-    kubeadmConfig_api_version: v1alpha3
-  when:
-    - kubeadm_output.stdout is version('v1.12.0', '>=')
-    - kubeadm_output.stdout is version('v1.13.0', '<')
-
-- name: sets kubeadm api version to v1beta1
-  set_fact:
-    kubeadmConfig_api_version: v1beta1
-  when: kubeadm_output.stdout is version('v1.13.0', '>=')
-
 # Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint.
 - name: set kubeadm_config_api_fqdn define
   set_fact:
     kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}"
   when: loadbalancer_apiserver is defined
 
-- name: kubeadm | Create kubeadm config
+- name: kubeadm | set kubeadm version
-  template:
+  import_tasks: kubeadm-version.yml
-    src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2"
-    dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
-  register: kubeadm_config
 
 - name: kubeadm | Initialize first master
   command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all
@@ -135,24 +104,6 @@
   failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
   notify: Master | restart kubelet
 
-- name: kubeadm | Upgrade first master
-  command: >-
-    timeout -k 600s 600s
-    {{ bin_dir }}/kubeadm
-    upgrade apply -y {{ kube_version }}
-    --config={{ kube_config_dir }}/kubeadm-config.yaml
-    --ignore-preflight-errors=all
-    --allow-experimental-upgrades
-    --allow-release-candidate-upgrades
-    --etcd-upgrade=false
-    --force
-  register: kubeadm_upgrade
-  # Retry is because upload config sometimes fails
-  retries: 3
-  when: inventory_hostname == groups['kube-master']|first and (kubeadm_config.changed and kubeadm_already_run.stat.exists)
-  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
-  notify: Master | restart kubelet
-
 - name: slurp kubeadm certs
   slurp:
     src: "{{ item }}"
@@ -194,20 +145,9 @@
   failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
   notify: Master | restart kubelet
 
-- name: kubeadm | Upgrade other masters
+- name: kubeadm | upgrage kubernetes cluster
-  command: >-
+  import_tasks: kubeadm-upgrade.yml
-    timeout -k 600s 600s
+  when: upgrade_cluster_setup
-    {{ bin_dir }}/kubeadm
-    upgrade apply -y {{ kube_version }}
-    --config={{ kube_config_dir }}/kubeadm-config.yaml
-    --ignore-preflight-errors=all
-    --allow-experimental-upgrades
-    --allow-release-candidate-upgrades
-    --etcd-upgrade=false
-  register: kubeadm_upgrade
-  when: inventory_hostname != groups['kube-master']|first and (kubeadm_config.changed and kubeadm_already_run.stat.exists)
-  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
-  notify: Master | restart kubelet
 
 - name: kubeadm | Check service account key again
   stat:

roles/kubernetes/master/tasks/kubeadm-upgrade.yml

@@ -0,0 +1,33 @@
+---
+- name: kubeadm | Upgrade first master
+  command: >-
+    timeout -k 600s 600s
+    {{ bin_dir }}/kubeadm
+    upgrade apply -y {{ kube_version }}
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    --ignore-preflight-errors=all
+    --allow-experimental-upgrades
+    --allow-release-candidate-upgrades
+    --etcd-upgrade=false
+    --force
+  register: kubeadm_upgrade
+  # Retry is because upload config sometimes fails
+  retries: 3
+  when: inventory_hostname == groups['kube-master']|first
+  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
+  notify: Master | restart kubelet
+
+- name: kubeadm | Upgrade other masters
+  command: >-
+    timeout -k 600s 600s
+    {{ bin_dir }}/kubeadm
+    upgrade apply -y {{ kube_version }}
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    --ignore-preflight-errors=all
+    --allow-experimental-upgrades
+    --allow-release-candidate-upgrades
+    --etcd-upgrade=false
+  register: kubeadm_upgrade
+  when: inventory_hostname != groups['kube-master']|first
+  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
+  notify: Master | restart kubelet

roles/kubernetes/master/tasks/kubeadm-version.yml

@@ -0,0 +1,33 @@
+---
+- name: gets the kubeadm version
+  command: "{{ bin_dir }}/kubeadm version -o short"
+  register: kubeadm_output
+
+- name: sets kubeadm api version to v1alpha1
+  set_fact:
+    kubeadmConfig_api_version: v1alpha1
+  when: kubeadm_output.stdout is version('v1.11.0', '<')
+
+- name: sets kubeadm api version to v1alpha2
+  set_fact:
+    kubeadmConfig_api_version: v1alpha2
+  when:
+    - kubeadm_output.stdout is version('v1.11.0', '>=')
+    - kubeadm_output.stdout is version('v1.12.0', '<')
+
+- name: sets kubeadm api version to v1alpha3
+  set_fact:
+    kubeadmConfig_api_version: v1alpha3
+  when:
+    - kubeadm_output.stdout is version('v1.12.0', '>=')
+    - kubeadm_output.stdout is version('v1.13.0', '<')
+
+- name: sets kubeadm api version to v1beta1
+  set_fact:
+    kubeadmConfig_api_version: v1beta1
+  when: kubeadm_output.stdout is version('v1.13.0', '>=')
+
+- name: kubeadm | Create kubeadm config
+  template:
+    src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-config.yaml"

roles/kubernetes/master/tasks/pre-upgrade.yml

@@ -33,4 +33,4 @@
   register: remove_master_container
   retries: 4
   until: remove_master_container.rc == 0
-  delay: 5
+  delay: 5

upgrade-cluster.yml

@@ -72,7 +72,7 @@
     - { role: kubespray-defaults}
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: kubernetes/node, tags: node }
-    - { role: kubernetes/master, tags: master }
+    - { role: kubernetes/master, tags: master, upgrade_cluster_setup: true }
     - { role: kubernetes/client, tags: client }
     - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
     - { role: upgrade/post-upgrade, tags: post-upgrade }