Wait for kube-apiserver availability before starting upgrade (#6243)

* Wait for kube-apiserver availability before starting upgrade

I am experiencing a timing issue when upgrading from kubespray
2.11.0(k8s 1.15.3) to kubespray 2.12.6(k8s 1.16.9). The certificates get
replaced in `kubeadm-secondary-legacy.yml` and kube-apiserver notices a mismatch
(for a fraction of a second) between `apiserver.crt` and `apiserver.key`
which causes it to restart. And sometimes ( ~ 1 out of 5 upgrades) the
kube-apiserver isn't back on time for the start of the upgrade task. It
fails when kubeadm checks with the kube-apiserver to start the upgrade. The
kube-apiserver returns a `connect: connection refused`. I have created this
small task to check the availability of the kube-apiserver before starting the
upgrade, so that the upgrade will run without an issue.

Signed-off-by: Rick Haan <rickhaan94@gmail.com>

* Fix markdownlint

* Remove old CI

Co-authored-by: Maxime Guyot <maxime@root314.com>

.gitlab-ci/terraform.yml

@@ -92,70 +92,3 @@ tf-validate-aws:
 #     TF_VAR_facility: ams1
 #     TF_VAR_public_key_path: ""
 #     TF_VAR_operating_system: ubuntu_18_04
-
-.ovh_variables: &ovh_variables
-  OS_AUTH_URL: https://auth.cloud.ovh.net/v3
-  OS_PROJECT_ID: 8d3cd5d737d74227ace462dee0b903fe
-  OS_PROJECT_NAME: "9361447987648822"
-  OS_USER_DOMAIN_NAME: Default
-  OS_PROJECT_DOMAIN_ID: default
-  OS_USERNAME: 8XuhBMfkKVrk
-  OS_REGION_NAME: UK1
-  OS_INTERFACE: public
-  OS_IDENTITY_API_VERSION: "3"
-
-tf-ovh_ubuntu18-calico:
-  extends: .terraform_apply
-  when: on_success
-  variables:
-    <<: *ovh_variables
-    TF_VERSION: 0.12.12
-    PROVIDER: openstack
-    CLUSTER: $CI_COMMIT_REF_NAME
-    ANSIBLE_TIMEOUT: "60"
-    SSH_USER: ubuntu
-    TF_VAR_number_of_k8s_masters: "0"
-    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
-    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
-    TF_VAR_number_of_etcd: "0"
-    TF_VAR_number_of_k8s_nodes: "0"
-    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
-    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
-    TF_VAR_number_of_bastions: "0"
-    TF_VAR_number_of_k8s_masters_no_etcd: "0"
-    TF_VAR_use_neutron: "0"
-    TF_VAR_floatingip_pool: "Ext-Net"
-    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
-    TF_VAR_network_name: "Ext-Net"
-    TF_VAR_flavor_k8s_master: "defa64c3-bd46-43b4-858a-d93bbae0a229"    # s1-8
-    TF_VAR_flavor_k8s_node: "defa64c3-bd46-43b4-858a-d93bbae0a229"      # s1-8
-    TF_VAR_image: "Ubuntu 18.04"
-    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'
-
-tf-ovh_coreos-calico:
-  extends: .terraform_apply
-  when: on_success
-  variables:
-    <<: *ovh_variables
-    TF_VERSION: 0.12.12
-    PROVIDER: openstack
-    CLUSTER: $CI_COMMIT_REF_NAME
-    ANSIBLE_TIMEOUT: "60"
-    SSH_USER: core
-    TF_VAR_number_of_k8s_masters: "0"
-    TF_VAR_number_of_k8s_masters_no_floating_ip: "1"
-    TF_VAR_number_of_k8s_masters_no_floating_ip_no_etcd: "0"
-    TF_VAR_number_of_etcd: "0"
-    TF_VAR_number_of_k8s_nodes: "0"
-    TF_VAR_number_of_k8s_nodes_no_floating_ip: "1"
-    TF_VAR_number_of_gfs_nodes_no_floating_ip: "0"
-    TF_VAR_number_of_bastions: "0"
-    TF_VAR_number_of_k8s_masters_no_etcd: "0"
-    TF_VAR_use_neutron: "0"
-    TF_VAR_floatingip_pool: "Ext-Net"
-    TF_VAR_external_net: "6011fbc9-4cbf-46a4-8452-6890a340b60b"
-    TF_VAR_network_name: "Ext-Net"
-    TF_VAR_flavor_k8s_master: "4d4fd037-9493-4f2b-9afe-b542b5248eac"    # b2-7
-    TF_VAR_flavor_k8s_node: "4d4fd037-9493-4f2b-9afe-b542b5248eac"      # b2-7
-    TF_VAR_image: "CoreOS Stable"
-    TF_VAR_k8s_allowed_remote_ips: '["0.0.0.0/0"]'

docs/integration.md

@@ -7,7 +7,7 @@
 
 2. Add **forked repo** as submodule to desired folder in your existent ansible repo(for example 3d/kubespray):
   ```git submodule add https://github.com/YOUR_GITHUB/kubespray.git kubespray```
-  Git will create _.gitmodules_ file in your existent ansible repo:
+  Git will create `.gitmodules` file in your existent ansible repo:
 
    ```ini
    [submodule "3d/kubespray"]

roles/kubernetes/master/tasks/kubeadm-upgrade.yml

@@ -1,4 +1,15 @@
 ---
+- name: "kubeadm | Wait for master kube-apiserver"
+  uri:
+    url: "https://{{ kube_apiserver_access_address }}:{{ kube_apiserver_port }}/version"
+    status_code: 200
+    validate_certs: false
+  register: kube_api_server_available
+  until: kube_api_server_available.status == 200
+  retries: 180
+  delay: 1
+  when: inventory_hostname == groups['kube-master']
+
 - name: kubeadm | Upgrade first master
   command: >-
     timeout -k 600s 600s