Check kube-apiserver up on all masters before upgrade (#7193) (#7197)

Only checking the kubernetes api on the first master when upgrading is not enough.
Each master needs to be checked before it's upgrade.

Signed-off-by: Rick Haan <rickhaan94@gmail.com>

.gitlab-ci.yml

@@ -15,6 +15,7 @@ variables:
   MAGIC: "ci check this"
   TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID"
   CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml"
+  CI_TEST_REGISTRY_MIRROR: "./tests/common/_docker_hub_registry_mirror.yml"
   GS_ACCESS_KEY_ID: $GS_KEY
   GS_SECRET_ACCESS_KEY: $GS_SECRET
   CONTAINER_ENGINE: docker

.gitlab-ci/shellcheck.yml

@@ -12,5 +12,5 @@ shellcheck:
     - shellcheck --version
   script:
     # Run shellcheck for all *.sh except contrib/
-    - find . -name '*.sh' -not -path './contrib/*' | xargs shellcheck --severity error
+    - find . -name '*.sh' -not -path './contrib/*' -not -path './.git/*' | xargs shellcheck --severity error
   except: ['triggers', 'master']

README.md

@@ -116,7 +116,7 @@ Note: Upstart/SysV init based OS types are not supported.
 ## Supported Components
 
 - Core
-  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.18.8
+  - [kubernetes](https://github.com/kubernetes/kubernetes) v1.18.10
   - [etcd](https://github.com/coreos/etcd) v3.4.3
   - [docker](https://www.docker.com/) v19.03 (see note)
   - [containerd](https://containerd.io/) v1.2.13

contrib/inventory_builder/tests/test_inventory.py

@@ -51,7 +51,7 @@ class TestInventory(unittest.TestCase):
         groups = ['group1', 'group2']
         self.inv.ensure_required_groups(groups)
         for group in groups:
-            self.assertTrue(group in self.inv.yaml_config['all']['children'])
+            self.assertIn(group, self.inv.yaml_config['all']['children'])
 
     def test_get_host_id(self):
         hostnames = ['node99', 'no99de01', '01node01', 'node1.domain',
@@ -209,8 +209,8 @@ class TestInventory(unittest.TestCase):
             ('doesnotbelong2', {'whateveropts=ilike'})])
         self.inv.yaml_config['all']['hosts'] = existing_hosts
         self.inv.purge_invalid_hosts(proper_hostnames)
-        self.assertTrue(
-            bad_host not in self.inv.yaml_config['all']['hosts'].keys())
+        self.assertNotIn(
+            bad_host, self.inv.yaml_config['all']['hosts'].keys())
 
     def test_add_host_to_group(self):
         group = 'etcd'
@@ -227,8 +227,8 @@ class TestInventory(unittest.TestCase):
         host = 'node1'
 
         self.inv.set_kube_master([host])
-        self.assertTrue(
-            host in self.inv.yaml_config['all']['children'][group]['hosts'])
+        self.assertIn(
+            host, self.inv.yaml_config['all']['children'][group]['hosts'])
 
     def test_set_all(self):
         hosts = OrderedDict([
@@ -246,8 +246,8 @@ class TestInventory(unittest.TestCase):
 
         self.inv.set_k8s_cluster()
         for host in expected_hosts:
-            self.assertTrue(
-                host in
+            self.assertIn(
+                host,
                 self.inv.yaml_config['all']['children'][group]['children'])
 
     def test_set_kube_node(self):
@@ -255,16 +255,16 @@ class TestInventory(unittest.TestCase):
         host = 'node1'
 
         self.inv.set_kube_node([host])
-        self.assertTrue(
-            host in self.inv.yaml_config['all']['children'][group]['hosts'])
+        self.assertIn(
+            host, self.inv.yaml_config['all']['children'][group]['hosts'])
 
     def test_set_etcd(self):
         group = 'etcd'
         host = 'node1'
 
         self.inv.set_etcd([host])
-        self.assertTrue(
-            host in self.inv.yaml_config['all']['children'][group]['hosts'])
+        self.assertIn(
+            host, self.inv.yaml_config['all']['children'][group]['hosts'])
 
     def test_scale_scenario_one(self):
         num_nodes = 50

docs/azure.md

@@ -42,6 +42,10 @@ The type of the vm. Supported values are `standard` or `vmss`. If vm is type of
 
 The name of the virtual network your instances are in, can be retrieved via `az network vnet list`
 
+### azure\_vnet\_resource\_group
+
+The name of the resource group that contains the vnet.
+
 ### azure\_subnet\_name
 
 The name of the subnet your instances are in, can be retrieved via `az network vnet subnet list --resource-group RESOURCE_GROUP --vnet-name VNET_NAME`
@@ -50,6 +54,18 @@ The name of the subnet your instances are in, can be retrieved via `az network v
 
 The name of the network security group your instances are in, can be retrieved via `az network nsg list`
 
+### azure\_security\_group\_resource\_group
+
+The name of the resource group that contains the network security group.  Defaults to `azure_vnet_resource_group`
+
+### azure\_route\_table\_name
+
+The name of the route table used with your instances.
+
+### azure\_route\_table\_resource\_group
+
+The name of the resource group that contains the route table.  Defaults to `azure_vnet_resource_group`
+
 ### azure\_aad\_client\_id + azure\_aad\_client\_secret
 
 These will have to be generated first:

docs/centos8.md

@@ -2,7 +2,7 @@
 
 RHEL / CentOS 8 ships only with iptables-nft (ie without iptables-legacy)
 The only tested configuration for now is using Calico CNI
-You need to use K8S 1.17+ and to add `calico_iptables_backend: "NFT"` to your configuration
+You need to use K8S 1.17+ and to add `calico_iptables_backend: "NFT"` or `calico_iptables_backend: "Auto"` to your configuration
 
 If you have containers that are using iptables in the host network namespace (`hostNetwork=true`),
 you need to ensure they are using iptables-nft.

docs/containerd.md

@@ -0,0 +1,31 @@
+# conrainerd
+
+[containerd] An industry-standard container runtime with an emphasis on simplicity, robustness and portability
+Kubespray supports basic functionality for using containerd as the default container runtime in a cluster.
+
+_To use the containerd container runtime set the following variables:_
+
+## k8s-cluster.yml
+
+```yaml
+container_manager: containerd
+```
+
+## Containerd config
+
+Example: define registry mirror for docker hub
+
+```yaml
+containerd_config:
+  grpc:
+    max_recv_message_size: 16777216
+    max_send_message_size: 16777216
+  debug:
+    level: ""
+  registries:
+    "docker.io":
+      - "https://mirror.gcr.io"
+      - "https://registry-1.docker.io"
+```
+
+[containerd]: https://containerd.io/

docs/cri-o.md

@@ -4,10 +4,9 @@
 Kubespray supports basic functionality for using CRI-O as the default container runtime in a cluster.
 
 * Kubernetes supports CRI-O on v1.11.1 or later.
-* Helm and other tools may not function as normal due to dependency on Docker.
 * `scale.yml` and `upgrade-cluster.yml` are not supported on clusters using CRI-O.
 
-_To use CRI-O instead of Docker, set the following variables:_
+_To use the CRI-O container runtime set the following variables:_
 
 ## all.yml
 

inventory/sample/group_vars/all/azure.yml

@@ -10,9 +10,11 @@
 # azure_location:
 # azure_subnet_name:
 # azure_security_group_name:
+# azure_security_group_resource_group:
 # azure_vnet_name:
 # azure_vnet_resource_group:
 # azure_route_table_name:
+# azure_route_table_resource_group:
 # supported values are 'standard' or 'vmss'
 # azure_vmtype: standard
 

inventory/sample/group_vars/all/containerd.yml

@@ -1,6 +1,8 @@
 ---
 # Please see roles/container-engine/containerd/defaults/main.yml for more configuration options
 
+# Example: define registry mirror for docker hub
+
 # containerd_config:
 #   grpc:
 #     max_recv_message_size: 16777216
@@ -8,7 +10,9 @@
 #   debug:
 #     level: ""
 #   registries:
-#     "docker.io": "https://registry-1.docker.io"
+#     "docker.io":
+#       - "https://mirror.gcr.io"
+#       - "https://registry-1.docker.io"
 #   max_container_log_line_size: -1
 #   metrics:
 #     address: ""

inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml

@@ -20,7 +20,7 @@ kube_users_dir: "{{ kube_config_dir }}/users"
 kube_api_anonymous_auth: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.18.8
+kube_version: v1.18.10
 
 # kubernetes image repo define
 kube_image_repo: "k8s.gcr.io"

roles/bootstrap-os/tasks/bootstrap-debian.yml

@@ -86,6 +86,10 @@
   when:
     - need_bootstrap.rc != 0
 
+- name: Set the ansible_python_interpreter fact
+  set_fact:
+    ansible_python_interpreter: "/usr/bin/python3"
+
 # Workaround for https://github.com/ansible/ansible/issues/25543
 - name: Install dbus for the hostname module
   package:

roles/container-engine/containerd/templates/config.toml.j2

@@ -62,7 +62,7 @@ disabled_plugins = ["restart"]
 [plugins.cri.registry.mirrors]
 {% for registry, addr in containerd_config.registries.items() %}
 [plugins.cri.registry.mirrors."{{ registry }}"]
-  endpoint = ["{{ addr }}"]
+  endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"]
 {% endfor %}
 {% endif %}
 

roles/container-engine/containerd/templates/rh_containerd.repo.j2

@@ -5,5 +5,9 @@ enabled=1
 gpgcheck={{ '1' if docker_rh_repo_gpgkey else '0' }}
 keepcache={{ docker_rpm_keepcache | default('1') }}
 gpgkey={{ docker_rh_repo_gpgkey }}
-{% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %}
-{% if ansible_os_family == "RedHat" and ansible_distribution_major_version|int == 8 %}module_hotfixes=True{% endif %}
+{% if http_proxy is defined %}
+proxy={{ http_proxy }}
+{% endif %}
+{% if ansible_os_family == "RedHat" and ansible_distribution_major_version|int == 8 %}
+module_hotfixes=True
+{% endif %}

roles/container-engine/cri-o/defaults/main.yml

@@ -21,6 +21,7 @@ crio_stream_port: "10010"
 crio_required_version: "{{ kube_version | regex_replace('^v(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)$', '\\g<major>.\\g<minor>') }}"
 
 crio_kubernetes_version_matrix:
+  "1.19": "1.19"
   "1.18": "1.18"
   "1.17": "1.17"
   "1.16": "1.16"

roles/container-engine/cri-o/molecule/default/molecule.yml

@@ -26,7 +26,7 @@ platforms:
     groups:
       - kube-master
   - name: fedora
-    box: fedora/31-cloud-base
+    box: fedora/33-cloud-base
     cpus: 2
     memory: 1024
     groups:

roles/container-engine/cri-o/tasks/crio_repo.yml

@@ -5,7 +5,7 @@
     crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x','')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}"
   when: ansible_os_family == "Debian"
 
-- name: Add CRI-O kubic repo key
+- name: Add CRI-O kubic apt repo key
   apt_key:
     url: "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/{{ crio_kubic_debian_repo_name }}/Release.key"
     state: present
@@ -15,29 +15,37 @@
   retries: 4
   delay: "{{ retry_stagger | d(3) }}"
 
-- name: Add CRI-O kubic repo
+- name: Add CRI-O kubic apt repo
   apt_repository:
     repo: "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ crio_kubic_debian_repo_name }}/ /"
     state: present
-    filename: devel:kubic:libcontainers:stable
+    filename: devel-kubic-libcontainers-stable
   when: crio_kubic_debian_repo_name is defined
 
-- name: Add CRI-O kubic repo
+- name: Add CRI-O kubic cri-o apt repo
+  apt_repository:
+    repo: "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: present
+    filename: devel-kubic-libcontainers-stable-cri-o
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Add CRI-O kubic yum repo
   yum_repository:
     name: devel_kubic_libcontainers_stable
     description: Stable Releases of Upstream github.com/containers packages (CentOS_$releasever)
-    baseurl: http://widehat.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/
+    baseurl: http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/
     gpgcheck: yes
     gpgkey: http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/repodata/repomd.xml.key
+    keepcache: false
   when: ansible_distribution in ["CentOS"]
 
-- name: Add CRI-O kubic repo
+- name: Add CRI-O kubic yum repo
   yum_repository:
     name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
     description: "CRI-O {{ crio_version }} (CentOS_$releasever)"
-    baseurl: "http://widehat.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crio_version }}/CentOS_$releasever/"
+    baseurl: "http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crio_version }}/CentOS_$releasever/"
     gpgcheck: yes
-    gpgkey: "http://widehat.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crio_version }}/CentOS_$releasever/repodata/repomd.xml.key"
+    gpgkey: "http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ crio_version }}/CentOS_$releasever/repodata/repomd.xml.key"
   when: ansible_distribution in ["CentOS"]
 
 - name: Enable modular repos for CRI-O

roles/container-engine/cri-o/vars/centos-7.yml

@@ -1,5 +1,12 @@
 ---
+crio_versioned_pkg:
+  "1.19":
+    - "cri-o-1.19.*"
+  "1.18":
+    - "cri-o-1.18.*"
+  "1.17":
+    - "cri-o-1.17.*"
 
-crio_packages:
-  - cri-o
-  - oci-systemd-hook
+default_crio_packages: "{{ crio_versioned_pkg[crio_version] }}"
+
+crio_packages: "{{ centos_crio_packages | default(default_crio_packages) }}"

roles/container-engine/cri-o/vars/centos-8.yml

@@ -1,4 +1,12 @@
 ---
+crio_versioned_pkg:
+  "1.19":
+    - "cri-o-1.19.*"
+  "1.18":
+    - "cri-o-1.18.*"
+  "1.17":
+    - "cri-o-1.17.*"
 
-crio_packages:
-  - cri-o
+default_crio_packages: "{{ crio_versioned_pkg[crio_version] }}"
+
+crio_packages: "{{ centos_crio_packages | default(default_crio_packages) }}"

roles/container-engine/cri-o/vars/debian.yml

@@ -1,7 +1,17 @@
 ---
+# Debian-10 has pkg only for cri-o 1.19
+crio_kubernetes_version_matrix:
+  "1.19": "1.19"
+  "1.18": "1.19"
+  "1.17": "1.19"
 
-crio_packages:
-  - "cri-o-{{ crio_version }}"
-  - runc
+crio_versioned_pkg:
+  "1.19":
+    - "cri-o=1.19*"
+    - cri-o-runc
+
+default_crio_packages: "{{ crio_versioned_pkg[crio_version] }}"
+
+crio_packages: "{{ debian_crio_packages | default(default_crio_packages) }}"
 
 crio_runc_path: /usr/sbin/runc

roles/container-engine/cri-o/vars/fedora.yml

@@ -4,3 +4,10 @@ crio_packages:
   - cri-tools
 
 crio_conmon: /usr/libexec/crio/conmon
+
+# TODO: remove crio_kubernetes_version_matrix and crio_version once Fedora supports 1.19
+crio_kubernetes_version_matrix:
+  "1.18": "1.18"
+  "1.17": "1.17"
+
+crio_version: "{{ crio_kubernetes_version_matrix[crio_required_version] | default('1.17') }}"

roles/container-engine/cri-o/vars/ubuntu.yml

@@ -1,7 +1,18 @@
 ---
+crio_versioned_pkg:
+  "1.19":
+    - "cri-o=1.19*"
+    - cri-o-runc
+  "1.18":
+    - "cri-o=1.18*"
+    - cri-o-runc
+  "1.17":
+    - "cri-o=1.17*"
+    - cri-o-runc
 
-crio_packages:
-  - "cri-o-{{ crio_version }}"
+default_crio_packages: "{{ crio_versioned_pkg[crio_version] }}"
+
+crio_packages: "{{ ubuntu_crio_packages | default(default_crio_packages) }}"
 
 crio_runc_path: /usr/sbin/runc
 

roles/container-engine/docker/templates/rh_docker.repo.j2

@@ -5,6 +5,9 @@ enabled=1
 gpgcheck={{ '1' if docker_rh_repo_gpgkey else '0' }}
 keepcache={{ docker_rpm_keepcache | default('1') }}
 gpgkey={{ docker_rh_repo_gpgkey }}
-{% if http_proxy is defined %}proxy={{ http_proxy }}{% endif %}
-
-{% if ansible_os_family == "RedHat" and ansible_distribution_major_version|int == 8 %}module_hotfixes=True{% endif %}
+{% if http_proxy is defined %}
+proxy={{ http_proxy }}
+{% endif %}
+{% if ansible_os_family == "RedHat" and ansible_distribution_major_version|int == 8 %}
+module_hotfixes=True
+{% endif %}

roles/download/defaults/main.yml

@@ -49,7 +49,7 @@ download_delegate: "{% if download_localhost %}localhost{% else %}{{ groups['kub
 image_arch: "{{host_architecture | default('amd64')}}"
 
 # Versions
-kube_version: v1.18.8
+kube_version: v1.18.10
 kubeadm_version: "{{ kube_version }}"
 etcd_version: v3.4.3
 
@@ -120,7 +120,21 @@ crictl_checksums:
 # Checksums
 kubelet_checksums:
   arm:
+    v1.19.7: 59284dcf4ee3f62475d0d6d1070c64e0e5362280e8d70884513e1e1cee918cb0
+    v1.19.6: 59179a52615cb0811909558425e046cc9c5c051e3b55733b96e40a9aed0967de
+    v1.19.5: 112e247538b889057dd55a35350415494785c6e0f5ad54a273fcab51f4c30805
+    v1.19.4: 51ae27723be81ea63906ec375d652bfe97da312cb35f532d5d483e88aad2fc01
+    v1.19.3: 3c0214d2d583440233b6bd0088614fe4fb5748e5b153f2ec96d72d3f8165e214
+    v1.19.2: 631e686c34911a40a798817dcff89532c88bb649885f93ec66b339e227ebd974
+    v1.19.1: 3985c8d02c1c2f2016fceccd9cc14865e2d047f32c8f0b42aeedcc8450de572e
     v1.19.0: bb433ef7981297bdee6ffc4e23376e8db24a0e47321ebe94bf9d4b9f7a2f0e3b
+    v1.18.15: 27c8d5ea1f837cb0148d829944d46b028ff9d9b9edf12cd39e1cb5f65fd9f41c
+    v1.18.14: e380c14330c4f34b7682a19a493769a4ba7cf9132a5ae9d2ab4109a0ba60973b
+    v1.18.13: ed7e1adef473e4c49cbd4b8f2363239ff08b72f9a2c153f29c4021ef04ca9526
+    v1.18.12: 604db89eca279867aab5b6d0e4619f5577fa85f7817db0620ab5f06806358c2f
+    v1.18.11: 1e5314c30aa480ddb381e003b0e0a8823eccf6b7f49c62718c34afb8b2c833c3
+    v1.18.10: 716b8a1971d2f96b28f31a65e6769e2ce60979703d143a67d340c7ce16137db5
+    v1.18.9: 24f7559fe5214b5f3b625ab035e29159441e6cfd248befbeb78b63e660fccd23
     v1.18.8: 831f50ea45384884c50395c288d493e75dd134a962dc95261ce122de5e6a17ec
     v1.18.6: 5f8367f9f5de77c022ec5d0cd86e897b7a33318185eaadb7736033d8dabcdbdb
     v1.18.5: 9f8ab727964c6f42f1c17089bf2f7b4b2f2a5c61ffab3bad16eb02d9feb05855
@@ -129,6 +143,8 @@ kubelet_checksums:
     v1.18.2: b7b9c43851dde9cbaa2061828410c60ee63e53fbf3ebc5559b7f4387dae67bb9
     v1.18.1: 04d8e0a080dcb23d579c69e769e75bd5abaa1977d43550ec891560d76f1f7f37
     v1.18.0: 985c1a1b492ccc6e46e1cd454790dae539d5b93208efb05e35114f66a183de99
+    v1.17.13: f98d4eaf65c65bce55479b1435baa032944856c098ccf8038a8aca318c530f24
+    v1.17.12: 2b6160f5c15b1294573e37031ce0bcbbcd3ee3ea055f4f8fe4faf3ab74774f40
     v1.17.11: 53784f9bea1508b6d82f8be6f40af2e6aef86381021b500c1647f9d297af3efd
     v1.17.9: 37fc89360f4dcff4788032e60fe6388da4194a68503992868261ba840413e8d4
     v1.17.8: 82320569bc9deff33d148c759a105f1a32de3d83855165100261a4ad395d1845
@@ -141,7 +157,21 @@ kubelet_checksums:
     v1.17.1: 0219c940bad3238dfbdf8e4518241d861bbdd8fc93d172cc632c225d7dd57094
     v1.17.0: 75ae6ad8f4a7f2ac3988b37a01c28093f240745d17c1781135d1844057c8ae94
   arm64:
+    v1.19.7: 473016cf1165ab5c705bd29c65451984e48deade6ff96bf861c46e15fc52ba3f
+    v1.19.6: e168d57a0a04abe58db13aaf4f54ba691115d0b1a743d549136794bbecde7dc8
+    v1.19.5: 646373c5337f6f348573bbbef3d92a675fe18d3c54b752805c6d3a9f9fc22957
+    v1.19.4: 8f534dfc4a836d7f434f09c8f726d747204a312eb47f3b874705451c5b04a243
+    v1.19.3: 228695df98c5cb8a5f64d1235021f54a482a8e63f5b9b1b2addfc95df9b671ee
+    v1.19.2: 86b9336aa7f6215e6f9b387bb82105657668aa8a38b0e0f7c2e647ef45c1b723
+    v1.19.1: 143bed1f04cba4e6749a72abf9258d19f31e1a310f94bd041cd30ce62a1f73ff
     v1.19.0: d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf
+    v1.18.15: c74f46e87aae7b9bb590319632fcb106b5efd998046dd47634c89bbb875d9fb6
+    v1.18.14: b4ca12f6b1a5f7c838831eb3c9152ccf0f53b83a3492c812da6ac680959a4801
+    v1.18.13: 953b7f9c70d04be0cf69745ab30f676375d93272362c97bb3cd527d6e27b38e4
+    v1.18.12: 2593c564811f38202fdde351c8ee8d3cfac2dd7c52e42b393157ac6fa1569cbd
+    v1.18.11: d595561315eb7fd270363464a2e36c702684086410000212333478761b8c13c1
+    v1.18.10: 1490550560b9afcb6e74d5bd69d61ae60dabe466e05e0408da48f17b4ccd91b4
+    v1.18.9: 21b0fb4682deea19be3ac160403db9858dc9d02b101d60eb6fc22a86523ec434
     v1.18.8: d36e2d656bad232e8b48b19c948164ee3966669f4566cf5ea43ca22f6eed1aa5
     v1.18.6: 257fd42be375025fb93724bda9bef23b73eb40531f22bab9e19f6d6ff1ca57cf
     v1.18.5: c3815bc740755aa9fd3ec240ad808a13628a4deb6ec2b4338e772fd0cf77e1a2
@@ -150,6 +180,8 @@ kubelet_checksums:
     v1.18.2: 89b5066ae17df8488c76a83c70cbcac0771fa36803e31b826f2770b5efcdbfbf
     v1.18.1: 2181cde9e6b24055d262b78758b365363273896968df673eb13d4f17a4f69c4a
     v1.18.0: db91a26f8baa2bce017172305e717e77be5cfc4272592be8cb0155e1cfa7719e
+    v1.17.13: e9396034d079d3574370faf47eba78055cf8fb897093929e796f571cf2f91cd6
+    v1.17.12: bfd5ff97040a2f1017bc14991757ba62ad8c8218d75a5f646690c1e76f93bf22
     v1.17.11: 0d9705c284054b2fdb7627d4867edd0863f67a2fbf64a2e1710d928936539c89
     v1.17.9: d57c25a3d67c937a9d6778de07295478185f73938937868525030a01d15c372f
     v1.17.8: 673355f62aa422915682ae595e4e53813e4656f2c272eb032f97492211cfced5
@@ -162,7 +194,21 @@ kubelet_checksums:
     v1.17.1: c773512ade5da3188ed4c312d5ba01bfbf3f376f6e580e5b074827a5b25450aa
     v1.17.0: b1a4a2325383854a69ec768e7dc00f69378d3ccbc554859d910bf5b582264ea2
   amd64:
+    v1.19.7: d8b296825f6dd7a17287b73cd6604d32210abbba86c88fb68c1b1c5016906c54
+    v1.19.6: 8162fa58f7fbb9b1f07b2b666c9759ad5c30950bc9f6f2084de1c0d9bc64a4c0
+    v1.19.5: b64dbc4a1a47d21be0a67f7d0050eebc02dd610c31175865da08e3a59e0834af
+    v1.19.4: 078cd88f521092fb1b9e9596f64ab72de5af3d4935e22d8421ad5e45c9729b1d
+    v1.19.3: daa02a34efd936bf9940d9c52fe24c299fc36ba4b31a051968efb3652f289fa9
+    v1.19.2: 7ff6d3663e8de0c654909e7a279e386286aa7ed3fc262d021bed77c92d62780f
+    v1.19.1: 2ca2a3104d4cce26db128e3a0b7a042385df4f2c51bdbe740e067fdfaa2fcdd1
     v1.19.0: 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab
+    v1.18.15: 2d079bc9f7ac2c2d1f86df842df55f2ec8e74e01edc347994ccf1a054b9e6077
+    v1.18.14: 1876c3aad83beeea1bc949fe6121d0d0d9002a0e026c15ccb568d8c5e748fba6
+    v1.18.13: dbdecaec19da684055242e2684ec8a2bff1674adf9ae8b8ed87f9cb46101a87f
+    v1.18.12: 928900515d7f4bb35b80bb39ef934d133c2917aa9d65ec33f00304079f474305
+    v1.18.11: 3e5c5a784b8be476248e31819204e02779d5444cbe3c77789562e34a86c11c0b
+    v1.18.10: 8daecd339993342c0693b6cb8a8e053d4a21d2d829389cc7ab823f52ea0589a1
+    v1.18.9: 5028b6548e8838e1e0851f10e8bd8d9a6ef1693e3f1dac09f7d50c4c2873f20b
     v1.18.8: a4116675ac52bf80e224fba8ff6db6f2d7aed192bf6fffd5f8e4d5efb4368f31
     v1.18.6: 2eb9baf5a65a7b94c653dbd7af03a768a520961eb27ef369e43ef12711e22d4a
     v1.18.5: 8c328f65d30f0edd0fd4f529b09d6fc588cfb7b524d5c9f181e36de6e494e19c
@@ -171,6 +217,8 @@ kubelet_checksums:
     v1.18.2: bc13d29b58300c328f0078c7f72e37e1254c4303277348862af1e7f2b356b9e3
     v1.18.1: 4c5737235e62a5bb0b5d3f51939ccd255ebda376d75941222b25241251b67fbc
     v1.18.0: 3a90e7abf9910aebf9ef5845918c665afd4136a8832604ccfabca2defb35ce0f
+    v1.17.13: e71c3ce50f93abc2735ba601781355d86a49aec992e8cb235a369254c304fa7d
+    v1.17.12: 196263deb2605507ff137b3edea39a914e7047f4c4de0a681c4a6f8dcfa5fe65
     v1.17.11: 71bcc8443a6e6f226727ea007fdc4b96327e302d1724a15ce2bffc2d94a3dac6
     v1.17.9: 3b6cdfcd38a646c7b553821ef9bb67e93541da658305c00705e6ab2ba15e73af
     v1.17.8: b39081fb40332ae12d262b04dc81630e5c6550fb196f09b60f3d726283dff17f
@@ -184,7 +232,21 @@ kubelet_checksums:
     v1.17.0: c2af77f501c3164e80171903028d35c632366f53dec0c8419828d4e55d86146f
 kubectl_checksums:
   arm:
+    v1.19.7: 11d49f87bf8b557066c339eea4775b32dd463fc61ce24b24744f82cb412e9277
+    v1.19.6: fde28a2bff5892e461d5c72c5391da5eef20450a5a074bbbfce4f476687dac95
+    v1.19.5: 10409729115f66e32263dfa8a320b74ef1e5845a9103470d3d18095ca4c1dc80
+    v1.19.4: 4ca2dc66fb5ea5f71afbb1f769e7eae237da4c9287c875da935a6b9f15619112
+    v1.19.3: fb611ff64139bc8712fe93497f2419c236d62c5f689e1cb4cc68037fda698f82
+    v1.19.2: c8cad74a586dfee41436ce866c91d79c9d2053468eccde9fed976cdf32b7849f
+    v1.19.1: e63bbf3161c49d60e46ffaf7d3cfd689834516205be1be881d2e652115535e93
     v1.19.0: 5885bb723a55ab95e8687e0ad52620ce4c08f76fd20c0973f5cd19c753b513c8
+    v1.18.15: 952530dd6b272eed932e49a29625e20303b9621ba36c1cc0394476d483f6860a
+    v1.18.14: b2b88ee6bea8ee17dd1c7687add53c9db5139abb7013ded77050d57b62070aa7
+    v1.18.13: f3e9a4786a774441ee448edb6f842f9c6825f12245b5e4ee5ffe8b2ab1c85058
+    v1.18.12: 4380ddb37d49e1199ebaa1f2785bd81ae22ffac15ad470d48f7308e9bf131b26
+    v1.18.11: b357d042224f13be5cb7f11bd25bcd735ef2fee8bcdd43653598d616ac8bb1f0
+    v1.18.10: 3d5b7cb1c54d5e9dec157a512d2d21dddc6b9fd5b9a0b8df9493553871d21668
+    v1.18.9: 8f49ade7875aaca82f7471901963796815b786d5437e6af0ae4d6d784dc92c08
     v1.18.8: 21769e01e17f3809d0e9188a88a71fb1f479dfeeb22590e56006d5dbb3689605
     v1.18.6: 1b6668bae432a3c6034f2fe83e72b2db5ea794a02121cf0a22be7f4bd8f460bf
     v1.18.5: 5fc8dc6e3d09ceaf900dd06b9af3a7abb291293cea5219aea577bad852aa84b1
@@ -193,6 +255,8 @@ kubectl_checksums:
     v1.18.2: 353d61297cfbf01ed3f72b7df1658110c065355d670556ea3bdbf0d1b2824aea
     v1.18.1: 896c90b1b9d88e121876d93718591f3ecbab880b304767806c6c9fcb3b145805
     v1.18.0: 34fc6d16c2f535ed381b5fd3a4d40b642fff4f9ff95f8250b8043a29b8c062b9
+    v1.17.13: 1053624c88881d1fe9d8f2adbb07831fc23c829127b8466da9b15cc122004344
+    v1.17.12: e3bfbb4a82183c094a6538ffe4e837856cb3849ed1348d23830f0945852a81e4
     v1.17.11: e8178745c3010a57de068138e63bf19087ca4f83acfdc1df82f7844ce73cf3be
     v1.17.9: 4bcad42cb8721bcb636f88e02f143fa9e2ad8141a37025f7622bd04516dab391
     v1.17.8: e1a75ee55e1270583143422cc611547623aeef2c69689354c69b0b8f445cf6ba
@@ -205,7 +269,21 @@ kubectl_checksums:
     v1.17.1: a1e580e9140536c4a370c207ee66481cfe8d8876dc9021755a9d20232a97033d
     v1.17.0: 594b3e2f89dca09d82b176b51bf6c8c0fa524ed209c14ec915c9b36fa876601d
   arm64:
+    v1.19.7: a0f58222e8d62b86b1148746531fa0c3b91fa208586cb068883550fc3dcd498b
+    v1.19.6: 828ddf7183c647306a2432ff098a5b22fd6c68c069cb7047ffb1e2b2a967ef83
+    v1.19.5: eaf26ca30f1696e72754d86aeda1886287127295809a400821f8d0ffa6329359
+    v1.19.4: 334ca7ce181d8b83e100b1ad4347af93bee6f261d792e9f52d9769095e970919
+    v1.19.3: a4f2e2dbdcead30eed5aa47468e669b9574fd99457b860679eba84e1cb9cf863
+    v1.19.2: a460f918c03e5cd916f4ac28da137596232e344cc0755d4ceb222fc4cd203e09
+    v1.19.1: 332bbdb4560f9b7fcbb96c8f2cebbc4996e409384ca07510e5c5990998158c20
     v1.19.0: d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f
+    v1.18.15: 6b4a63df325cdb523f16ffd8799745a8fdf979ef89e228c309a715671b6aa984
+    v1.18.14: ac4014f7d9001375cb600a15d77e90eb6a20053afac82f167f4b7732aa073388
+    v1.18.13: 8e5271e95442e373df1c67473484e387086f344a2e1445ee9f5a878ca7f4442c
+    v1.18.12: 3cae828a3dca46b12284839e4273a1cfcf35177937ca5aa5aa49193d6eb0649b
+    v1.18.11: fefd809b8749f2884ec1a99acf987286b487eac90c98145aef8ac866d0ceae99
+    v1.18.10: 394fa475f7688778eeeecb31e01acfae4cc37a72926d9bf33290c968e6dc037a
+    v1.18.9: 9f466ff8d40097914a1ded0288ef8b9eb6f4ad39a9990cb2c0f8d1a710013a4f
     v1.18.8: 9046c4086528427462544e1a6dcbe709de4d7ae44d1a155375de330fecd067b1
     v1.18.6: 7b3d6cc019747a7ee5f6cc2b187423daaac4e153140cb290e60d316c3f456430
     v1.18.5: 28c1edb2d76f80e70e10fa8cd2a30b9fccc5f003d8b3e853535d8317db7f424a
@@ -214,6 +292,8 @@ kubectl_checksums:
     v1.18.2: 8d4bd6a716e32187e03c5998b4d9570f3b2eb9fb041ac9ed6e9728f04935c2fb
     v1.18.1: 39e9645c6bed1e7340b3d764db983b9fc35326b11325fd509c3660c9f55469ed
     v1.18.0: 0de307f90502cd58e5785cdcbebeb552df81fa2399190f8a662afea9e30bc74d
+    v1.17.13: 9d62bb6f21a64fd464237b7c81e45075e2ce0a83b6e13c54a6539c076f3b536f
+    v1.17.12: 22f383cef1a429cab5d4c98df6704295722044613dbea49c306e931d383d723f
     v1.17.11: 8eb18f37148d2786205bc70dded66c173df7517577e5ae024a19e72400263ed2
     v1.17.9: 4d818e97073113eb1e62bf97d63876757be0f273c47807c09f34511155e25afd
     v1.17.8: 4dfd36dbd637b8dca9a7c4e789fb3fe4ca420062c90d3a872ae751dfb9777cb6
@@ -226,7 +306,21 @@ kubectl_checksums:
     v1.17.1: 4208be10e2c12b67e71219cd39b0b2ab065d4ec1b26e19c5da88cb8ebc64ea2f
     v1.17.0: cba12bfe0ee447b06f00813d7d4ba3fbdbf5116eccc4d3291987044f2d6f93c2
   amd64:
+    v1.19.7: d46eb3bbe2575e5b6bedbc6d3519424b4f2f57929d7da1ef7e11c09068f37297
+    v1.19.6: d8a46600bcdcd01f39c11e609e8277975f04c0593f79b2a7b5c67646e1c792d8
+    v1.19.5: 5f5a0bebde8a8782b2d9de9bd314dce722f9f960bee090b121ea5ac38cf4377d
+    v1.19.4: 7df333f1fc1207d600139fe8196688303d05fbbc6836577808cda8fe1e3ea63f
+    v1.19.3: 84eeb8237448e4f431fef0f0ec0ba8b07558d8e52d5a7e89b4ae64dadcffbe66
+    v1.19.2: f51adfe7968ee173dbfb3dabfc10dc774983cbf8a3a7c1c75a1423b91fda6821
+    v1.19.1: da4de99d4e713ba0c0a5ef6efe1806fb09c41937968ad9da5c5f74b79b3b38f5
     v1.19.0: 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f
+    v1.18.15: eb5a5dd0a72795942ab81d1e4331625e80a90002c8bb39b2cb15aa707a3812c6
+    v1.18.14: 8c924c1fdf743c2a3bf0edbd4333f54c1bce64871abc1a729243321d99b567d4
+    v1.18.13: 8914a4529aaa5f358c663c03bc2cb741e5667f8142e37435689a851647b7697f
+    v1.18.12: ea2f6d0542b7bd259ccf7dd139fdad20767f7c5bf94c19c50df227396a08d76d
+    v1.18.11: 1bdab3fd6aecf1ab40fa9076e8ac487cd028a16f911da2d25d121033f6f61de0
+    v1.18.10: b25c445cb36ab168de590c13b50bced4e7877ed15ca899a261053b4ba3ba1456
+    v1.18.9: 6a68756a2d3d04b4d0f52b00de6493ba2c1fcb28b32f3e4a0e99b3d9f6c4e8ed
     v1.18.8: a076f5eff0710de94d1eb77bee458ea43b8f4d9572bbb3a3aec1edf0dde0a3e7
     v1.18.6: 62fcb9922164725c7cba5747562f2ad2f4d834ad0a458c1e4c794cc203dcdfb3
     v1.18.5: 69d9b044ffaf544a4d1d4b40272f05d56aaf75d7e3c526d5418d1d3c78249e45
@@ -235,6 +329,8 @@ kubectl_checksums:
     v1.18.2: 6ea8261b503c6c63d616878837dc70b758d4a3aeb9996ade8e83b51aedac9698
     v1.18.1: f5144823e6d8a0b78611a8d12e7a25202126d079c3a232b18f37e61e872ff563
     v1.18.0: bb16739fcad964c197752200ff89d89aad7b118cb1de5725dc53fe924c40e3f7
+    v1.17.13: 25824bf20d8d9501e9b6cabdd6bd25ec10cc649d4e3155dba86037f57bba842e
+    v1.17.12: d6472a5f5d3ff125b4a1aa5cefd6197faadfb578c36f639240a04df1a71597e1
     v1.17.11: 002d640a12d6fad48f7d8a5c56cb4ff656397caf3511f50426d9c69c94b2d137
     v1.17.9: 2ca83eecd221bedf3eceb0ccfcf45bb2e27950c382c2326211303adb0a9c4232
     v1.17.8: 01283cbc2b09555cbf2a71c162097552a62a4fd48a0a4c06e34e9b853b815486
@@ -248,7 +344,21 @@ kubectl_checksums:
     v1.17.0: 6e0aaaffe5507a44ec6b1b8a0fb585285813b78cc045f8804e70a6aac9d1cb4c
 kubeadm_checksums:
   arm:
+    v1.19.7: 48722b7c93d18a13e734200de202912a324769139d0d434ff1dfff82af164814
+    v1.19.6: e3ecde2f2eccb177c0e4b8c6bd19ae471bc19977a8f812cb17094743d42b5b6e
+    v1.19.5: b034594ebe9096f27b3bc5a0b1a98baed9d54ba1457d3d13208f0d490db39000
+    v1.19.4: 9b83c9dcda775e9b5a20d7254218e8b70cea8a37266d254b6289faa1d5602cd9
+    v1.19.3: 522358c8596d10cac1a04a9e52f0ae59a1c06ca122292429d36773e7f6ad0a01
+    v1.19.2: effc35d1e3ab01ac80185ff9f7ca5afabbb94f5f91d7326b04b09e903315647d
+    v1.19.1: 0e910cf9c771976f6eb079098ad428f3e99080624f478e3d71b670005a7c3651
     v1.19.0: 62fca8b373f8c63409bcfcb7c585f8de882a8a119d88d39666e1ab3a11be188b
+    v1.18.15: b242890123a5ecc574157a959475c903eeb14ed17b4578902a17b8d994d4c401
+    v1.18.14: 7e5d4beedcaf13c0076f03c9232464946faa061effd5db8c7705d317a4ee6e95
+    v1.18.13: 3785825c1b9a1fbb90abc00077d9ccd43610b147302c29501b9ce12959fb13bf
+    v1.18.12: 020d1adba1c339fa390fac18cf839f70d52d981b33b62b7b2f1ca4166c263bd0
+    v1.18.11: 0647721b5a4a4b6932f2ce43f8a35f3cf0d875b598a73eee209b8955beaede40
+    v1.18.10: 49f53573bdefd4ed37376800119b082666d03d5657d5886a4caa35e63a11d658
+    v1.18.9: 026cd1ec3b75703994254ae44998a544f46723b424775218f90c07754bb42bb6
     v1.18.8: 52ec1a5d8a6826762c112d55734e35cf895a02e746b8d6ca4a9c942289aab077
     v1.18.6: 84abadc0abc01970ee73bf25078b2120a0725e4afcb9bb8c0d03077c978d7452
     v1.18.5: 461641c8fb8db2afe6e103aca925a4ef9d161dcae08a96fc24674b0ea0122e04
@@ -257,6 +367,8 @@ kubeadm_checksums:
     v1.18.2: c3558beca26c1b970cee8419dcf24f9812483f6ef384cea9a704491bc3af1e2c
     v1.18.1: 4f919ad7215209dee97ea4c61668e44a2cce8f575b9cf4032e47f0c377924854
     v1.18.0: 0f05bd526bb38be11459675c69bc882a2d3e583e48339fab49b620d292c2433e
+    v1.17.13: 086de433c9e77d37a6fd38fca059b7ab4cae9d0a3e57350d245f753cb391cec0
+    v1.17.12: c18a1c4aa3788c8e860a8653987bd50df9f2da70e15e5a4e1cce6f9a4bee4831
     v1.17.11: df79e696668290091cf93b4c68ff614b3886cecbe40dbf76fdfff799ba41901d
     v1.17.9: 69c28a8e35394cf72b3926bb557724b7d0c5bbd07a22fdd9c207b943b4c617b3
     v1.17.8: 1e8e653a07438131126f62b853b442356b341d2950f0d7c30d2a96e773a54611
@@ -269,7 +381,21 @@ kubeadm_checksums:
     v1.17.1: 501d1bacb863713dd9d0101d0021b0227869c4b1b9e903f6498333c613d384e1
     v1.17.0: 5fcf1234d89bc2a364c53b76b36134fc57278b456138d93c278805f2c9b186f1
   arm64:
+    v1.19.7: 43615e65092de34dcaf34725d8adc0af91b660fa54a885421fdb29f743478911
+    v1.19.6: 082ceac5f542cb49a62cf47212bf1ea9dbb15d1831126f526d15e429b4f0687d
+    v1.19.5: 8c4363e910f0070d597f3e5b9a7b8741e1ef20778a8dddc15ac47bf02389d964
+    v1.19.4: 1357ce6314a2b4a2303a96c33c996e7e9e8269cf4cc464557e7fa09f9a6ade34
+    v1.19.3: c398c23019f988514ac0f2c1e32a388cf11ca9d48634530092dbf54d9e00eaa6
+    v1.19.2: b6900a44558ba1a0a364406e2072163f5fc561030da97045e1403cdc69ff2682
+    v1.19.1: dcdabd2fdec9b4dd8febd1625e1fbbe15362919041e5f4ce3aab629e4aea7540
     v1.19.0: db1c432646e6e6484989b6f7191f3610996ac593409f12574290bfc008ea11f5
+    v1.18.15: d6d4d9b8e4992c218ff6185da7928289c9938796d5c08a7427625a563c74a075
+    v1.18.14: e4c1aaed946dd38ca6d9fdef0ef333b4d488a25810aa6f8a98de1b9dd6d47db3
+    v1.18.13: 54d035ea9356b2f7f146d7e287aba128d3a0c594e0f0112f6ccba1d0294770c9
+    v1.18.12: d4cbd57c504000e7eeb68dfcb3b5e08625baa6a60b970b053a7234907b173272
+    v1.18.11: c36aec51851c9b3cbce61fbea26a70af2923b339c40b92973f4a0e8b1c584190
+    v1.18.10: dc4a2daa3bf3e652fc7a81f5b449752c08e6a91e27aa1bbffad7ade35508a77b
+    v1.18.9: c17e29b8cec1892b6cd72aed1af6d9abfd39816c222d3cc5c97c6637a284162d
     v1.18.8: 71f6d95f165a9e8066c6f299217af779829ab3d798f6130caf6daa4784dc0464
     v1.18.6: df5a3d7c70c3f8221d57093c5cb17558aad6e65725d7a096c6620302fbf64730
     v1.18.5: 0e2a9de622177015c2514498382b0d821ac8f71c7ed5f02e5684d456ff3c0e4d
@@ -278,6 +404,8 @@ kubeadm_checksums:
     v1.18.2: e5a1f738443c15f5f8f3b316c6c7f8038f84f24b5d4bf2eef5bee39ca208952a
     v1.18.1: 0cb6589d9b4c09b007eae977ab8a185fc4140eda886408dced4f500a508e4e83
     v1.18.0: 2ef1785159c80a9acd454a1c8be3c6b8db2260200b22e4359426e709ff786d01
+    v1.17.13: eb84d8cd772a02c6db35ffd77f2ee2703db5b92571f9b7c8c30108ee7bb35b0a
+    v1.17.12: fb2f090c4bd62229a738b8e1faa81bd3b01252a49351464a1e1be9fe689ab959
     v1.17.11: 0875b49d7eacb80db3b9a375086a091304420c875ae8b54191ed9aa0af4c54e3
     v1.17.9: b56dc03177636fdafb4f8ab329d087b804cb7395c142f76e8246e86083c6d750
     v1.17.8: 5a52e7d0306890e68ed66fc47ecd70bf14628c70527442fd0cd2973dbde7064c
@@ -290,7 +418,21 @@ kubeadm_checksums:
     v1.17.1: c640eb50406962628ac6e31fd840506a360b5d9c57d14007d0eaada28c49d64f
     v1.17.0: 0b94d1ace240a8f9995358ca2b66ac92072e3f3cd0543275b315dcd317798546
   amd64:
+    v1.19.7: c63ef1842533cd7888c7452cab9f320dcf45fc1c173e9d40abb712d45992db24
+    v1.19.6: 6204d9f16554480fe64a09e9efef31559f1da623fb34492a9a18b085afac876a
+    v1.19.5: 00a2eb2b919661b46cd845c5645e3258806f331f1ea8015e136e733c8c5dd515
+    v1.19.4: e1977c6ba7acc7b2b3896202c93a73dbdf1c649b93843350e0f10172e26cf59d
+    v1.19.3: 0a7581fdebe05fb101ce30d4e1f85e865e18f5c034e4f7cc785c786e861f9be2
+    v1.19.2: 377dbf06469709aafb7a9a6c925e890bf48727b905455c2a81fee28727716f2f
+    v1.19.1: d5afcf4ff916b4c6810f10f76176c73238f0854b7748b2cde8afbd91de65a3c9
     v1.19.0: 88ce7dc5302d8847f6e679aab9e4fa642a819e8a33d70731fb7bc8e110d8659f
+    v1.18.15: 8a5be9e04343e0ac10320455b32a78e5ffc60f450c5c0a11914edeb86ca178d7
+    v1.18.14: d6143cd822218daa5faf583c9b8b862c609e66052232e3d3d23c72957fdae341
+    v1.18.13: cad917c516bc326ecc17c50bf81b1c0bb153a5090206ccf88d48c23839941d08
+    v1.18.12: 6a0d1aa32adabfe84a1b12ab71b570631dd4db24e2747830f8209fb7d94ecca6
+    v1.18.11: a4b7caa7fbe1de94834b3715567956527643cdbb0e3a4688a8788ac6861dc26f
+    v1.18.10: 9bf46e5276bc14d42d6dcf05ac507bb3236ce8dc0fa21aad985d9328c377c18d
+    v1.18.9: 3f7f61e0fe3de43f5b345343f85d7ba5145737efb80974baa6076965f3a6963e
     v1.18.8: 27c8f4d4398d57762998b157d35802a36a7ea9b2b6f9a363c397a9d65b4f3c89
     v1.18.6: 11b4180b9f82a8b6bb30250e3d7341b104521f3b654076b8569853ec9451b2a9
     v1.18.5: e428fc9d1cf860090346a83eb66082c3be6b6032f0db9e4f8e6d52492d46231f
@@ -299,6 +441,8 @@ kubeadm_checksums:
     v1.18.2: 290bb6acb12c844f76affbab1ce374903bd97c4f19ac8cd3e6fdb7208d638ac8
     v1.18.1: fdb194647048f3e3ebdc93613b21a5b678fcbe0d212d08c0d56758d1bf2d2c85
     v1.18.0: 0261331c2ea718c0cd39114871aa098f1b4685f6101cb78cc880f645e72d0654
+    v1.17.13: eaea610b88369dfd46aaaaad343865f5ef2e5bffe97dda9638adb467d86cd5cd
+    v1.17.12: 214f547788410a643d57aee8215d99f44ebe59967e973f83b024e2ba7832593f
     v1.17.11: 328ca55af305fef59e72544d0faf76167b5b5dc5779cec17c670961e6529d907
     v1.17.9: 5ef1660d3d56e93e3d87d6a7028aa64745984be0b0678c45c32f66043b4d69b4
     v1.17.8: c59b85696c4cbabe896ba71f4bbc99e4ad2444fcea851e3ee740705584420aad
@@ -356,13 +500,13 @@ etcd_image_repo: "{{ quay_image_repo }}/coreos/etcd"
 etcd_image_tag: "{{ etcd_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
 flannel_image_repo: "{{ quay_image_repo }}/coreos/flannel"
 flannel_image_tag: "{{ flannel_version }}"
-calico_node_image_repo: "{{ docker_image_repo }}/calico/node"
+calico_node_image_repo: "{{ quay_image_repo }}/calico/node"
 calico_node_image_tag: "{{ calico_version }}"
-calico_cni_image_repo: "{{ docker_image_repo }}/calico/cni"
+calico_cni_image_repo: "{{ quay_image_repo }}/calico/cni"
 calico_cni_image_tag: "{{ calico_cni_version }}"
-calico_policy_image_repo: "{{ docker_image_repo }}/calico/kube-controllers"
+calico_policy_image_repo: "{{ quay_image_repo }}/calico/kube-controllers"
 calico_policy_image_tag: "{{ calico_policy_version }}"
-calico_typha_image_repo: "{{ docker_image_repo }}/calico/typha"
+calico_typha_image_repo: "{{ quay_image_repo }}/calico/typha"
 calico_typha_image_tag: "{{ calico_typha_version }}"
 pod_infra_image_repo: "{{ kube_image_repo }}/pause"
 pod_infra_image_tag: "{{ pod_infra_version }}"
@@ -387,11 +531,11 @@ contiv_etcd_init_image_repo: "{{ docker_image_repo }}/ferest/etcd-initer"
 contiv_etcd_init_image_tag: latest
 contiv_ovs_image_repo: "{{ docker_image_repo }}/contiv/ovs"
 contiv_ovs_image_tag: "latest"
-cilium_image_repo: "{{ docker_image_repo }}/cilium/cilium"
+cilium_image_repo: "{{ quay_image_repo }}/cilium/cilium"
 cilium_image_tag: "{{ cilium_version }}"
-cilium_init_image_repo: "{{ docker_image_repo }}/cilium/cilium-init"
+cilium_init_image_repo: "{{ quay_image_repo }}/cilium/cilium-init"
 cilium_init_image_tag: "2019-04-05"
-cilium_operator_image_repo: "{{ docker_image_repo }}/cilium/operator"
+cilium_operator_image_repo: "{{ quay_image_repo }}/cilium/operator"
 cilium_operator_image_tag: "{{ cilium_version }}"
 kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn"
 kube_ovn_container_image_tag: "{{ kube_ovn_version }}"
@@ -415,8 +559,8 @@ coredns_version: "1.6.7"
 coredns_image_repo: "{{ docker_image_repo }}/coredns/coredns"
 coredns_image_tag: "{{ coredns_version }}"
 
-nodelocaldns_version: "1.15.13"
-nodelocaldns_image_repo: "{{ kube_image_repo }}/k8s-dns-node-cache"
+nodelocaldns_version: "1.16.0"
+nodelocaldns_image_repo: "{{ kube_image_repo }}/dns/k8s-dns-node-cache"
 nodelocaldns_image_tag: "{{ nodelocaldns_version }}"
 
 dnsautoscaler_version: 1.8.1

roles/etcd/tasks/configure.yml

@@ -78,7 +78,7 @@
   when: is_etcd_master and etcd_events_cluster_setup
 
 - name: Configure | Wait for etcd cluster to be healthy
-  shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -q -v 'Error: unhealthy cluster'"
+  shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null"
   args:
     executable: /bin/bash
   register: etcd_cluster_is_healthy
@@ -101,7 +101,7 @@
     ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
 
 - name: Configure | Wait for etcd-events cluster to be healthy
-  shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -q -v 'Error: unhealthy cluster'"
+  shell: "set -o pipefail && {{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -v 'Error: unhealthy cluster' >/dev/null"
   args:
     executable: /bin/bash
   register: etcd_events_cluster_is_healthy

roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-write-cacert.yml

@@ -0,0 +1,12 @@
+---
+# include to workaround mitogen issue
+# https://github.com/dw/mitogen/issues/663
+
+- name: Cinder CSI Driver | Write cacert file
+  copy:
+    src: "{{ cinder_cacert }}"
+    dest: "{{ kube_config_dir }}/cinder-cacert.pem"
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  tags: cinder-csi-driver
+  delegate_to: "{{ delegate_host_to_write_cacert }}"

roles/kubernetes-apps/csi_driver/cinder/tasks/main.yml

@@ -3,11 +3,11 @@
   tags: cinder-csi-driver
 
 - name: Cinder CSI Driver | Write cacert file
-  copy:
-    src: "{{ cinder_cacert }}"
-    dest: "{{ kube_config_dir }}/cinder-cacert.pem"
-    group: "{{ kube_cert_group }}"
-    mode: 0640
+  include_tasks: cinder-write-cacert.yml
+  run_once: true
+  loop: "{{ groups['k8s-cluster'] }}"
+  loop_control:
+    loop_var: delegate_host_to_write_cacert
   when:
     - inventory_hostname in groups['k8s-cluster']
     - cinder_cacert is defined

roles/kubernetes-apps/external_cloud_controller/openstack/tasks/main.yml

@@ -3,11 +3,11 @@
   tags: external-openstack
 
 - name: External OpenStack Cloud Controller | Write cacert file
-  copy:
-    src: "{{ external_openstack_cacert }}"
-    dest: "{{ kube_config_dir }}/external-openstack-cacert.pem"
-    group: "{{ kube_cert_group }}"
-    mode: 0640
+  include_tasks: openstack-write-cacert.yml
+  run_once: true
+  loop: "{{ groups['k8s-cluster'] }}"
+  loop_control:
+    loop_var: delegate_host_to_write_cacert
   when:
     - inventory_hostname in groups['k8s-cluster']
     - external_openstack_cacert is defined

roles/kubernetes-apps/external_cloud_controller/openstack/tasks/openstack-write-cacert.yml

@@ -0,0 +1,12 @@
+---
+# include to workaround mitogen issue
+# https://github.com/dw/mitogen/issues/663
+
+- name: External OpenStack Cloud Controller | Write cacert file
+  copy:
+    src: "{{ external_openstack_cacert }}"
+    dest: "{{ kube_config_dir }}/external-openstack-cacert.pem"
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  tags: external-openstack
+  delegate_to: "{{ delegate_host_to_write_cacert }}"

roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2

@@ -59,8 +59,10 @@ spec:
               name: openstack-cacert
               readOnly: true
 {% endif %}
+{% if kubelet_flexvolumes_plugins_dir is defined %}
             - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
               name: flexvolume-dir
+{% endif %}
           resources:
             requests:
               cpu: 200m
@@ -69,10 +71,12 @@ spec:
               value: /etc/config/cloud.conf
       hostNetwork: true
       volumes:
+{% if kubelet_flexvolumes_plugins_dir is defined %}
       - hostPath:
           path: "{{ kubelet_flexvolumes_plugins_dir }}"
           type: DirectoryOrCreate
         name: flexvolume-dir
+{% endif %}
       - hostPath:
           path: /etc/kubernetes/pki
           type: DirectoryOrCreate

roles/kubernetes/master/tasks/kubeadm-upgrade.yml

@@ -3,7 +3,7 @@
   uri:
     url: "https://{{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }}/healthz"
     validate_certs: false
-  when: inventory_hostname == groups['kube-master']|first
+  when: inventory_hostname == groups['kube-master']
   register: _result
   retries: 60
   delay: 5

roles/kubernetes/node/tasks/main.yml

@@ -103,23 +103,13 @@
   tags:
     - kube-proxy
 
-- name: Modprobe nf_conntrack_ipv4 for kernels < 4.19
+- name: Modprobe nf_conntrack_ipv4
   modprobe:
     name: nf_conntrack_ipv4
     state: present
-  register: enable_nf_conntrack
+  register: modprobe_nf_conntrack_ipv4
+  ignore_errors: yes
   when:
-    - ansible_kernel.split('.')[0:3] | join('.')  < '4.19'
-    - kube_proxy_mode == 'ipvs'
-  tags:
-    - kube-proxy
-
-- name: Modprobe nf_conntrack for kernels >= 4.19
-  modprobe:
-    name: nf_conntrack
-    state: present
-  when:
-    - ansible_kernel.split('.')[0:3] | join('.')  >= '4.19'
     - kube_proxy_mode == 'ipvs'
   tags:
     - kube-proxy
@@ -132,9 +122,7 @@
       ip_vs_rr
       ip_vs_wrr
       ip_vs_sh
-      {% if enable_nf_conntrack is failed -%}
-      nf_conntrack
-      {%-   else -%}
+      {% if modprobe_nf_conntrack_ipv4 is success -%}
       nf_conntrack_ipv4
       {%-   endif -%}
   when: kube_proxy_mode == 'ipvs'

roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2

@@ -8,17 +8,19 @@
   "location": "{{ azure_location }}",
   "subnetName": "{{ azure_subnet_name }}",
   "securityGroupName": "{{ azure_security_group_name }}",
+  "securityGroupResourceGroup": "{{ azure_security_group_resource_group | default(azure_vnet_resource_group) }}",
   "vnetName": "{{ azure_vnet_name }}",
   "vnetResourceGroup": "{{ azure_vnet_resource_group }}",
   "routeTableName": "{{ azure_route_table_name }}",
+  "routeTableResourceGroup": "{{ azure_route_table_resource_group | default(azure_vnet_resource_group) }}",
   "vmType": "{{ azure_vmtype }}",
 {% if azure_primary_availability_set_name is defined %}
   "primaryAvailabilitySetName": "{{ azure_primary_availability_set_name }}",
 {%endif%}
-  "useInstanceMetadata": {{azure_use_instance_metadata }},
+  "useInstanceMetadata": {{azure_use_instance_metadata | lower }},
 {% if azure_loadbalancer_sku == "standard" %}
-  "excludeMasterFromStandardLB": {{ azure_exclude_master_from_standard_lb }},
-  "disableOutboundSNAT": {{ azure_disable_outbound_snat }},
+  "excludeMasterFromStandardLB": {{ azure_exclude_master_from_standard_lb | lower }},
+  "disableOutboundSNAT": {{ azure_disable_outbound_snat | lower }},
 {% endif%}
   "loadBalancerSku": "{{ azure_loadbalancer_sku }}"
 }

roles/kubernetes/preinstall/defaults/main.yml

@@ -53,3 +53,6 @@ minimal_node_memory_mb: 1024
 minimal_master_memory_mb: 1500
 
 yum_repo_dir: /etc/yum.repos.d
+
+# number of times package install task should be retried
+pkg_install_retries: 4

roles/kubernetes/preinstall/tasks/0062-networkmanager.yml

@@ -4,7 +4,7 @@
     path: /etc/NetworkManager/conf.d/dns.conf
     section: global-dns-domain-*
     option: servers
-    value: "{{ ( coredns_server + nameservers|d([]) + cloud_resolver|d([])) | unique | join(';') }}"
+    value: "{{ ( coredns_server + nameservers|d([]) + cloud_resolver|d([])) | unique | join(',') }}"
     mode: '0600'
     backup: yes
   notify: Preinstall | update resolvconf for Fedora CoreOS
@@ -14,7 +14,7 @@
     path: /etc/NetworkManager/conf.d/dns.conf
     section: global-dns
     option: searches
-    value: "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(';') }}"
+    value: "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(',') }}"
     mode: '0600'
     backup: yes
   notify: Preinstall | update resolvconf for Fedora CoreOS

roles/kubernetes/preinstall/tasks/0070-system-packages.yml

@@ -77,7 +77,7 @@
     state: latest
   register: pkgs_task_result
   until: pkgs_task_result is succeeded
-  retries: 4
+  retries: "{{ pkg_install_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   when: not (ansible_os_family in ["Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos)
   tags:

roles/kubespray-defaults/defaults/main.yaml

@@ -15,7 +15,7 @@ is_fedora_coreos: false
 disable_swap: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.18.8
+kube_version: v1.18.10
 
 ## The minimum version working
 kube_version_min_required: v1.17.0
@@ -431,7 +431,7 @@ contiv_peer_with_uplink_leaf: false
 contiv_global_as: "65002"
 contiv_global_neighbor_as: "500"
 
-ssl_ca_dirs: >-
+ssl_ca_dirs: |-
   [
   {% if ansible_os_family in ['Flatcar Container Linux by Kinvolk'] -%}
   '/usr/share/ca-certificates',

roles/remove-node/remove-etcd-node/tasks/main.yml

@@ -21,6 +21,8 @@
   assert:
     that: node_ip is defined and node_ip | length > 0
     msg: "Etcd node ip is not set !"
+  when:
+    - inventory_hostname in groups['etcd']
 
 - name: Lookup etcd member id
   shell: "{{ bin_dir }}/etcdctl member list | grep {{ node_ip }} | cut -d, -f1"

tests/common/_docker_hub_registry_mirror.yml

@@ -0,0 +1,15 @@
+---
+docker_registry_mirrors:
+  - "https://mirror.gcr.io"
+
+containerd_config:
+  grpc:
+    max_recv_message_size: 16777216
+    max_send_message_size: 16777216
+  debug:
+    level: ""
+  registries:
+    "docker.io":
+      - "https://mirror.gcr.io"
+      - "https://registry-1.docker.io"
+  max_container_log_line_size: -1

tests/files/tf-ovh_ubuntu18-calico.yml

@@ -2,6 +2,8 @@
 dns_min_replicas: 1
 deploy_netchecker: true
 sonobuoy_enabled: true
+pkg_install_retries: 10
+retry_stagger: 10
 
 # Ignore ping errors
 ignore_assert_errors: true

tests/scripts/testcases_run.sh

@@ -42,6 +42,7 @@ fi
 test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout "$KUBESPRAY_VERSION"
 # Checkout the CI vars file so it is available
 test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
+test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" ${CI_TEST_REGISTRY_MIRROR}
 
 # Install mitogen ansible plugin
 if [ "${MITOGEN_ENABLE}" = "true" ]; then
@@ -51,20 +52,20 @@ if [ "${MITOGEN_ENABLE}" = "true" ]; then
 fi
 
 # Create cluster
-ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 
 # Repeat deployment if testing upgrade
 if [ "${UPGRADE_TEST}" != "false" ]; then
   test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml"
   test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml"
   git checkout "${CI_BUILD_REF}"
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" $PLAYBOOK
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" $PLAYBOOK
 fi
 
 # Test control plane recovery
 if [ "${RECOVER_CONTROL_PLANE_TEST}" != "false" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit etcd,kube-master:!fake_hosts recover-control-plane.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit etcd,kube-master:!fake_hosts recover-control-plane.yml
 fi
 
 # Tests Cases
@@ -88,7 +89,7 @@ ansible-playbook -i ${ANSIBLE_INVENTORY} -e @${CI_TEST_VARS} --limit "all:!fake_
 
 ## Idempotency checks 1/5 (repeat deployment)
 if [ "${IDEMPOT_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 fi
 
 ## Idempotency checks 2/5 (Advanced DNS checks)
@@ -98,12 +99,12 @@ fi
 
 ## Idempotency checks 3/5 (reset deployment)
 if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
 fi
 
 ## Idempotency checks 4/5 (redeploy after reset)
 if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 fi
 
 ## Idempotency checks 5/5 (Advanced DNS checks)