[2.14] fix ci (#7021)

* fix flake8 errors in Kubespray CI - tox-inventory-builder

* fix flake8 errors in Kubespray CI - tox-inventory-builder

* Invalidate CRI-O kubic repo's cache

Signed-off-by: Victor Morales <v.morales@samsung.com>

* add support to configure pkg install retries

and use in CI job tf-ovh_ubuntu18-calico (due to it failing often)

* Switch Calico and Cilium image repos to Quay.io

Co-authored-by: Victor Morales <v.morales@samsung.com>
Co-authored-by: Barry Melbourne <9964974+bmelbourne@users.noreply.github.com>

Conflicts:
	roles/download/defaults/main.yml

* up vagrant box to fedora/33-cloud-base in cri-o molecule tests

(cherry picked from commit 06ec5393d7)

* add Google proxy-mirror-cache for docker hub to CI tests

(cherry picked from commit d739a6bb2f)

* containerd docker hub registry mirror support

* containerd docker hub registry mirror support

* add docs

* fix typo

* fix yamllint

* fix indent in sample
and ansible-playbook param in testcases_run

* fix md

* mv common vars to tests/common/_docker_hub_registry_mirror.yml

* checkout vars to upgrade tests

(cherry picked from commit 4a8a52bad9)

* Exclude .git/ from shellcheck

If a branch name contains '.sh', current shellcheck checks the branch
file under .git/ and outputs error because the format is not shell
script one.
This makes shellcheck exclude files under .git/ to avoid this issue.

(cherry picked from commit e2467d87b6)

Co-authored-by: Hans Feldt <2808287+hafe@users.noreply.github.com>
Co-authored-by: Sergey <s.bondarev@southbridge.ru>
Co-authored-by: Kenichi Omichi <ken-oomichi@wx.jp.nec.com>

.gitlab-ci.yml

@@ -15,6 +15,7 @@ variables:
   MAGIC: "ci check this"
   TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID"
   CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml"
+  CI_TEST_REGISTRY_MIRROR: "./tests/common/_docker_hub_registry_mirror.yml"
   GS_ACCESS_KEY_ID: $GS_KEY
   GS_SECRET_ACCESS_KEY: $GS_SECRET
   CONTAINER_ENGINE: docker

.gitlab-ci/shellcheck.yml

@@ -12,5 +12,5 @@ shellcheck:
     - shellcheck --version
   script:
     # Run shellcheck for all *.sh except contrib/
-    - find . -name '*.sh' -not -path './contrib/*' | xargs shellcheck --severity error
+    - find . -name '*.sh' -not -path './contrib/*' -not -path './.git/*' | xargs shellcheck --severity error
   except: ['triggers', 'master']

contrib/inventory_builder/tests/test_inventory.py

@@ -51,7 +51,7 @@ class TestInventory(unittest.TestCase):
         groups = ['group1', 'group2']
         self.inv.ensure_required_groups(groups)
         for group in groups:
-            self.assertTrue(group in self.inv.yaml_config['all']['children'])
+            self.assertIn(group, self.inv.yaml_config['all']['children'])
 
     def test_get_host_id(self):
         hostnames = ['node99', 'no99de01', '01node01', 'node1.domain',
@@ -209,8 +209,8 @@ class TestInventory(unittest.TestCase):
             ('doesnotbelong2', {'whateveropts=ilike'})])
         self.inv.yaml_config['all']['hosts'] = existing_hosts
         self.inv.purge_invalid_hosts(proper_hostnames)
-        self.assertTrue(
+        self.assertNotIn(
-            bad_host not in self.inv.yaml_config['all']['hosts'].keys())
+            bad_host, self.inv.yaml_config['all']['hosts'].keys())
 
     def test_add_host_to_group(self):
         group = 'etcd'
@@ -227,8 +227,8 @@ class TestInventory(unittest.TestCase):
         host = 'node1'
 
         self.inv.set_kube_master([host])
-        self.assertTrue(
+        self.assertIn(
-            host in self.inv.yaml_config['all']['children'][group]['hosts'])
+            host, self.inv.yaml_config['all']['children'][group]['hosts'])
 
     def test_set_all(self):
         hosts = OrderedDict([
@@ -246,8 +246,8 @@ class TestInventory(unittest.TestCase):
 
         self.inv.set_k8s_cluster()
         for host in expected_hosts:
-            self.assertTrue(
+            self.assertIn(
-                host in
+                host,
                 self.inv.yaml_config['all']['children'][group]['children'])
 
     def test_set_kube_node(self):
@@ -255,16 +255,16 @@ class TestInventory(unittest.TestCase):
         host = 'node1'
 
         self.inv.set_kube_node([host])
-        self.assertTrue(
+        self.assertIn(
-            host in self.inv.yaml_config['all']['children'][group]['hosts'])
+            host, self.inv.yaml_config['all']['children'][group]['hosts'])
 
     def test_set_etcd(self):
         group = 'etcd'
         host = 'node1'
 
         self.inv.set_etcd([host])
-        self.assertTrue(
+        self.assertIn(
-            host in self.inv.yaml_config['all']['children'][group]['hosts'])
+            host, self.inv.yaml_config['all']['children'][group]['hosts'])
 
     def test_scale_scenario_one(self):
         num_nodes = 50

docs/containerd.md

@@ -0,0 +1,31 @@
+# conrainerd
+
+[containerd] An industry-standard container runtime with an emphasis on simplicity, robustness and portability
+Kubespray supports basic functionality for using containerd as the default container runtime in a cluster.
+
+_To use the containerd container runtime set the following variables:_
+
+## k8s-cluster.yml
+
+```yaml
+container_manager: containerd
+```
+
+## Containerd config
+
+Example: define registry mirror for docker hub
+
+```yaml
+containerd_config:
+  grpc:
+    max_recv_message_size: 16777216
+    max_send_message_size: 16777216
+  debug:
+    level: ""
+  registries:
+    "docker.io":
+      - "https://mirror.gcr.io"
+      - "https://registry-1.docker.io"
+```
+
+[containerd]: https://containerd.io/

inventory/sample/group_vars/all/containerd.yml

@@ -1,6 +1,8 @@
 ---
 # Please see roles/container-engine/containerd/defaults/main.yml for more configuration options
 
+# Example: define registry mirror for docker hub
+
 # containerd_config:
 #   grpc:
 #     max_recv_message_size: 16777216
@@ -8,7 +10,9 @@
 #   debug:
 #     level: ""
 #   registries:
-#     "docker.io": "https://registry-1.docker.io"
+#     "docker.io":
+#       - "https://mirror.gcr.io"
+#       - "https://registry-1.docker.io"
 #   max_container_log_line_size: -1
 #   metrics:
 #     address: ""

roles/container-engine/containerd/templates/config.toml.j2

@@ -62,7 +62,7 @@ disabled_plugins = ["restart"]
 [plugins.cri.registry.mirrors]
 {% for registry, addr in containerd_config.registries.items() %}
 [plugins.cri.registry.mirrors."{{ registry }}"]
-  endpoint = ["{{ addr }}"]
+  endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"]
 {% endfor %}
 {% endif %}
 

roles/container-engine/cri-o/molecule/default/molecule.yml

@@ -26,7 +26,7 @@ platforms:
     groups:
       - kube-master
   - name: fedora
-    box: fedora/31-cloud-base
+    box: fedora/33-cloud-base
     cpus: 2
     memory: 1024
     groups:

roles/container-engine/cri-o/tasks/crio_repo.yml

@@ -29,6 +29,7 @@
     baseurl: http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/
     gpgcheck: yes
     gpgkey: http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/repodata/repomd.xml.key
+    keepcache: false
   when: ansible_distribution in ["CentOS"]
 
 - name: Add CRI-O kubic repo

roles/download/defaults/main.yml

@@ -419,13 +419,13 @@ etcd_image_repo: "{{ quay_image_repo }}/coreos/etcd"
 etcd_image_tag: "{{ etcd_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
 flannel_image_repo: "{{ quay_image_repo }}/coreos/flannel"
 flannel_image_tag: "{{ flannel_version }}"
-calico_node_image_repo: "{{ docker_image_repo }}/calico/node"
+calico_node_image_repo: "{{ quay_image_repo }}/calico/node"
 calico_node_image_tag: "{{ calico_version }}"
-calico_cni_image_repo: "{{ docker_image_repo }}/calico/cni"
+calico_cni_image_repo: "{{ quay_image_repo }}/calico/cni"
 calico_cni_image_tag: "{{ calico_cni_version }}"
-calico_policy_image_repo: "{{ docker_image_repo }}/calico/kube-controllers"
+calico_policy_image_repo: "{{ quay_image_repo }}/calico/kube-controllers"
 calico_policy_image_tag: "{{ calico_policy_version }}"
-calico_typha_image_repo: "{{ docker_image_repo }}/calico/typha"
+calico_typha_image_repo: "{{ quay_image_repo }}/calico/typha"
 calico_typha_image_tag: "{{ calico_typha_version }}"
 pod_infra_image_repo: "{{ kube_image_repo }}/pause"
 pod_infra_image_tag: "{{ pod_infra_version }}"
@@ -450,11 +450,11 @@ contiv_etcd_init_image_repo: "{{ docker_image_repo }}/ferest/etcd-initer"
 contiv_etcd_init_image_tag: latest
 contiv_ovs_image_repo: "{{ docker_image_repo }}/contiv/ovs"
 contiv_ovs_image_tag: "latest"
-cilium_image_repo: "{{ docker_image_repo }}/cilium/cilium"
+cilium_image_repo: "{{ quay_image_repo }}/cilium/cilium"
 cilium_image_tag: "{{ cilium_version }}"
-cilium_init_image_repo: "{{ docker_image_repo }}/cilium/cilium-init"
+cilium_init_image_repo: "{{ quay_image_repo }}/cilium/cilium-init"
 cilium_init_image_tag: "2019-04-05"
-cilium_operator_image_repo: "{{ docker_image_repo }}/cilium/operator"
+cilium_operator_image_repo: "{{ quay_image_repo }}/cilium/operator"
 cilium_operator_image_tag: "{{ cilium_version }}"
 kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn"
 kube_ovn_container_image_tag: "{{ kube_ovn_version }}"

roles/kubernetes/preinstall/defaults/main.yml

@@ -53,3 +53,6 @@ minimal_node_memory_mb: 1024
 minimal_master_memory_mb: 1500
 
 yum_repo_dir: /etc/yum.repos.d
+
+# number of times package install task should be retried
+pkg_install_retries: 4

roles/kubernetes/preinstall/tasks/0070-system-packages.yml

@@ -77,7 +77,7 @@
     state: latest
   register: pkgs_task_result
   until: pkgs_task_result is succeeded
-  retries: 4
+  retries: "{{ pkg_install_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   when: not (ansible_os_family in ["Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_fedora_coreos)
   tags:

tests/common/_docker_hub_registry_mirror.yml

@@ -0,0 +1,15 @@
+---
+docker_registry_mirrors:
+  - "https://mirror.gcr.io"
+
+containerd_config:
+  grpc:
+    max_recv_message_size: 16777216
+    max_send_message_size: 16777216
+  debug:
+    level: ""
+  registries:
+    "docker.io":
+      - "https://mirror.gcr.io"
+      - "https://registry-1.docker.io"
+  max_container_log_line_size: -1

tests/files/tf-ovh_ubuntu18-calico.yml

@@ -2,6 +2,8 @@
 dns_min_replicas: 1
 deploy_netchecker: true
 sonobuoy_enabled: true
+pkg_install_retries: 10
+retry_stagger: 10
 
 # Ignore ping errors
 ignore_assert_errors: true

tests/scripts/testcases_run.sh

@@ -42,6 +42,7 @@ fi
 test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout "$KUBESPRAY_VERSION"
 # Checkout the CI vars file so it is available
 test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
+test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" ${CI_TEST_REGISTRY_MIRROR}
 
 # Install mitogen ansible plugin
 if [ "${MITOGEN_ENABLE}" = "true" ]; then
@@ -51,20 +52,20 @@ if [ "${MITOGEN_ENABLE}" = "true" ]; then
 fi
 
 # Create cluster
-ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 
 # Repeat deployment if testing upgrade
 if [ "${UPGRADE_TEST}" != "false" ]; then
   test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml"
   test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml"
   git checkout "${CI_BUILD_REF}"
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" $PLAYBOOK
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" $PLAYBOOK
 fi
 
 # Test control plane recovery
 if [ "${RECOVER_CONTROL_PLANE_TEST}" != "false" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit etcd,kube-master:!fake_hosts recover-control-plane.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit etcd,kube-master:!fake_hosts recover-control-plane.yml
 fi
 
 # Tests Cases
@@ -88,7 +89,7 @@ ansible-playbook -i ${ANSIBLE_INVENTORY} -e @${CI_TEST_VARS} --limit "all:!fake_
 
 ## Idempotency checks 1/5 (repeat deployment)
 if [ "${IDEMPOT_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 fi
 
 ## Idempotency checks 2/5 (Advanced DNS checks)
@@ -98,12 +99,12 @@ fi
 
 ## Idempotency checks 3/5 (reset deployment)
 if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
 fi
 
 ## Idempotency checks 4/5 (redeploy after reset)
 if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 fi
 
 ## Idempotency checks 5/5 (Advanced DNS checks)