Cleanup tests (#8234)

* Add Fedora 35 image, support and CI

* Cleanup tests and allow_failure for vagrant

.gitlab-ci.yml

@@ -16,6 +16,7 @@ variables:
   TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID"
   CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml"
   CI_TEST_REGISTRY_MIRROR: "./tests/common/_docker_hub_registry_mirror.yml"
+  CI_TEST_SETTING: "./tests/common/_kubespray_test_settings.yml"
   GS_ACCESS_KEY_ID: $GS_KEY
   GS_SECRET_ACCESS_KEY: $GS_SECRET
   CONTAINER_ENGINE: docker

.gitlab-ci/packet.yml

@@ -58,7 +58,6 @@ packet_centos7-flannel-addons-ha:
   extends: .packet_pr
   stage: deploy-part2
   when: on_success
-  allow_failure: true
 
 packet_centos8-crio:
   extends: .packet_pr
@@ -150,6 +149,11 @@ packet_fedora34-docker-weave:
   extends: .packet_pr
   when: on_success
 
+packet_fedora35-kube-router:
+  stage: deploy-part2
+  extends: .packet_pr
+  when: on_success
+
 packet_opensuse-canal:
   stage: deploy-part2
   extends: .packet_periodic

.gitlab-ci/vagrant.yml

@@ -39,6 +39,7 @@ molecule_tests:
     - ./tests/scripts/testcases_run.sh
   after_script:
     - chronic ./tests/scripts/testcases_cleanup.sh
+  allow_failure: true
 
 vagrant_ubuntu18-calico-dual-stack:
   stage: deploy-part2

README.md

@@ -118,7 +118,7 @@ vagrant up
 - **Debian** Bullseye, Buster, Jessie, Stretch
 - **Ubuntu** 16.04, 18.04, 20.04
 - **CentOS/RHEL** 7, [8](docs/centos8.md)
-- **Fedora** 33, 34
+- **Fedora** 33, 34, 35
 - **Fedora CoreOS** (see [fcos Note](docs/fcos.md))
 - **openSUSE** Leap 15.x/Tumbleweed
 - **Oracle Linux** 7, [8](docs/centos8.md)

Vagrantfile

@@ -28,6 +28,7 @@ SUPPORTED_OS = {
   "centos8-bento"       => {box: "bento/centos-8",             user: "vagrant"},
   "fedora33"            => {box: "fedora/33-cloud-base",       user: "vagrant"},
   "fedora34"            => {box: "fedora/34-cloud-base",       user: "vagrant"},
+  "fedora35"            => {box: "fedora/35-cloud-base",       user: "vagrant"},
   "opensuse"            => {box: "bento/opensuse-leap-15.2",   user: "vagrant"},
   "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
   "oraclelinux"         => {box: "generic/oracle7",            user: "vagrant"},

docs/ci.md

@@ -14,6 +14,7 @@ debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
 fedora33 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora34 |  :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
+fedora35 |  :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
 opensuse |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 oracle7 |  :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu16 |  :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: |
@@ -32,6 +33,7 @@ debian11 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora33 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora34 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora35 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
@@ -50,6 +52,7 @@ debian11 |  :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 debian9 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora33 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 fedora34 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
+fedora35 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 opensuse |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 oracle7 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
 ubuntu16 |  :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |

roles/kubernetes/node/vars/fedora-34.yml

@@ -1,2 +0,0 @@
----
-kube_resolv_conf: "/run/systemd/resolve/resolv.conf"

roles/kubernetes/preinstall/tasks/0010-swapoff.yml

@@ -17,3 +17,7 @@
   command: /sbin/swapoff -a
   when: swapon.stdout
   ignore_errors: "{{ ansible_check_mode }}"  # noqa ignore-errors
+
+- name: Disable swapOnZram for Fedora
+  command: touch /etc/systemd/zram-generator.conf
+  when: swapon.stdout and ansible_distribution in ['Fedora']

test-infra/image-builder/roles/kubevirt-images/defaults/main.yml

@@ -41,6 +41,13 @@ images:
     converted: true
     tag: "latest"
 
+  fedora-35:
+    filename: Fedora-Cloud-Base-35-1.2.x86_64.qcow2
+    url: https://download.fedoraproject.org/pub/fedora/linux/releases/35/Cloud/x86_64/images/Fedora-Cloud-Base-35-1.2.x86_64.qcow2
+    checksum: sha256:fe84502779b3477284a8d4c86731f642ca10dd3984d2b5eccdf82630a9ca2de6
+    converted: true
+    tag: "latest"
+
   fedora-coreos:
     filename: fedora-coreos-32.20200601.3.0-openstack.x86_64.qcow2.xz
     url: https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/32.20200601.3.0/x86_64/fedora-coreos-32.20200601.3.0-openstack.x86_64.qcow2.xz

tests/cloud_playbooks/roles/packet-ci/defaults/main.yml

@@ -26,6 +26,7 @@ cloud_init:
   debian-11: "I2Nsb3VkLWNvbmZpZwp1c2VyczoKIC0gbmFtZToga3ViZXNwcmF5CiAgIHN1ZG86IEFMTD0oQUxMKSBOT1BBU1NXRDpBTEwKICAgc2hlbGw6IC9iaW4vYmFzaAogICBob21lOiAvaG9tZS91YnVudHUKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1"
   fedora-33: "I2Nsb3VkLWNvbmZpZwp1c2VyczoKIC0gbmFtZToga3ViZXNwcmF5CiAgIHN1ZG86IEFMTD0oQUxMKSBOT1BBU1NXRDpBTEwKICAgc2hlbGw6IC9iaW4vYmFzaAogICBob21lOiAvaG9tZS91YnVudHUKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1"
   fedora-34: "I2Nsb3VkLWNvbmZpZwp1c2VyczoKIC0gbmFtZToga3ViZXNwcmF5CiAgIHN1ZG86IEFMTD0oQUxMKSBOT1BBU1NXRDpBTEwKICAgc2hlbGw6IC9iaW4vYmFzaAogICBob21lOiAvaG9tZS91YnVudHUKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1"
+  fedora-35: "I2Nsb3VkLWNvbmZpZwp1c2VyczoKIC0gbmFtZToga3ViZXNwcmF5CiAgIHN1ZG86IEFMTD0oQUxMKSBOT1BBU1NXRDpBTEwKICAgc2hlbGw6IC9iaW4vYmFzaAogICBob21lOiAvaG9tZS91YnVudHUKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1"
   opensuse-leap-15: "I2Nsb3VkLWNvbmZpZwpzeXN0ZW1faW5mbzoKICBkaXN0cm86IHJoZWwKdXNlcnM6CiAtIG5hbWU6IGt1YmVzcHJheQogICBncm91cHM6IHdoZWVsCiAgIHN1ZG86ICdBTEw9KEFMTCkgTk9QQVNTV0Q6QUxMJwogICBzaGVsbDogL2Jpbi9iYXNoCiAgIGxvY2tfcGFzc3dkOiBGYWxzZQogICBob21lOiAvaG9tZS9rdWJlc3ByYXkKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1Cgo="
   rhel-server-7: "I2Nsb3VkLWNvbmZpZwpzeXN0ZW1faW5mbzoKICBkaXN0cm86IHJoZWwKdXNlcnM6CiAtIG5hbWU6IGt1YmVzcHJheQogICBncm91cHM6IHdoZWVsCiAgIHN1ZG86ICdBTEw9KEFMTCkgTk9QQVNTV0Q6QUxMJwogICBzaGVsbDogL2Jpbi9iYXNoCiAgIGxvY2tfcGFzc3dkOiBGYWxzZQogICBob21lOiAvaG9tZS9rdWJlc3ByYXkKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1Cgo="
   amazon-linux-2: "I2Nsb3VkLWNvbmZpZwpzeXN0ZW1faW5mbzoKICBkaXN0cm86IHJoZWwKdXNlcnM6CiAtIG5hbWU6IGt1YmVzcHJheQogICBncm91cHM6IHdoZWVsCiAgIHN1ZG86ICdBTEw9KEFMTCkgTk9QQVNTV0Q6QUxMJwogICBzaGVsbDogL2Jpbi9iYXNoCiAgIGxvY2tfcGFzc3dkOiBGYWxzZQogICBob21lOiAvaG9tZS9rdWJlc3ByYXkKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1Cgo="

tests/common/_docker_hub_registry_mirror.yml

@@ -23,3 +23,7 @@ crio_registries_mirrors:
 
 netcheck_agent_image_repo: "{{ quay_image_repo }}/kubespray/k8s-netchecker-agent"
 netcheck_server_image_repo: "{{ quay_image_repo }}/kubespray/k8s-netchecker-server"
+
+# Kubespray settings for tests
+deploy_netchecker: true
+dns_min_replicas: 1

tests/common/_kubespray_test_settings.yml

@@ -0,0 +1,4 @@
+---
+# Kubespray settings for tests
+deploy_netchecker: true
+dns_min_replicas: 1

tests/files/packet_amazon-linux-2-aio.yml

@@ -2,7 +2,3 @@
 # Instance settings
 cloud_image: amazon-linux-2
 mode: aio
-
-# Kubespray settings
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/packet_centos7-calico-ha-once-localhost.yml

@@ -4,16 +4,12 @@ cloud_image: centos-7
 mode: ha
 
 # Kubespray settings
-kube_network_plugin: calico
 download_localhost: true
 download_run_once: true
-deploy_netchecker: true
-dns_min_replicas: 1
 typha_enabled: true
 calico_backend: kdd
 typha_secure: true
 disable_ipv6_dns: true
-
 auto_renew_certificates: true
 
 # Docker settings

tests/files/packet_centos7-calico-ha.yml

@@ -4,13 +4,9 @@ cloud_image: centos-7
 mode: ha
 
 # Kubespray settings
-kube_network_plugin: calico
 download_localhost: false
 download_run_once: true
-deploy_netchecker: true
-dns_min_replicas: 1
 typha_enabled: true
 calico_backend: kdd
 typha_secure: true
-
 auto_renew_certificates: true

tests/files/packet_centos7-docker-weave-upgrade-ha.yml

@@ -5,9 +5,7 @@ mode: ha
 
 # Kubespray settings
 kube_network_plugin: weave
-deploy_netchecker: true
 kubernetes_audit: true
-dns_min_replicas: 1
 
 # Docker specific settings:
 container_manager: docker

tests/files/packet_centos7-flannel-addons-ha.yml

@@ -14,8 +14,6 @@ krew_enabled: true
 kubernetes_audit: true
 etcd_events_cluster_enabled: true
 local_volume_provisioner_enabled: true
-deploy_netchecker: true
-dns_min_replicas: 1
 kube_encrypt_secret_data: true
 ingress_nginx_enabled: true
 cert_manager_enabled: true

tests/files/packet_centos7-kube-router.yml

@@ -5,6 +5,4 @@ mode: default
 
 # Kubespray settings
 kube_network_plugin: kube-router
-deploy_netchecker: true
 enable_network_policy: true
-dns_min_replicas: 1

tests/files/packet_centos7-multus-calico.yml

@@ -5,6 +5,3 @@ mode: default
 
 # Kubespray settings
 kube_network_plugin_multus: true
-kube_network_plugin: calico
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/packet_centos8-calico-ha-ebpf.yml

@@ -5,11 +5,7 @@ mode: ha
 vm_memory: 3072Mi
 
 # Kubespray settings
-kube_network_plugin: calico
-deploy_netchecker: true
-
 calico_bpf_enabled: true
 loadbalancer_apiserver_localhost: true
 use_localhost_as_kubeapi_loadbalancer: true
-
 auto_renew_certificates: true

tests/files/packet_centos8-calico-nodelocaldns-secondary.yml

@@ -5,9 +5,6 @@ mode: default
 vm_memory: 3072Mi
 
 # Kubespray settings
-kube_network_plugin: calico
-deploy_netchecker: true
-dns_min_replicas: 1
 enable_nodelocaldns_secondary: true
 loadbalancer_apiserver_type: haproxy
 

tests/files/packet_centos8-calico.yml

@@ -5,9 +5,6 @@ mode: default
 vm_memory: 3072Mi
 
 # Kubespray settings
-kube_network_plugin: calico
-deploy_netchecker: true
-dns_min_replicas: 1
 metrics_server_enabled: true
 dashboard_namespace: "kube-dashboard"
 dashboard_enabled: true

tests/files/packet_centos8-crio.yml

@@ -4,11 +4,8 @@ cloud_image: centos-8
 mode: default
 
 # Kubespray settings
-deploy_netchecker: true
-dns_min_replicas: 1
 container_manager: crio
+auto_renew_certificates: true
 
 # required
 calico_iptables_backend: "Auto"
-
-auto_renew_certificates: true

tests/files/packet_centos8-docker.yml

@@ -4,10 +4,6 @@ cloud_image: centos-8
 mode: default
 vm_memory: 3072Mi
 
-# Kubespray settings
-deploy_netchecker: true
-dns_min_replicas: 1
-
 # required
 calico_iptables_backend: "Auto"
 

tests/files/packet_centos8-kube-ovn.yml

@@ -6,5 +6,3 @@ vm_memory: 3072Mi
 
 # Kubespray settings
 kube_network_plugin: kube-ovn
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/packet_debian10-aio.yml

@@ -4,10 +4,8 @@ cloud_image: debian-10
 mode: default
 
 # Kubespray settings
-deploy_netchecker: true
+auto_renew_certificates: true
-dns_min_replicas: 1
 
+# plugins
 helm_enabled: true
 krew_enabled: true
-
-auto_renew_certificates: true

tests/files/packet_debian10-calico-upgrade-once.yml

@@ -4,9 +4,6 @@ cloud_image: debian-10
 mode: default
 
 # Kubespray settings
-kube_network_plugin: calico
-deploy_netchecker: true
-dns_min_replicas: 1
 download_run_once: true
 
 # Docker specific settings:

tests/files/packet_debian10-calico-upgrade.yml

@@ -3,11 +3,6 @@
 cloud_image: debian-10
 mode: default
 
-# Kubespray settings
-kube_network_plugin: calico
-deploy_netchecker: true
-dns_min_replicas: 1
-
 # Docker specific settings:
 container_manager: docker
 etcd_deployment_type: docker

tests/files/packet_debian10-cilium-svc-proxy.yml

@@ -5,8 +5,6 @@ mode: ha
 
 # Kubespray settings
 kube_network_plugin: cilium
-deploy_netchecker: true
 enable_network_policy: true
-dns_min_replicas: 1
 
 cilium_kube_proxy_replacement: strict

tests/files/packet_debian10-docker.yml

@@ -3,10 +3,6 @@
 cloud_image: debian-10
 mode: default
 
-# Kubespray settings
-deploy_netchecker: true
-dns_min_replicas: 1
-
 # Use docker
 container_manager: docker
 etcd_deployment_type: docker

tests/files/packet_debian11-calico.yml

@@ -2,7 +2,3 @@
 # Instance settings
 cloud_image: debian-11
 mode: default
-
-# Kubespray settings
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/packet_debian11-docker.yml

@@ -3,10 +3,6 @@
 cloud_image: debian-11
 mode: default
 
-# Kubespray settings
-deploy_netchecker: true
-dns_min_replicas: 1
-
 # Use docker
 container_manager: docker
 etcd_deployment_type: docker

tests/files/packet_debian9-macvlan.yml

@@ -5,10 +5,7 @@ mode: default
 
 # Kubespray settings
 kube_network_plugin: macvlan
-deploy_netchecker: true
 enable_nodelocaldns: false
-dns_min_replicas: 1
 kube_proxy_masquerade_all: true
 macvlan_interface: "eth0"
-
 auto_renew_certificates: true

tests/files/packet_fedora33-calico.yml

@@ -4,8 +4,4 @@ cloud_image: fedora-33
 mode: default
 
 # Kubespray settings
-deploy_netchecker: true
-dns_min_replicas: 1
-kube_network_plugin: calico
-
 auto_renew_certificates: true

tests/files/packet_fedora34-calico-selinux.yml

@@ -4,10 +4,6 @@ cloud_image: fedora-34
 mode: default
 
 # Kubespray settings
-deploy_netchecker: true
-dns_min_replicas: 1
-kube_network_plugin: calico
-
 auto_renew_certificates: true
 
 # Test with SELinux in enforcing mode

tests/files/packet_fedora34-docker-weave.yml

@@ -4,8 +4,6 @@ cloud_image: fedora-34
 mode: default
 
 # Kubespray settings
-deploy_netchecker: true
-dns_min_replicas: 1
 kube_network_plugin: weave
 
 # Docker specific settings:

tests/files/packet_fedora34-kube-ovn.yml

@@ -4,6 +4,4 @@ cloud_image: fedora-34
 mode: default
 
 # Kubespray settings
-deploy_netchecker: true
-dns_min_replicas: 1
 kube_network_plugin: kube-ovn

tests/files/packet_fedora35-kube-router.yml

@@ -0,0 +1,7 @@
+---
+# Instance settings
+cloud_image: fedora-35
+mode: default
+
+# Kubespray settings
+kube_network_plugin: kube-router

tests/files/packet_opensuse-canal.yml

@@ -6,7 +6,4 @@ mode: default
 # Kubespray settings
 calico_datastore: etcd
 kube_network_plugin: canal
-deploy_netchecker: true
-dns_min_replicas: 1
-
 auto_renew_certificates: true

tests/files/packet_oracle7-canal-ha.yml

@@ -6,7 +6,4 @@ mode: ha
 # Kubespray settings
 calico_datastore: etcd
 kube_network_plugin: canal
-deploy_netchecker: true
-dns_min_replicas: 1
-
 auto_renew_certificates: true

tests/files/packet_ubuntu16-canal-ha.yml

@@ -6,5 +6,3 @@ mode: ha
 # Kubespray settings
 calico_datastore: etcd
 kube_network_plugin: canal
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/packet_ubuntu16-canal-sep.yml

@@ -6,5 +6,3 @@ mode: separate
 # Kubespray settings
 calico_datastore: etcd
 kube_network_plugin: canal
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/packet_ubuntu16-docker-weave-sep.yml

@@ -5,11 +5,8 @@ mode: separate
 
 # Kubespray settings
 kube_network_plugin: weave
-deploy_netchecker: true
+auto_renew_certificates: true
-dns_min_replicas: 1
 
 # Docker specific settings:
 container_manager: docker
 etcd_deployment_type: docker
-
-auto_renew_certificates: true

tests/files/packet_ubuntu16-flannel-ha.yml

@@ -8,5 +8,3 @@ kube_network_plugin: flannel
 etcd_kubeadm_enabled: true
 kubeadm_certificate_key: 3998c58db6497dd17d909394e62d515368c06ec617710d02edea31c06d741085
 skip_non_kubeadm_warning: true
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/packet_ubuntu16-kube-router-sep.yml

@@ -6,5 +6,3 @@ mode: separate
 # Kubespray settings
 bootstrap_os: ubuntu
 kube_network_plugin: kube-router
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/packet_ubuntu16-kube-router-svc-proxy.yml

@@ -6,7 +6,5 @@ mode: separate
 # Kubespray settings
 bootstrap_os: ubuntu
 kube_network_plugin: kube-router
-deploy_netchecker: true
-dns_min_replicas: 1
 
 kube_router_run_service_proxy: true

tests/files/packet_ubuntu18-calico-aio.yml

@@ -3,8 +3,3 @@
 cloud_image: ubuntu-1804
 mode: aio
 vm_memory: 1600Mi
-
-# Kubespray settings
-kube_network_plugin: calico
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/packet_ubuntu18-calico-ha-recover-noquorum.yml

@@ -3,8 +3,3 @@
 cloud_image: ubuntu-1804
 mode: ha-recover-noquorum
 vm_memory: 1600Mi
-
-# Kubespray settings
-kube_network_plugin: calico
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/packet_ubuntu18-calico-ha-recover.yml

@@ -3,8 +3,3 @@
 cloud_image: ubuntu-1804
 mode: ha-recover
 vm_memory: 1600Mi
-
-# Kubespray settings
-kube_network_plugin: calico
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/packet_ubuntu18-cilium-sep.yml

@@ -5,8 +5,5 @@ mode: separate
 
 # Kubespray settings
 kube_network_plugin: cilium
-deploy_netchecker: true
 enable_network_policy: true
-dns_min_replicas: 1
-
 auto_renew_certificates: true

tests/files/packet_ubuntu18-crio.yml

@@ -4,8 +4,6 @@ cloud_image: ubuntu-1804
 mode: default
 
 # Kubespray settings
-deploy_netchecker: true
-dns_min_replicas: 1
 container_manager: crio
 
 download_localhost: false

tests/files/packet_ubuntu18-docker.yml

@@ -4,10 +4,6 @@ cloud_image: ubuntu-1804
 mode: aio
 vm_memory: 1600Mi
 
-# Kubespray settings
-deploy_netchecker: true
-dns_min_replicas: 1
-
 # Use docker
 container_manager: docker
 etcd_deployment_type: docker

tests/files/packet_ubuntu18-flannel-ha-once.yml

@@ -13,8 +13,6 @@ krew_enabled: true
 kubernetes_audit: true
 etcd_events_cluster_enabled: true
 local_volume_provisioner_enabled: true
-deploy_netchecker: true
-dns_min_replicas: 1
 kube_encrypt_secret_data: true
 ingress_nginx_enabled: true
 cert_manager_enabled: true

tests/files/packet_ubuntu18-flannel-ha.yml

@@ -13,8 +13,6 @@ krew_enabled: true
 kubernetes_audit: true
 etcd_events_cluster_enabled: true
 local_volume_provisioner_enabled: true
-deploy_netchecker: true
-dns_min_replicas: 1
 kube_encrypt_secret_data: true
 ingress_nginx_enabled: true
 cert_manager_enabled: true

tests/files/packet_ubuntu18-ovn4nfv.yml

@@ -5,5 +5,3 @@ mode: default
 
 # Kubespray settings
 kube_network_plugin: ovn4nfv
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/packet_ubuntu20-calico-aio.yml

@@ -5,12 +5,8 @@ mode: aio
 vm_memory: 1600Mi
 
 # Kubespray settings
-kube_network_plugin: calico
+auto_renew_certificates: true
-deploy_netchecker: true
-dns_min_replicas: 1
 
 # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko
 kube_proxy_mode: iptables
 enable_nodelocaldns: False
-
-auto_renew_certificates: true

tests/files/packet_ubuntu20-calico-ha-wireguard.yml

@@ -5,14 +5,10 @@ mode: ha
 vm_memory: 1600Mi
 
 # Kubespray settings
-kube_network_plugin: calico
-deploy_netchecker: true
-
 calico_wireguard_enabled: true
+auto_renew_certificates: true
 
 # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko
 kube_proxy_mode: iptables
 # KVM kernel used by packet instances is missing the dummy.ko kernel module so it cannot enable nodelocaldns
 enable_nodelocaldns: false
-
-auto_renew_certificates: true

tests/files/packet_ubuntu20-docker.yml

@@ -5,15 +5,12 @@ mode: aio
 vm_memory: 1600Mi
 
 # Kubespray settings
-deploy_netchecker: true
+auto_renew_certificates: true
-dns_min_replicas: 1
 
 # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko
 kube_proxy_mode: iptables
 enable_nodelocaldns: False
 
-auto_renew_certificates: true
-
 # Use docker
 container_manager: docker
 etcd_deployment_type: docker

tests/files/tf-elastx_ubuntu18-calico.yml

@@ -1,6 +1,4 @@
 ---
-dns_min_replicas: 1
-deploy_netchecker: true
 sonobuoy_enabled: true
 
 # Ignore ping errors

tests/files/tf-ovh_ubuntu18-calico.yml

@@ -1,6 +1,4 @@
 ---
-dns_min_replicas: 1
-deploy_netchecker: true
 sonobuoy_enabled: true
 pkg_install_retries: 25
 retry_stagger: 10

tests/files/vagrant_ubuntu18-calico-dual-stack.yml

@@ -1,8 +1,3 @@
 ---
 # Kubespray settings
-
-kube_network_plugin: calico
 enable_dual_stack_networks: true
-
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/vagrant_ubuntu18-flannel.yml

@@ -1,7 +1,3 @@
 ---
 # Kubespray settings
-
 kube_network_plugin: flannel
-
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/vagrant_ubuntu18-weave-medium.yml

@@ -1,7 +1,3 @@
 ---
 # Kubespray settings
-
 kube_network_plugin: weave
-
-deploy_netchecker: true
-dns_min_replicas: 1

tests/files/vagrant_ubuntu20-flannel.yml

@@ -1,7 +1,3 @@
 ---
 # Kubespray settings
-
 kube_network_plugin: flannel
-
-deploy_netchecker: true
-dns_min_replicas: 1

tests/scripts/testcases_run.sh

@@ -49,22 +49,23 @@ test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout "$KUBESPRAY
 # Checkout the CI vars file so it is available
 test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
 test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" ${CI_TEST_REGISTRY_MIRROR}
+test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" ${CI_TEST_SETTING}
 
 # Create cluster
-ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_SETTING} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 
 # Repeat deployment if testing upgrade
 if [ "${UPGRADE_TEST}" != "false" ]; then
   test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml"
   test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml"
   git checkout "${CI_BUILD_REF}"
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" $PLAYBOOK
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_SETTING} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" $PLAYBOOK
 fi
 
 # Test control plane recovery
 if [ "${RECOVER_CONTROL_PLANE_TEST}" != "false" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_SETTING} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit etcd,kube_control_plane:!fake_hosts recover-control-plane.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_SETTING} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit etcd,kube_control_plane:!fake_hosts recover-control-plane.yml
 fi
 
 # Tests Cases
@@ -88,24 +89,24 @@ ansible-playbook -i ${ANSIBLE_INVENTORY} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONA
 
 if [ "${IDEMPOT_CHECK}" = "true" ]; then
   ## Idempotency checks 1/5 (repeat deployment)
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} ${CI_TEST_ADDITIONAL_VARS} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_SETTING} -e @${CI_TEST_REGISTRY_MIRROR} ${CI_TEST_ADDITIONAL_VARS} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 
   ## Idempotency checks 2/5 (Advanced DNS checks)
   ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml
 
   if [ "${RESET_CHECK}" = "true" ]; then
     ## Idempotency checks 3/5 (reset deployment)
-    ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR}  -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
+    ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_SETTING} -e @${CI_TEST_REGISTRY_MIRROR}  -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
 
     ## Idempotency checks 4/5 (redeploy after reset)
-    ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+    ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_SETTING} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 
     ## Idempotency checks 5/5 (Advanced DNS checks)
-    ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml
+    ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_SETTING} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml
   fi
 fi
 
 # Clean up at the end, this is to allow stage1 tests to include cleanup test
 if [ "${RESET_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR}  -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_SETTING} -e @${CI_TEST_REGISTRY_MIRROR}  -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
 fi