Add a testcase to check that pods are running (#4555)

.gitlab-ci.yml

@@ -98,6 +98,9 @@ before_script:
     ## Test Master API
     - ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL
 
+    ## Test that all pods are Running
+    - ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/015_check-pods-running.yml $LOG_LEVEL
+
     ## Ping the between 2 pod
     - ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $LOG_LEVEL
 

tests/files/packet_ubuntu18-flannel-aio.yml

@@ -6,4 +6,4 @@ mode: aio
 # Kubespray settings
 kube_network_plugin: flannel
 deploy_netchecker: true
-kubedns_min_replicas: 1
+dns_min_replicas: 1

tests/testcases/015_check-pods-running.yml

@@ -0,0 +1,40 @@
+---
+- hosts: kube-master[0]
+  tasks:
+
+  - name: Force binaries directory for Container Linux by CoreOS
+    set_fact:
+      bin_dir: "/opt/bin"
+    when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]
+
+  - set_fact:
+      bin_dir: "/usr/local/bin"
+    when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]
+
+  - name: Check kubectl output
+    shell: "{{bin_dir}}/kubectl get pods --all-namespaces -owide"
+    register: get_pods
+    no_log: true
+
+  - debug: msg="{{get_pods.stdout.split('\n')}}"
+
+  - name: Check that all pods are running and ready
+    shell: "{{bin_dir}}/kubectl get pods --all-namespaces --no-headers -o yaml"
+    register: run_pods_log
+    until:
+    # Check that all pods are running
+    - '(run_pods_log.stdout | from_yaml)["items"] | map(attribute = "status.phase") | unique | list == ["Running"]'
+    # Check that all pods are ready
+    - '(run_pods_log.stdout | from_yaml)["items"] | map(attribute = "status.containerStatuses") | map("map", attribute = "ready") | map("min") | min'
+    retries: 18
+    delay: 10
+    failed_when: false
+    no_log: true
+
+  - name: Check kubectl output
+    shell: "{{bin_dir}}/kubectl get pods --all-namespaces -owide"
+    register: get_pods
+    no_log: true
+
+  - debug: msg="{{get_pods.stdout.split('\n')}}"
+    failed_when: not run_pods_log is success

tests/testcases/030_check-network.yml

@@ -12,16 +12,17 @@
       bin_dir: "/usr/local/bin"
     when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]
 
-
-  - name: Wait for pods to be ready
-    shell: "{{bin_dir}}/kubectl get pods -n test"
-    register: pods
+  - name: Check that all pods are running and ready
+    shell: "{{bin_dir}}/kubectl get pods --namespace test --no-headers -o yaml"
+    register: run_pods_log
     until:
-    - '"ContainerCreating" not in pods.stdout'
-    - '"Pending" not in pods.stdout'
-    - '"Terminating" not in pods.stdout'
-    retries: 60
-    delay: 2
+    # Check that all pods are running
+    - '(run_pods_log.stdout | from_yaml)["items"] | map(attribute = "status.phase") | unique | list == ["Running"]'
+    # Check that all pods are ready
+    - '(run_pods_log.stdout | from_yaml)["items"] | map(attribute = "status.containerStatuses") | map("map", attribute = "ready") | map("min") | min'
+    retries: 18
+    delay: 10
+    failed_when: false
     no_log: true
 
   - name: Get pod names