Merge pull request #3532 from jjo/jjo-improve-dind

[jjo] improve contrib/dind/run-test-distros.sh via spec files

contrib/dind/README.md

@@ -150,3 +150,27 @@ kube-system   weave-net-xr46t                         2/2     Running   0
 $ docker exec kube-node1 curl -s http://localhost:31081/api/v1/connectivity_check
 {"Message":"All 10 pods successfully reported back to the server","Absent":null,"Outdated":null}
 ~~~
+
+## Using ./run-test-distros.sh
+
+You can use `./run-test-distros.sh` to run a set of tests via DIND,
+and excerpt from this script, to get an idea:
+
+~~~
+# The SPEC file(s) must have two arrays as e.g.
+# DISTROS=(debian centos)
+# EXTRAS=(
+#     'kube_network_plugin=calico'
+#     'kube_network_plugin=flannel'
+#     'kube_network_plugin=weave'
+# )
+# that will be tested in a "combinatory" way (e.g. from above there'll be
+# be 6 test runs), creating a sequenced <spec_filename>-nn.out with each output.
+#
+# Each $EXTRAS element will be whitespace split, and passed as --extra-vars
+# to main kubespray ansible-playbook run.
+~~~
+
+See e.g. `test-some_distros-most_CNIs.env` and
+`test-some_distros-kube_router_combo.env` in particular for a richer
+set of CNI specific `--extra-vars` combo.

contrib/dind/kubespray-dind.yaml

@@ -15,8 +15,6 @@ docker_storage_options: -s overlay2 --storage-opt overlay2.override_kernel_check
 
 dns_mode: coredns
 
-kube_network_plugin: weave
-
 deploy_netchecker: True
 netcheck_agent_image_repo: quay.io/l23network/k8s-netchecker-agent
 netcheck_server_image_repo: quay.io/l23network/k8s-netchecker-server

contrib/dind/run-test-distros.sh

@@ -2,45 +2,92 @@
 # Q&D test'em all: creates full DIND kubespray deploys
 # for each distro, verifying it via netchecker.
 
+info() {
+    local msg="$*"
+    local date="$(date -Isec)"
+    echo "INFO: [$date] $msg"
+}
 pass_or_fail() {
     local rc="$?"
     local msg="$*"
     local date="$(date -Isec)"
     [ $rc -eq 0 ] && echo "PASS: [$date] $msg" || echo "FAIL: [$date] $msg"
+    return $rc
 }
 test_distro() {
-    local distro=${1:?}
+    local distro=${1:?};shift
-    ansible-playbook -i hosts dind-cluster.yaml --extra-vars node_distro=$distro
+    local extra="${*:-}"
-    pass_or_fail "$distro: dind-nodes"
+    local prefix="$distro[${extra}]}"
+    ansible-playbook -i hosts dind-cluster.yaml -e node_distro=$distro
+    pass_or_fail "$prefix: dind-nodes" || return 1
     (cd ../..
         INVENTORY_DIR=inventory/local-dind
         mkdir -p ${INVENTORY_DIR}
         rm -f ${INVENTORY_DIR}/hosts.ini
         CONFIG_FILE=${INVENTORY_DIR}/hosts.ini /tmp/kubespray.dind.inventory_builder.sh
+        # expand $extra with -e in front of each word
+        extra_args=""; for extra_arg in $extra; do extra_args="$extra_args -e $extra_arg"; done
         ansible-playbook --become -e ansible_ssh_user=$distro -i \
             ${INVENTORY_DIR}/hosts.ini cluster.yml \
-            --extra-vars @contrib/dind/kubespray-dind.yaml --extra-vars bootstrap_os=$distro
+            -e @contrib/dind/kubespray-dind.yaml -e bootstrap_os=$distro ${extra_args}
-        pass_or_fail "$distro: kubespray"
+        pass_or_fail "$prefix: kubespray"
-    )
+    ) || return 1
-    docker exec kube-node1 kubectl get pod --all-namespaces
+    local node0=${NODES[0]}
-    pass_or_fail "$distro: kube-api"
+    docker exec ${node0} kubectl get pod --all-namespaces
-    let n=60
+    pass_or_fail "$prefix: kube-api" || return 1
-    while ((n--)); do
+    let retries=60
-        docker exec kube-node1 curl -s http://localhost:31081/api/v1/connectivity_check | grep successfully && break
+    while ((retries--)); do
+        # Some CNI may set NodePort on "main" node interface address (thus no localhost NodePort)
+        # e.g. kube-router: https://github.com/cloudnativelabs/kube-router/pull/217
+        docker exec ${node0} curl -m2 -s http://${NETCHECKER_HOST:?}:31081/api/v1/connectivity_check | grep successfully && break
         sleep 2
     done
-    [ $n -ge 0 ]
+    [ $retries -ge 0 ]
-    pass_or_fail "$distro: netcheck"
+    pass_or_fail "$prefix: netcheck" || return 1
 }
 
-# Get all DISTROS from distro.yaml if $* unset (shame no yaml parsing, but nuff anyway)
+NODES=($(egrep ^kube-node hosts))
-DISTROS="${*:-$(egrep -o '^  \w+' group_vars/all/distro.yaml|paste -s)}"
+NETCHECKER_HOST=localhost
-NODES="$(egrep ^kube-node hosts|paste -s)"
+
-echo "DISTROS=${DISTROS}"
+: ${OUTPUT_DIR:=./out}
-for distro in ${DISTROS}; do
+mkdir -p ${OUTPUT_DIR}
-    docker rm -f ${NODES}
+
-    { time test_distro ${distro} ;} |& tee test-${distro}.out
+# The SPEC file(s) must have two arrays as e.g.
+# DISTROS=(debian centos)
+# EXTRAS=(
+#     'kube_network_plugin=calico'
+#     'kube_network_plugin=flannel'
+#     'kube_network_plugin=weave'
+# )
+# that will be tested in a "combinatory" way (e.g. from above there'll be
+# be 6 test runs), creating a sequenced <spec_filename>-nn.out with each output.
+#
+# Each $EXTRAS element will be whitespace split, and passed as --extra-vars
+# to main kubespray ansible-playbook run.
+
+SPECS=${*:?Missing SPEC files, e.g. test-most_distros-some_CNIs.env}
+for spec in ${SPECS}; do
+    unset DISTROS EXTRAS
+    echo "Loading file=${spec} ..."
+    . ${spec} || continue
+    : ${DISTROS:?} || continue
+    echo "DISTROS=${DISTROS[@]}"
+    echo "EXTRAS->"
+    printf "  %s\n" "${EXTRAS[@]}"
+    let n=1
+    for distro in ${DISTROS[@]}; do
+        for extra in "${EXTRAS[@]:-NULL}"; do
+            # Magic value to let this for run once:
+            [[ ${extra} == NULL ]] && unset extra
+            docker rm -f ${NODES[@]}
+            printf -v file_out "%s/%s-%02d.out" ${OUTPUT_DIR} ${spec} $((n++))
+            {
+                info "${distro}[${extra}] START: file_out=${file_out}"
+                time test_distro ${distro} ${extra}
+            } |& tee ${file_out}
             # sleeping for the sake of the human to verify if they want
             sleep 2m
+        done
+    done
 done
-egrep '^(PASS:|FAIL:|real)' $(ls -tr test-*out)
+egrep -H '^(....:|real)' $(ls -tr ${OUTPUT_DIR}/*.out)

contrib/dind/test-most_distros-some_CNIs.env

@@ -0,0 +1,11 @@
+# Test spec file: used from ./run-test-distros.sh, will run
+# each distro in $DISTROS overloading main kubespray ansible-playbook run
+# Get all DISTROS from distro.yaml (shame no yaml parsing, but nuff anyway)
+# DISTROS="${*:-$(egrep -o '^  \w+' group_vars/all/distro.yaml|paste -s)}"
+DISTROS=(debian ubuntu centos fedora)
+
+# Each line below will be added as --extra-vars to main playbook run
+EXTRAS=(
+    'kube_network_plugin=calico'
+    'kube_network_plugin=weave'
+)

contrib/dind/test-some_distros-kube_router_combo.env

@@ -0,0 +1,8 @@
+DISTROS=(debian centos)
+NETCHECKER_HOST=${NODES[0]}
+EXTRAS=(
+  'kube_network_plugin=kube-router {"kubeadm_enabled":true,"kube_router_run_service_proxy":false}'
+  'kube_network_plugin=kube-router {"kubeadm_enabled":true,"kube_router_run_service_proxy":true}'
+  'kube_network_plugin=kube-router {"kubeadm_enabled":false,"kube_router_run_service_proxy":false}'
+  'kube_network_plugin=kube-router {"kubeadm_enabled":false,"kube_router_run_service_proxy":true}'
+)

contrib/dind/test-some_distros-most_CNIs.env

@@ -0,0 +1,8 @@
+DISTROS=(debian centos)
+EXTRAS=(
+  'kube_network_plugin=calico {"kubeadm_enabled":true}'
+  'kube_network_plugin=canal {"kubeadm_enabled":true}'
+  'kube_network_plugin=cilium {"kubeadm_enabled":true}'
+  'kube_network_plugin=flannel {"kubeadm_enabled":true}'
+  'kube_network_plugin=weave {"kubeadm_enabled":true}'
+)