diff --git a/contrib/dind/README.md b/contrib/dind/README.md index af0bf30ba..4eb334a54 100644 --- a/contrib/dind/README.md +++ b/contrib/dind/README.md @@ -150,3 +150,27 @@ kube-system weave-net-xr46t 2/2 Running 0 $ docker exec kube-node1 curl -s http://localhost:31081/api/v1/connectivity_check {"Message":"All 10 pods successfully reported back to the server","Absent":null,"Outdated":null} ~~~ + +## Using ./run-test-distros.sh + +You can use `./run-test-distros.sh` to run a set of tests via DIND, +and excerpt from this script, to get an idea: + +~~~ +# The SPEC file(s) must have two arrays as e.g. +# DISTROS=(debian centos) +# EXTRAS=( +# 'kube_network_plugin=calico' +# 'kube_network_plugin=flannel' +# 'kube_network_plugin=weave' +# ) +# that will be tested in a "combinatory" way (e.g. from above there'll be +# be 6 test runs), creating a sequenced -nn.out with each output. +# +# Each $EXTRAS element will be whitespace split, and passed as --extra-vars +# to main kubespray ansible-playbook run. +~~~ + +See e.g. `test-some_distros-most_CNIs.env` and +`test-some_distros-kube_router_combo.env` in particular for a richer +set of CNI specific `--extra-vars` combo. diff --git a/contrib/dind/kubespray-dind.yaml b/contrib/dind/kubespray-dind.yaml index 628ed787b..02cd16989 100644 --- a/contrib/dind/kubespray-dind.yaml +++ b/contrib/dind/kubespray-dind.yaml @@ -15,8 +15,6 @@ docker_storage_options: -s overlay2 --storage-opt overlay2.override_kernel_check dns_mode: coredns -kube_network_plugin: weave - deploy_netchecker: True netcheck_agent_image_repo: quay.io/l23network/k8s-netchecker-agent netcheck_server_image_repo: quay.io/l23network/k8s-netchecker-server diff --git a/contrib/dind/run-test-distros.sh b/contrib/dind/run-test-distros.sh old mode 100644 new mode 100755 index e1c2c3ef7..0e3510fd0 --- a/contrib/dind/run-test-distros.sh +++ b/contrib/dind/run-test-distros.sh @@ -2,45 +2,92 @@ # Q&D test'em all: creates full DIND kubespray deploys # for each distro, verifying it via netchecker. +info() { + local msg="$*" + local date="$(date -Isec)" + echo "INFO: [$date] $msg" +} pass_or_fail() { local rc="$?" local msg="$*" local date="$(date -Isec)" [ $rc -eq 0 ] && echo "PASS: [$date] $msg" || echo "FAIL: [$date] $msg" + return $rc } test_distro() { - local distro=${1:?} - ansible-playbook -i hosts dind-cluster.yaml --extra-vars node_distro=$distro - pass_or_fail "$distro: dind-nodes" + local distro=${1:?};shift + local extra="${*:-}" + local prefix="$distro[${extra}]}" + ansible-playbook -i hosts dind-cluster.yaml -e node_distro=$distro + pass_or_fail "$prefix: dind-nodes" || return 1 (cd ../.. INVENTORY_DIR=inventory/local-dind mkdir -p ${INVENTORY_DIR} rm -f ${INVENTORY_DIR}/hosts.ini CONFIG_FILE=${INVENTORY_DIR}/hosts.ini /tmp/kubespray.dind.inventory_builder.sh + # expand $extra with -e in front of each word + extra_args=""; for extra_arg in $extra; do extra_args="$extra_args -e $extra_arg"; done ansible-playbook --become -e ansible_ssh_user=$distro -i \ ${INVENTORY_DIR}/hosts.ini cluster.yml \ - --extra-vars @contrib/dind/kubespray-dind.yaml --extra-vars bootstrap_os=$distro - pass_or_fail "$distro: kubespray" - ) - docker exec kube-node1 kubectl get pod --all-namespaces - pass_or_fail "$distro: kube-api" - let n=60 - while ((n--)); do - docker exec kube-node1 curl -s http://localhost:31081/api/v1/connectivity_check | grep successfully && break + -e @contrib/dind/kubespray-dind.yaml -e bootstrap_os=$distro ${extra_args} + pass_or_fail "$prefix: kubespray" + ) || return 1 + local node0=${NODES[0]} + docker exec ${node0} kubectl get pod --all-namespaces + pass_or_fail "$prefix: kube-api" || return 1 + let retries=60 + while ((retries--)); do + # Some CNI may set NodePort on "main" node interface address (thus no localhost NodePort) + # e.g. kube-router: https://github.com/cloudnativelabs/kube-router/pull/217 + docker exec ${node0} curl -m2 -s http://${NETCHECKER_HOST:?}:31081/api/v1/connectivity_check | grep successfully && break sleep 2 done - [ $n -ge 0 ] - pass_or_fail "$distro: netcheck" + [ $retries -ge 0 ] + pass_or_fail "$prefix: netcheck" || return 1 } -# Get all DISTROS from distro.yaml if $* unset (shame no yaml parsing, but nuff anyway) -DISTROS="${*:-$(egrep -o '^ \w+' group_vars/all/distro.yaml|paste -s)}" -NODES="$(egrep ^kube-node hosts|paste -s)" -echo "DISTROS=${DISTROS}" -for distro in ${DISTROS}; do - docker rm -f ${NODES} - { time test_distro ${distro} ;} |& tee test-${distro}.out - # sleeping for the sake of the human to verify if they want - sleep 2m +NODES=($(egrep ^kube-node hosts)) +NETCHECKER_HOST=localhost + +: ${OUTPUT_DIR:=./out} +mkdir -p ${OUTPUT_DIR} + +# The SPEC file(s) must have two arrays as e.g. +# DISTROS=(debian centos) +# EXTRAS=( +# 'kube_network_plugin=calico' +# 'kube_network_plugin=flannel' +# 'kube_network_plugin=weave' +# ) +# that will be tested in a "combinatory" way (e.g. from above there'll be +# be 6 test runs), creating a sequenced -nn.out with each output. +# +# Each $EXTRAS element will be whitespace split, and passed as --extra-vars +# to main kubespray ansible-playbook run. + +SPECS=${*:?Missing SPEC files, e.g. test-most_distros-some_CNIs.env} +for spec in ${SPECS}; do + unset DISTROS EXTRAS + echo "Loading file=${spec} ..." + . ${spec} || continue + : ${DISTROS:?} || continue + echo "DISTROS=${DISTROS[@]}" + echo "EXTRAS->" + printf " %s\n" "${EXTRAS[@]}" + let n=1 + for distro in ${DISTROS[@]}; do + for extra in "${EXTRAS[@]:-NULL}"; do + # Magic value to let this for run once: + [[ ${extra} == NULL ]] && unset extra + docker rm -f ${NODES[@]} + printf -v file_out "%s/%s-%02d.out" ${OUTPUT_DIR} ${spec} $((n++)) + { + info "${distro}[${extra}] START: file_out=${file_out}" + time test_distro ${distro} ${extra} + } |& tee ${file_out} + # sleeping for the sake of the human to verify if they want + sleep 2m + done + done done -egrep '^(PASS:|FAIL:|real)' $(ls -tr test-*out) +egrep -H '^(....:|real)' $(ls -tr ${OUTPUT_DIR}/*.out) diff --git a/contrib/dind/test-most_distros-some_CNIs.env b/contrib/dind/test-most_distros-some_CNIs.env new file mode 100644 index 000000000..f6e4e1a8c --- /dev/null +++ b/contrib/dind/test-most_distros-some_CNIs.env @@ -0,0 +1,11 @@ +# Test spec file: used from ./run-test-distros.sh, will run +# each distro in $DISTROS overloading main kubespray ansible-playbook run +# Get all DISTROS from distro.yaml (shame no yaml parsing, but nuff anyway) +# DISTROS="${*:-$(egrep -o '^ \w+' group_vars/all/distro.yaml|paste -s)}" +DISTROS=(debian ubuntu centos fedora) + +# Each line below will be added as --extra-vars to main playbook run +EXTRAS=( + 'kube_network_plugin=calico' + 'kube_network_plugin=weave' +) diff --git a/contrib/dind/test-some_distros-kube_router_combo.env b/contrib/dind/test-some_distros-kube_router_combo.env new file mode 100644 index 000000000..4b0767b96 --- /dev/null +++ b/contrib/dind/test-some_distros-kube_router_combo.env @@ -0,0 +1,8 @@ +DISTROS=(debian centos) +NETCHECKER_HOST=${NODES[0]} +EXTRAS=( + 'kube_network_plugin=kube-router {"kubeadm_enabled":true,"kube_router_run_service_proxy":false}' + 'kube_network_plugin=kube-router {"kubeadm_enabled":true,"kube_router_run_service_proxy":true}' + 'kube_network_plugin=kube-router {"kubeadm_enabled":false,"kube_router_run_service_proxy":false}' + 'kube_network_plugin=kube-router {"kubeadm_enabled":false,"kube_router_run_service_proxy":true}' +) diff --git a/contrib/dind/test-some_distros-most_CNIs.env b/contrib/dind/test-some_distros-most_CNIs.env new file mode 100644 index 000000000..830695e5f --- /dev/null +++ b/contrib/dind/test-some_distros-most_CNIs.env @@ -0,0 +1,8 @@ +DISTROS=(debian centos) +EXTRAS=( + 'kube_network_plugin=calico {"kubeadm_enabled":true}' + 'kube_network_plugin=canal {"kubeadm_enabled":true}' + 'kube_network_plugin=cilium {"kubeadm_enabled":true}' + 'kube_network_plugin=flannel {"kubeadm_enabled":true}' + 'kube_network_plugin=weave {"kubeadm_enabled":true}' +)