From c4594022ca56415e940143f4a42314d95eb83105 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 3 Feb 2017 18:50:58 +0300 Subject: [PATCH 1/2] Add CI cases for testing upgrade from v2.0.1 release These are manual trigger jobs, but should be run if any PR impacts upgrades. --- .gitlab-ci.yml | 37 +++++++++++++++++++++++++-- tests/testcases/030_check-network.yml | 10 ++++++-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 72f8de7e9..2d281cd72 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,8 +48,10 @@ before_script: GS_ACCESS_KEY_ID: $GS_KEY GS_SECRET_ACCESS_KEY: $GS_SECRET ANSIBLE_KEEP_REMOTE_FILES: "1" + ANSIBLE_CONFIG: ./tests/ansible.cfg BOOTSTRAP_OS: none IDEMPOT_CHECK: "false" + UPGRADE_TEST: "false" RESOLVCONF_MODE: docker_dns LOG_LEVEL: "-vv" ETCD_DEPLOYMENT: "docker" @@ -73,14 +75,12 @@ before_script: - pip install apache-libcloud==0.20.1 - pip install boto==2.9.0 - mkdir -p /.ssh - - cp tests/ansible.cfg . - mkdir -p $HOME/.ssh - echo $PRIVATE_KEY | base64 -d > $HOME/.ssh/id_rsa - echo $GCE_PEM_FILE | base64 -d > $HOME/.ssh/gce - echo $GCE_CREDENTIALS > $HOME/.ssh/gce.json - chmod 400 $HOME/.ssh/id_rsa - ansible-playbook --version - - cp tests/ansible.cfg . - export PYPATH=$([ $BOOTSTRAP_OS = none ] && echo /usr/bin/python || echo /opt/bin/python) script: - pwd @@ -99,6 +99,12 @@ before_script: -e mode=${CLUSTER_MODE} -e test_id=${TEST_ID} + # Check out latest tag if testing upgrade + # Uncomment when gitlab kargo repo has tags + #- test "${UPGRADE_TEST}" = "true" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1)) + - test "${UPGRADE_TEST}" = "true" && git checkout 031cf565ec3ccd3ebbe80eeef3454c3780e5c598 && pip install ansible==2.2.0 + + # Create cluster - > ansible-playbook -i inventory/inventory.ini -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER @@ -120,6 +126,31 @@ before_script: -e vault_deployment_type=${VAULT_DEPLOYMENT} cluster.yml + # Repeat deployment if testing upgrade + #FIXME(mattymo): repeat "Create cluster" above without duplicating code + - > + if [ "${UPGRADE_TEST}" = "true" ]; then + pip install ansible==2.2.1.0; + git checkout "${CI_BUILD_REF}"; + ansible-playbook -i inventory/inventory.ini -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER + ${SSH_ARGS} + ${LOG_LEVEL} + -e ansible_python_interpreter=${PYPATH} + -e ansible_ssh_user=${SSH_USER} + -e bootstrap_os=${BOOTSTRAP_OS} + -e cloud_provider=gce + -e deploy_netchecker=true + -e download_localhost=true + -e download_run_once=true + -e etcd_deployment_type=${ETCD_DEPLOYMENT} + -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} + -e kubelet_deployment_type=${KUBELET_DEPLOYMENT} + -e local_release_dir=${PWD}/downloads + -e resolvconf_mode=${RESOLVCONF_MODE} + -e weave_cpu_requests=${WEAVE_CPU_LIMIT} + -e weave_cpu_limit=${WEAVE_CPU_LIMIT} + cluster.yml; + fi # Tests Cases ## Test Master API @@ -222,6 +253,7 @@ before_script: KUBE_NETWORK_PLUGIN: canal CLOUD_IMAGE: debian-8-kubespray CLOUD_REGION: us-east1-b + UPGRADE_TEST: "true" CLUSTER_MODE: ha .rhel7_weave_variables: &rhel7_weave_variables @@ -229,6 +261,7 @@ before_script: KUBE_NETWORK_PLUGIN: weave CLOUD_IMAGE: rhel-7 CLOUD_REGION: europe-west1-b + UPGRADE_TEST: "true" CLUSTER_MODE: default .centos7_flannel_variables: ¢os7_flannel_variables diff --git a/tests/testcases/030_check-network.yml b/tests/testcases/030_check-network.yml index 0932d63ff..ac5e4e4aa 100644 --- a/tests/testcases/030_check-network.yml +++ b/tests/testcases/030_check-network.yml @@ -12,6 +12,12 @@ bin_dir: "/usr/local/bin" when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + - name: Check kubectl output + shell: "{{bin_dir}}/kubectl get pods --all-namespaces -owide" + register: get_pods + + - debug: msg="{{get_pods.stdout}}" + - name: Get pod names shell: "{{bin_dir}}/kubectl get pods -o json" register: pods @@ -19,12 +25,12 @@ - name: Get hostnet pods command: "{{bin_dir}}/kubectl get pods -o - jsonpath='{range .items[?(.spec.hostNetwork)]}{.metadata.name} {.status.podIP} {end}'" + jsonpath='{range .items[?(.spec.hostNetwork)]}{.metadata.name} {.status.podIP} {.status.containerStatuses} {end}'" register: hostnet_pods - name: Get running pods command: "{{bin_dir}}/kubectl get pods -o - jsonpath='{range .items[?(.status.phase==\"Running\")]}{.metadata.name} {.status.podIP} {end}'" + jsonpath='{range .items[?(.status.phase==\"Running\")]}{.metadata.name} {.status.podIP} {.status.containerStatuses} {end}'" register: running_pods - set_fact: From ccd865c56428ac346600fb766002876e2ea7c61a Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 10 Feb 2017 13:27:41 +0300 Subject: [PATCH 2/2] fixup upgrades for canal and weave --- roles/kubernetes-apps/network_plugin/canal/tasks/main.yaml | 2 +- roles/network_plugin/weave/tasks/pre-upgrade.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/roles/kubernetes-apps/network_plugin/canal/tasks/main.yaml b/roles/kubernetes-apps/network_plugin/canal/tasks/main.yaml index 2133e85fb..f5ffc4393 100644 --- a/roles/kubernetes-apps/network_plugin/canal/tasks/main.yaml +++ b/roles/kubernetes-apps/network_plugin/canal/tasks/main.yaml @@ -13,7 +13,7 @@ kube: name: "canal-node" kubectl: "{{ bin_dir }}/kubectl" - filename: "{{ kube_config_dir }}/canal-node.yml" + filename: "{{ kube_config_dir }}/canal-node.yaml" resource: "ds" namespace: "{{system_namespace}}" state: absent diff --git a/roles/network_plugin/weave/tasks/pre-upgrade.yml b/roles/network_plugin/weave/tasks/pre-upgrade.yml index 42a031f45..0b10a7551 100644 --- a/roles/network_plugin/weave/tasks/pre-upgrade.yml +++ b/roles/network_plugin/weave/tasks/pre-upgrade.yml @@ -37,3 +37,5 @@ - name: Weave pre-upgrade | Purge legacy weave docker containers shell: "docker ps -af 'name=^/weave.*' -q | xargs --no-run-if-empty docker rm -f" + retries: 3 + failed_when: false