From 9c1701f2aae3e2d8f411f18d5ee98a0864777f6e Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Mon, 13 Feb 2017 14:13:28 +0300 Subject: [PATCH] Add synthetic scale deployment mode New deploy modes: scale, ha-scale, separate-scale Creates 200 fake hosts for deployment with fake hostvars. Useful for testing certificate generation and propagation to other master nodes. Updated test cases descriptions. --- .gitignore | 1 + .gitlab-ci.yml | 19 +++++++++---- docs/test_cases.md | 41 +++++++++++++++++++--------- roles/etcd/tasks/check_certs.yml | 1 + tests/cloud_playbooks/create-gce.yml | 14 +++++++++- tests/templates/fake_hosts.yml.j2 | 3 ++ tests/templates/inventory-gce.j2 | 16 ++++++++--- 7 files changed, 71 insertions(+), 24 deletions(-) create mode 100644 tests/templates/fake_hosts.yml.j2 diff --git a/.gitignore b/.gitignore index 074dd431d..b9b150a3d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .vagrant *.retry inventory/vagrant_ansible_inventory +inventory/group_vars/fake_hosts.yml temp .idea .tox diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c6630306..d326aecbd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -124,6 +124,7 @@ before_script: -e local_release_dir=${PWD}/downloads -e resolvconf_mode=${RESOLVCONF_MODE} -e vault_deployment_type=${VAULT_DEPLOYMENT} + --limit "all:!fake_hosts" cluster.yml # Repeat deployment if testing upgrade @@ -150,18 +151,19 @@ before_script: -e resolvconf_mode=${RESOLVCONF_MODE} -e weave_cpu_requests=${WEAVE_CPU_LIMIT} -e weave_cpu_limit=${WEAVE_CPU_LIMIT} + --limit "all:!fake_hosts" $PLAYBOOK; fi # Tests Cases ## Test Master API - - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/010_check-apiserver.yml $LOG_LEVEL + - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL ## Ping the between 2 pod - - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL + - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $LOG_LEVEL ## Advanced DNS checks - - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/040_check-network-adv.yml $LOG_LEVEL + - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml $LOG_LEVEL ## Idempotency checks 1/5 (repeat deployment) - > @@ -178,6 +180,7 @@ before_script: -e local_release_dir=${PWD}/downloads -e etcd_deployment_type=${ETCD_DEPLOYMENT} -e kubelet_deployment_type=${KUBELET_DEPLOYMENT} + --limit "all:!fake_hosts" cluster.yml; fi @@ -186,6 +189,7 @@ before_script: if [ "${IDEMPOT_CHECK}" = "true" ]; then ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root + --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml $LOG_LEVEL; fi @@ -197,7 +201,8 @@ before_script: --private-key=${HOME}/.ssh/id_rsa -e bootstrap_os=${BOOTSTRAP_OS} -e ansible_python_interpreter=${PYPATH} - -e reset_confirmation=yes + -e reset_confirmation=yes + --limit "all:!fake_hosts" reset.yml; fi @@ -216,6 +221,7 @@ before_script: -e local_release_dir=${PWD}/downloads -e etcd_deployment_type=${ETCD_DEPLOYMENT} -e kubelet_deployment_type=${KUBELET_DEPLOYMENT} + --limit "all:!fake_hosts" cluster.yml; fi @@ -224,6 +230,7 @@ before_script: if [ "${IDEMPOT_CHECK}" = "true" ]; then ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root + --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml $LOG_LEVEL; fi @@ -244,7 +251,7 @@ before_script: .coreos_calico_sep_variables: &coreos_calico_sep_variables # stage: deploy-gce-part1 KUBE_NETWORK_PLUGIN: calico - CLOUD_IMAGE: coreos-stable-1235-6-0-v20170111 + CLOUD_IMAGE: coreos-stable-1235-6-0-v20170111 CLOUD_REGION: us-west1-b CLUSTER_MODE: separate BOOTSTRAP_OS: coreos @@ -310,7 +317,7 @@ before_script: KUBE_NETWORK_PLUGIN: calico CLOUD_IMAGE: centos-7 CLOUD_REGION: europe-west1-b - CLUSTER_MODE: ha + CLUSTER_MODE: ha-scale IDEMPOT_CHECK: "true" .coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables diff --git a/docs/test_cases.md b/docs/test_cases.md index 20b052861..5e232b63a 100644 --- a/docs/test_cases.md +++ b/docs/test_cases.md @@ -4,25 +4,40 @@ Travis CI test matrix GCE instances ------------- -Here is the test matrix for the Travis CI gates: +Here is the test matrix for the CI gates: | Network plugin| OS type| GCE region| Nodes layout| |-------------------------|-------------------------|-------------------------|-------------------------| -| canal| debian-8-kubespray| asia-east1-a| ha| +| canal| debian-8-kubespray| asia-east1-a| ha-scale| | calico| debian-8-kubespray| europe-west1-c| default| | flannel| centos-7| asia-northeast1-c| default| | calico| centos-7| us-central1-b| ha| | weave| rhel-7| us-east1-c| default| -| canal| coreos-stable| us-west1-b| default| +| canal| coreos-stable| us-west1-b| ha-scale| | canal| rhel-7| asia-northeast1-b| separate| | weave| ubuntu-1604-xenial| europe-west1-d| separate| | calico| coreos-stable| us-central1-f| separate| -Where the nodes layout `default` is a non-HA two nodes setup with the separate `kube-node` -and the `etcd` group merged with the `kube-master`. The `separate` layout is when -there is only node of each type, which is a kube master, compute and etcd cluster member. -And the `ha` layout stands for a two etcd nodes, two masters and a single worker node, -partially intersecting though. + +Node Layouts +------------ + +There are four node layout types: `default`, `separate`, `ha`, and `scale`. + + +`default` is a non-HA two nodes setup with one separate `kube-node` +and the `etcd` group merged with the `kube-master`. + +`separate` layout is when there is only node of each type, which includes + a kube-master, kube-node, and etcd cluster member. + +`ha` layout consists of two etcd nodes, two masters and a single worker node, +with role intersection. + +`scale` layout can be combined with above layouts. It includes 200 fake hosts +in the Ansible inventory. This helps test TLS certificate generation at scale +to prevent regressions and profile certain long-running tasks. These nodes are +never actually deployed, but certificates are generated for them. Note, the canal network plugin deploys flannel as well plus calico policy controller. @@ -40,15 +55,15 @@ GCE instances | Stage| Network plugin| OS type| GCE region| Nodes layout |--------------------|--------------------|--------------------|--------------------|--------------------| -| part1| calico| coreos-stable| us-west1-b| separated| +| part1| calico| coreos-stable| us-west1-b| separate| | part1| canal| debian-8-kubespray| us-east1-b| ha| -| part1| weave| rhel-7| europe-west1-b| default| +| part1| weave| rhel-7| europe-west1-b| scale| | part2| flannel| centos-7| us-west1-a| default| | part2| calico| debian-8-kubespray| us-central1-b| default| | part2| canal| coreos-stable| us-east1-b| default| -| special| canal| rhel-7| us-east1-b| separated| -| special| weave| ubuntu-1604-xenial| us-central1-b| separated| -| special| calico| centos-7| europe-west1-b| ha| +| special| canal| rhel-7| us-east1-b| separate| +| special| weave| ubuntu-1604-xenial| us-central1-b| scale| +| special| calico| centos-7| europe-west1-b| ha-scale| | special| weave| coreos-alpha| us-west1-a| ha| The "Stage" means a build step of the build pipeline. The steps are ordered as `part1->part2->special`. diff --git a/roles/etcd/tasks/check_certs.yml b/roles/etcd/tasks/check_certs.yml index 270c0abfc..bc14e255f 100644 --- a/roles/etcd/tasks/check_certs.yml +++ b/roles/etcd/tasks/check_certs.yml @@ -2,6 +2,7 @@ - name: "Check_certs | check if all certs have already been generated on first master" stat: path: "{{ etcd_cert_dir }}/{{ item }}" + get_md5: no delegate_to: "{{groups['etcd'][0]}}" register: etcdcert_master run_once: true diff --git a/tests/cloud_playbooks/create-gce.yml b/tests/cloud_playbooks/create-gce.yml index 4ae32b787..d1e7c011e 100644 --- a/tests/cloud_playbooks/create-gce.yml +++ b/tests/cloud_playbooks/create-gce.yml @@ -13,7 +13,7 @@ - set_fact: instance_names: >- - {%- if mode in ['separate', 'ha'] -%} + {%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale'] -%} k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3 {%- else -%} k8s-{{test_name}}-1,k8s-{{test_name}}-2 @@ -39,6 +39,18 @@ src: ../templates/inventory-gce.j2 dest: "{{ inventory_path }}" + - name: Make group_vars directory + file: + path: "{{ inventory_path|dirname }}/group_vars" + state: directory + when: mode in ['scale', 'separate-scale', 'ha-scale'] + + - name: Template fake hosts group vars + template: + src: ../templates/fake_hosts.yml.j2 + dest: "{{ inventory_path|dirname }}/group_vars/fake_hosts.yml" + when: mode in ['scale', 'separate-scale', 'ha-scale'] + - name: Wait for SSH to come up wait_for: host={{item.public_ip}} port=22 delay=10 timeout=180 state=started with_items: "{{gce.instance_data}}" diff --git a/tests/templates/fake_hosts.yml.j2 b/tests/templates/fake_hosts.yml.j2 new file mode 100644 index 000000000..673109213 --- /dev/null +++ b/tests/templates/fake_hosts.yml.j2 @@ -0,0 +1,3 @@ +ansible_default_ipv4: + address: 255.255.255.255 +ansible_hostname: "{{ '{{' }}inventory_hostname}}" diff --git a/tests/templates/inventory-gce.j2 b/tests/templates/inventory-gce.j2 index f5326229c..8274fae8c 100644 --- a/tests/templates/inventory-gce.j2 +++ b/tests/templates/inventory-gce.j2 @@ -2,12 +2,11 @@ {% set node2 = gce.instance_data[1].name %} {{node1}} ansible_ssh_host={{gce.instance_data[0].public_ip}} {{node2}} ansible_ssh_host={{gce.instance_data[1].public_ip}} -{% if mode is defined and mode in ["separate", "ha"] %} +{% if mode is defined and mode in ["separate", "separate-scale"] %} {% set node3 = gce.instance_data[2].name %} {{node3}} ansible_ssh_host={{gce.instance_data[2].public_ip}} {% endif %} - -{% if mode is defined and mode == "separate" %} +{% if mode is defined and mode in ["separate", "separate-scale"] %} [kube-master] {{node1}} @@ -19,7 +18,7 @@ [vault] {{node3}} -{% elif mode is defined and mode == "ha" %} +{% elif mode is defined and mode in ["ha", "ha-scale"] %} [kube-master] {{node1}} {{node2}} @@ -51,3 +50,12 @@ [k8s-cluster:children] kube-node kube-master + +{% if mode is defined and mode in ["scale", "separate-scale", "ha-scale"] %} +[fake_hosts] +fake_scale_host[1:200] + +[kube-node:children] +fake_hosts +{% endif %} +