diff --git a/tests/Makefile b/tests/Makefile index 30442fb25..b92ac7dc0 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -12,6 +12,10 @@ init-gce: $(HOME)/.ssh/id_rsa init-do: $(HOME)/.ssh/id_rsa echo $(DO_PRIVATE_KEY) | base64 -d > $(HOME)/.ssh/id_rsa +init-packet: + echo $(PACKET_VM_SSH_PRIVATE_KEY) | base64 -d > $(HOME)/.ssh/id_rsa + chmod 400 $(HOME)/.ssh/id_rsa + create-gce: init-gce ansible-playbook cloud_playbooks/create-gce.yml -i local_inventory/hosts.cfg -c local \ $(LOG_LEVEL) \ @@ -41,7 +45,6 @@ create-do: init-do -e inventory_path=$(INVENTORY) \ -e test_id=${TEST_ID} - delete-do: ansible-playbook -i $(INVENTORY) cloud_playbooks/create-do.yml -c local \ $(LOG_LEVEL) \ @@ -49,3 +52,17 @@ delete-do: -e state=absent \ -e test_id=${TEST_ID} \ -e inventory_path=$(INVENTORY) + +create-packet: init-packet + ansible-playbook cloud_playbooks/create-packet.yml -c local \ + $(LOG_LEVEL) \ + -e @"files/${CI_JOB_NAME}.yml" \ + -e test_id=$(TEST_ID) \ + -e inventory_path=$(INVENTORY) + +delete-packet: + ansible-playbook cloud_playbooks/delete-packet.yml -c local \ + $(LOG_LEVEL) \ + -e @"files/${CI_JOB_NAME}.yml" \ + -e test_id=$(TEST_ID) \ + -e inventory_path=$(INVENTORY) diff --git a/tests/cloud_playbooks/create-packet.yml b/tests/cloud_playbooks/create-packet.yml new file mode 100644 index 000000000..0136ab38d --- /dev/null +++ b/tests/cloud_playbooks/create-packet.yml @@ -0,0 +1,10 @@ +--- + +- hosts: localhost + gather_facts: no + become: true + vars: + ci_job_name: "{{ lookup('env', 'CI_JOB_NAME') }}" + test_name: "{{ test_id | regex_replace('\\.', '-') }}" + roles: + - { role: packet-ci, vm_cleanup: false } diff --git a/tests/cloud_playbooks/delete-packet.yml b/tests/cloud_playbooks/delete-packet.yml new file mode 100644 index 000000000..3895263ce --- /dev/null +++ b/tests/cloud_playbooks/delete-packet.yml @@ -0,0 +1,10 @@ +--- + +- hosts: localhost + gather_facts: no + become: true + vars: + ci_job_name: "{{ lookup('env', 'CI_JOB_NAME') }}" + test_name: "{{ test_id | regex_replace('\\.', '-') }}" + roles: + - { role: packet-ci, vm_cleanup: true } diff --git a/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml b/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml new file mode 100644 index 000000000..f3a8fe56c --- /dev/null +++ b/tests/cloud_playbooks/roles/packet-ci/defaults/main.yml @@ -0,0 +1,23 @@ +--- + +# VM sizing +vm_cpu_cores: 2 +vm_cpu_sockets: 1 +vm_cpu_threads: 2 +vm_memory: 4096Mi + +# Default path for inventory +inventory_path: "/tmp/{{ test_name }}/inventory" + +# Deployment mode +mode: aio + +# Cloud init config for each os type +cloud_init: + centos-7: "I2Nsb3VkLWNvbmZpZwpzeXN0ZW1faW5mbzoKICBkaXN0cm86IHJoZWwKdXNlcnM6CiAtIG5hbWU6IGt1YmVzcHJheQogICBncm91cHM6IHdoZWVsCiAgIHN1ZG86ICdBTEw9KEFMTCkgTk9QQVNTV0Q6QUxMJwogICBzaGVsbDogL2Jpbi9iYXNoCiAgIGxvY2tfcGFzc3dkOiBGYWxzZQogICBob21lOiAvaG9tZS9rdWJlc3ByYXkKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1Cgo=" + debian-9: "I2Nsb3VkLWNvbmZpZwp1c2VyczoKIC0gbmFtZToga3ViZXNwcmF5CiAgIHN1ZG86IEFMTD0oQUxMKSBOT1BBU1NXRDpBTEwKICAgc2hlbGw6IC9iaW4vYmFzaAogICBob21lOiAvaG9tZS91YnVudHUKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1" + fedora-28: "I2Nsb3VkLWNvbmZpZwp1c2VyczoKIC0gbmFtZToga3ViZXNwcmF5CiAgIHN1ZG86IEFMTD0oQUxMKSBOT1BBU1NXRDpBTEwKICAgc2hlbGw6IC9iaW4vYmFzaAogICBob21lOiAvaG9tZS91YnVudHUKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1" + opensuse-leap-15: "I2Nsb3VkLWNvbmZpZwpzeXN0ZW1faW5mbzoKICBkaXN0cm86IHJoZWwKdXNlcnM6CiAtIG5hbWU6IGt1YmVzcHJheQogICBncm91cHM6IHdoZWVsCiAgIHN1ZG86ICdBTEw9KEFMTCkgTk9QQVNTV0Q6QUxMJwogICBzaGVsbDogL2Jpbi9iYXNoCiAgIGxvY2tfcGFzc3dkOiBGYWxzZQogICBob21lOiAvaG9tZS9rdWJlc3ByYXkKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1Cgo=" + rhel-server-7: "I2Nsb3VkLWNvbmZpZwpzeXN0ZW1faW5mbzoKICBkaXN0cm86IHJoZWwKdXNlcnM6CiAtIG5hbWU6IGt1YmVzcHJheQogICBncm91cHM6IHdoZWVsCiAgIHN1ZG86ICdBTEw9KEFMTCkgTk9QQVNTV0Q6QUxMJwogICBzaGVsbDogL2Jpbi9iYXNoCiAgIGxvY2tfcGFzc3dkOiBGYWxzZQogICBob21lOiAvaG9tZS9rdWJlc3ByYXkKICAgc3NoX2F1dGhvcml6ZWRfa2V5czoKICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1Cgo=" + ubuntu-1604: "I2Nsb3VkLWNvbmZpZwogdXNlcnM6CiAgLSBuYW1lOiBrdWJlc3ByYXkKICAgIHN1ZG86IEFMTD0oQUxMKSBOT1BBU1NXRDpBTEwKICAgIHNoZWxsOiAvYmluL2Jhc2gKICAgIGxvY2tfcGFzc3dkOiBGYWxzZQogICAgaG9tZTogL2hvbWUva3ViZXNwcmF5CiAgICBzc2hfYXV0aG9yaXplZF9rZXlzOgogICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1" + ubuntu-1804: "I2Nsb3VkLWNvbmZpZwogdXNlcnM6CiAgLSBuYW1lOiBrdWJlc3ByYXkKICAgIHN1ZG86IEFMTD0oQUxMKSBOT1BBU1NXRDpBTEwKICAgIHNoZWxsOiAvYmluL2Jhc2gKICAgIGxvY2tfcGFzc3dkOiBGYWxzZQogICAgaG9tZTogL2hvbWUva3ViZXNwcmF5CiAgICBzc2hfYXV0aG9yaXplZF9rZXlzOgogICAgICAtIHNzaC1yc2EgQUFBQUIzTnphQzF5YzJFQUFBQURBUUFCQUFBQkFRQ2FuVGkvZUt4MCt0SFlKQWVEaHErc0ZTMk9iVVAxL0k2OWY3aVYzVXRrS2xUMjBKZlcxZjZGZVh0LzA0VmYyN1dRcStOcXM2dkdCcUQ5UVhTWXVmK3QwL3M3RVBMalRlaTltZTFtcHFyK3VUZStLRHRUUDM5cGZEMy9lVkNhZUI3MjZHUDJGa2FEMEZ6cG1FYjY2TzNOcWh4T1E5Nkd4LzlYVHV3L0szbGxqNE9WRDZHcmpSM0I3YzRYdEVCc1pjWnBwTUovb0gxbUd5R1hkaDMxbVdRU3FBUk8vUDhVOEd3dDArSEdwVXdoL2hkeTN0K1NZb1RCMkd3VmIwem95Vnd0VnZmRFF6c204ZnEzYXY0S3ZlejhrWXVOREp2MDV4NGx2VVpnUjE1WkRSWHNBbmRoUXlxb1hkQ0xBZTArZWFLWHE5QmtXeEtGYjloUGUwQVVqamE1" diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml new file mode 100644 index 000000000..a5cb76baa --- /dev/null +++ b/tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml @@ -0,0 +1,58 @@ +--- + +- name: "Create CI namespace {{ test_name }} for test vms" + k8s: + name: "{{ test_name }}" + kind: Namespace + state: present + register: create_ci_ns + failed_when: create_ci_ns.changed == false + +- name: "Create temp dir /tmp/{{ test_name }} for CI files" + file: + path: "/tmp/{{ test_name }}" + state: directory + +- name: Template vm files for CI job + template: + src: "vm.yml.j2" + dest: "/tmp/{{ test_name }}/instance-{{ vm_id }}.yml" + loop: "{{ range(0, vm_count|int, 1) | list }}" + loop_control: + index_var: vm_id + +- name: Start vms for CI job + k8s: + state: present + src: "/tmp/{{ test_name }}/instance-{{ vm_id }}.yml" + loop: "{{ range(0, vm_count|int, 1) | list }}" + loop_control: + index_var: vm_id + +- name: Wait for vms to have ipaddress assigned + shell: "kubectl get vmis -n {{ test_name }} instance-{{ vm_id }} -o json | jq '.status.interfaces[].ipAddress' | tr -d '\"'" + register: vm_ips + loop: "{{ range(0, vm_count|int, 1) | list }}" + loop_control: + index_var: vm_id + retries: 10 + delay: 15 + until: + - vm_ips.stdout | ipaddr + +- name: Wait for SSH to become available on vms + wait_for: + host: "{{ item.stdout }}" + port: 22 + delay: 30 + timeout: 240 + state: started + with_items: + - "{{ vm_ips.results }}" + +- name: "Create inventory for CI test in file /tmp/{{ test_name }}/inventory" + template: + src: "inventory.j2" + dest: "{{ inventory_path }}" + vars: + vms: "{{ vm_ips }}" diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml new file mode 100644 index 000000000..dc66e2db7 --- /dev/null +++ b/tests/cloud_playbooks/roles/packet-ci/tasks/delete-vms.yml @@ -0,0 +1,29 @@ +--- + +- name: Check if temp directory for {{ test_name }} exists + stat: + path: "/tmp/{{ test_name }}" + register: temp_dir_details + +- name: "Cleanup temp directory for {{ test_name }}" + file: + path: "/tmp/{{ test_name }}" + state: absent + +- name: "Cleanup namespace for {{ test_name }}" + k8s: + kind: Namespace + state: absent + name: "{{ test_name }}" + +- name: Wait for namespace {{ test_name }} to be fully deleted + shell: kubectl get ns {{ test_name }} + register: delete_namespace + failed_when: + - delete_namespace.rc == 0 + changed_when: + - delete_namespace.rc == 0 + retries: 12 + delay: "10" + until: + - delete_namespace.rc != 0 diff --git a/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml b/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml new file mode 100644 index 000000000..6f7d0cdf6 --- /dev/null +++ b/tests/cloud_playbooks/roles/packet-ci/tasks/main.yml @@ -0,0 +1,16 @@ +--- + +- name: "Include custom vars for ci job: {{ ci_job_name }}" + include_vars: "../files/{{ ci_job_name }}.yml" + +- name: Set VM count needed for CI test_id + set_fact: + vm_count: "{%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale'] -%}{{ 3|int }}{%- elif mode == 'aio' -%}{{ 1|int }}{%- else -%}{{ 2|int }}{%- endif -%}" + +- import_tasks: create-vms.yml + when: + - not vm_cleanup + +- import_tasks: delete-vms.yml + when: + - vm_cleanup | default(false) diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 new file mode 100644 index 000000000..a2f301def --- /dev/null +++ b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 @@ -0,0 +1,54 @@ +[all] +{% for instance in vms.results %} +instance-{{ loop.index }} ansible_ssh_host={{instance.stdout}} +{% endfor %} + +{% if mode is defined and mode in ["separate", "separate-scale"] %} +[kube-master] +instance-1 + +[kube-node] +instance-2 + +[etcd] +instance-3 +{% elif mode is defined and mode in ["ha", "ha-scale"] %} +[kube-master] +instance-1 +instance-2 + +[kube-node] +instance-3 + +[etcd] +instance-1 +instance-2 +instance-3 +{% elif mode == "default" %} +[kube-master] +instance-1 + +[kube-node] +instance-2 + +[etcd] +instance-1 +{% elif mode == "aio" %} +[kube-master] +instance-1 + +[kube-node] +instance-1 + +[etcd] +instance-1 + +[vault] +instance-1 +{% endif %} + +[k8s-cluster:children] +kube-node +kube-master + +[fake_hosts] diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 new file mode 100644 index 000000000..52140219a --- /dev/null +++ b/tests/cloud_playbooks/roles/packet-ci/templates/vm.yml.j2 @@ -0,0 +1,49 @@ +--- +apiVersion: kubevirt.io/v1alpha3 +kind: VirtualMachine +metadata: + name: "instance-{{ vm_id }}" + namespace: "{{ test_name }}" + labels: + kubevirt.io/os: {{ cloud_image }} +spec: + running: true + template: + metadata: + labels: + kubevirt.io/size: small + kubevirt.io/domain: "{{ test_name }}" + spec: + domain: + devices: + blockMultiQueue: true + disks: + - disk: + bus: virtio + name: containervolume + cache: writethrough + - disk: + bus: virtio + name: cloudinitvolume + interfaces: + - name: default + bridge: {} + cpu: + cores: {{ vm_cpu_cores }} + sockets: {{ vm_cpu_sockets }} + threads: {{ vm_cpu_cores }} + resources: + requests: + memory: {{ vm_memory }} + cpu: {{ vm_cpu_cores }} + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 0 + volumes: + - name: containervolume + containerDisk: + image: quay.io/kubespray/vm-{{ cloud_image }} + - name: cloudinitvolume + cloudInitNoCloud: + userDataBase64: {{ cloud_init[cloud_image] }} diff --git a/tests/requirements.txt b/tests/requirements.txt index 9a0a959d3..2530c12bd 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,3 +6,4 @@ tox dopy PyCrypto ansible-lint==4.1.0 +openshift