Add complete test integration
This commit is contained in:
ant31
parent
09851621de
commit
caa2555b1d
140
.travis.yml
140
.travis.yml
|
|
@ -1,38 +1,128 @@
|
|||
sudo: required
|
||||
dist: trusty
|
||||
language: python
|
||||
python: "2.7"
|
||||
sudo: false
|
||||
|
||||
addons:
|
||||
hosts:
|
||||
- node1
|
||||
git:
|
||||
depth: 5
|
||||
|
||||
env:
|
||||
- SITE=cluster.yml ANSIBLE_VERSION=2.0.0
|
||||
global:
|
||||
GCE_USER=travis
|
||||
SSH_USER=$GCE_USER
|
||||
TEST_ID=$TRAVIS_JOB_NUMBER
|
||||
CONTAINER_ENGINE=docker
|
||||
PRIVATE_KEY=$GCE_PRIVATE_KEY
|
||||
ANSIBLE_KEEP_REMOTE_FILES=1
|
||||
matrix:
|
||||
# Debian Jessie
|
||||
- >-
|
||||
KUBE_NETWORK_PLUGIN=flannel
|
||||
CLOUD_IMAGE=debian-8
|
||||
CLOUD_REGION=europe-west1-b
|
||||
- >-
|
||||
KUBE_NETWORK_PLUGIN=calico
|
||||
CLOUD_IMAGE=debian-8
|
||||
CLOUD_REGION=europe-west1-b
|
||||
|
||||
install:
|
||||
# Centos 7
|
||||
- >-
|
||||
KUBE_NETWORK_PLUGIN=flannel
|
||||
CLOUD_IMAGE=centos-7-sudo
|
||||
CLOUD_REGION=us-central1-c
|
||||
|
||||
# - >-
|
||||
# KUBE_NETWORK_PLUGIN=calico
|
||||
# CLOUD_IMAGE=centos-7-sudo
|
||||
# CLOUD_REGION=us-central1-c
|
||||
|
||||
|
||||
# # Redhat 7
|
||||
# - >-
|
||||
# KUBE_NETWORK_PLUGIN=flannel
|
||||
# CLOUD_IMAGE=rhel-7-sudo
|
||||
# CLOUD_REGION=us-east1-d
|
||||
# SSH_ARGS='--ssh-extra-args="-t"'
|
||||
# - >-
|
||||
# KUBE_NETWORK_PLUGIN=calico
|
||||
# CLOUD_IMAGE=rhel-7-sudo
|
||||
# CLOUD_REGION=us-east1-d
|
||||
|
||||
# Ubuntu 14.04
|
||||
- >-
|
||||
KUBE_NETWORK_PLUGIN=flannel
|
||||
CLOUD_IMAGE=ubuntu-1404-trusty
|
||||
CLOUD_REGION=europe-west1-c
|
||||
- >-
|
||||
KUBE_NETWORK_PLUGIN=calico
|
||||
CLOUD_IMAGE=ubuntu-1404-trusty
|
||||
CLOUD_REGION=europe-west1-c
|
||||
|
||||
# Ubuntu 15.10
|
||||
- >-
|
||||
KUBE_NETWORK_PLUGIN=flannel
|
||||
CLOUD_IMAGE=ubuntu-1510-wily
|
||||
CLOUD_REGION=us-central1-a
|
||||
- >-
|
||||
KUBE_NETWORK_PLUGIN=calico
|
||||
CLOUD_IMAGE=ubuntu-1510-wily
|
||||
CLOUD_REGION=us-central1-a
|
||||
|
||||
|
||||
matrix:
|
||||
allow_failures:
|
||||
- env: KUBE_NETWORK_PLUGIN=flannel CLOUD_IMAGE=centos-7-sudo CLOUD_REGION=us-central1-c
|
||||
|
||||
before_install:
|
||||
# Install Ansible.
|
||||
- sudo -H pip install ansible==${ANSIBLE_VERSION}
|
||||
- sudo -H pip install netaddr
|
||||
- pip install --user boto -U
|
||||
- pip install --user ansible
|
||||
- pip install --user netaddr
|
||||
- pip install --user apache-libcloud
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- $HOME/releases
|
||||
- directories:
|
||||
- $HOME/.cache/pip
|
||||
- $HOME/.local
|
||||
|
||||
before_script:
|
||||
- export PATH=$PATH:/usr/local/bin
|
||||
- echo "RUN $TRAVIS_JOB_NUMBER $KUBE_NETWORK_PLUGIN $CONTAINER_ENGINE "
|
||||
- mkdir -p $HOME/.ssh
|
||||
- echo $PRIVATE_KEY | base64 -d > $HOME/.ssh/id_rsa
|
||||
- echo $GCE_PEM_FILE | base64 -d > $HOME/.ssh/gce
|
||||
- chmod 400 $HOME/.ssh/id_rsa
|
||||
- chmod 755 $HOME/.local/bin/ansible-playbook
|
||||
- $HOME/.local/bin/ansible-playbook --version
|
||||
- cp tests/ansible.cfg .
|
||||
# - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}' $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml"
|
||||
|
||||
script:
|
||||
# Check the role/playbook's syntax.
|
||||
- "sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --syntax-check"
|
||||
|
||||
# Run the role/playbook with ansible-playbook.
|
||||
- "sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --connection=local"
|
||||
|
||||
# Run the role/playbook again, checking to make sure it's idempotent.
|
||||
- >
|
||||
sudo -H ansible-playbook -i inventory/local-tests.cfg $SITE --connection=local
|
||||
| tee /dev/stderr | grep -q 'changed=0.*failed=0'
|
||||
&& (echo 'Idempotence test: pass' && exit 0)
|
||||
|| (echo 'Idempotence test: fail' && exit 1)
|
||||
$HOME/.local/bin/ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts -c local $LOG_LEVEL
|
||||
-e test_id=${TEST_ID}
|
||||
-e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
|
||||
-e gce_project_id=${GCE_PROJECT_ID}
|
||||
-e gce_service_account_email=${GCE_ACCOUNT}
|
||||
-e gce_pem_file=${HOME}/.ssh/gce
|
||||
-e cloud_image=${CLOUD_IMAGE}
|
||||
-e inventory_path=${PWD}/inventory/inventory.ini
|
||||
-e cloud_region=${CLOUD_REGION}
|
||||
|
||||
# Create cluster
|
||||
- "$HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}' $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} cluster.yml"
|
||||
# Tests Cases
|
||||
## Test Master API
|
||||
- $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/testcases/010_check-apiserver.yml $LOG_LEVEL
|
||||
## Create a POD
|
||||
- $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL
|
||||
## Ping the between 2 pod
|
||||
- $HOME/.local/bin/ansible-playbook -i setup-kubernetes/inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
|
||||
|
||||
after_script:
|
||||
- >
|
||||
$HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local $LOG_LEVEL
|
||||
-e test_id=${TEST_ID}
|
||||
-e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
|
||||
-e gce_project_id=${GCE_PROJECT_ID}
|
||||
-e gce_service_account_email=${GCE_ACCOUNT}
|
||||
-e gce_pem_file=${HOME}/.ssh/gce
|
||||
-e cloud_image=${CLOUD_IMAGE}
|
||||
-e inventory_path=${PWD}/inventory/inventory.ini
|
||||
-e cloud_region=${CLOUD_REGION}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
[ssh_connection]
|
||||
pipelining=True
|
||||
[defaults]
|
||||
host_key_checking=False
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
# k8s-integration-tests
|
||||
|
||||
*Work In Progress*
|
||||
|
||||
## Test environment variables
|
||||
|
||||
### Common
|
||||
|
||||
Variable | Description | Required | Default
|
||||
--------------------- | -------------------------------------- | ---------- | --------
|
||||
`TEST_ID` | A unique execution ID for this test | Yes |
|
||||
`KUBE_NETWORK_PLUGIN` | The network plugin (calico or flannel) | Yes |
|
||||
`PRIVATE_KEY_FILE` | The path to the SSH private key file | No |
|
||||
|
||||
### AWS Tests
|
||||
|
||||
Variable | Description | Required | Default
|
||||
--------------------- | ----------------------------------------------- | ---------- | ---------
|
||||
`AWS_ACCESS_KEY` | The Amazon Access Key ID | Yes |
|
||||
`AWS_SECRET_KEY` | The Amazon Secret Access Key | Yes |
|
||||
`AWS_AMI_ID` | The AMI ID to deploy | Yes |
|
||||
`AWS_KEY_PAIR_NAME` | The name of the EC2 key pair to use | Yes |
|
||||
`AWS_SECURITY_GROUP` | The EC2 Security Group to use | No | default
|
||||
`AWS_REGION` | The EC2 region | No | eu-central-1
|
||||
|
||||
#### Use private ssh key
|
||||
|
||||
##### Key
|
||||
|
||||
```bash
|
||||
openssl pkcs12 -in gce-secure.p12 -passin pass:notasecret -nodes -nocerts | openssl rsa -out gce-secure.pem
|
||||
cat gce-secure.pem |base64 -w0 > GCE_PEM_FILE`
|
||||
```
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
[ssh_connection]
|
||||
pipelining=True
|
||||
[defaults]
|
||||
host_key_checking=False
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
---
|
||||
- hosts: localhost
|
||||
sudo: False
|
||||
gather_facts: False
|
||||
|
||||
tasks:
|
||||
- name: Provision a set of instances
|
||||
ec2:
|
||||
key_name: "{{ aws.key_name }}"
|
||||
aws_access_key: "{{ aws.access_key }}"
|
||||
aws_secret_key: "{{ aws.secret_key }}"
|
||||
region: "{{ aws.region }}"
|
||||
group_id: "{{ aws.group}}"
|
||||
instance_type: "{{ aws.instance_type}}"
|
||||
image: "{{ aws.ami_id }}"
|
||||
wait: true
|
||||
count: "{{ aws.count }}"
|
||||
instance_tags: "{{ aws.tags }}"
|
||||
register: ec2
|
||||
|
||||
- name: Template the inventory
|
||||
template:
|
||||
src: templates/inventory.ini.j2
|
||||
dest: "{{ inventory_path }}"
|
||||
|
||||
- name: Wait until SSH is available
|
||||
local_action:
|
||||
module: wait_for
|
||||
host: "{{ item.public_ip }}"
|
||||
port: 22
|
||||
timeout: 300
|
||||
state: started
|
||||
with_items: ec2.instances
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
---
|
||||
- hosts: localhost
|
||||
sudo: False
|
||||
gather_facts: no
|
||||
vars:
|
||||
cloud_machine_type: g1-small
|
||||
|
||||
tasks:
|
||||
- name: replace_test_id
|
||||
set_fact:
|
||||
test_name: "{{test_id |regex_replace('\\.', '-')}}"
|
||||
|
||||
- name: Create gce instances
|
||||
gce:
|
||||
instance_names: "k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3"
|
||||
machine_type: "{{ cloud_machine_type }}"
|
||||
image: "{{ cloud_image }}"
|
||||
service_account_email: "{{ gce_service_account_email }}"
|
||||
pem_file: "{{ gce_pem_file }}"
|
||||
project_id: "{{ gce_project_id }}"
|
||||
zone: "{{cloud_region}}"
|
||||
metadata: '{"test_id": "{{test_id}}", "network": "{{kube_network_plugin}}"}'
|
||||
tags: "build-{{test_name}},{{kube_network_plugin}}"
|
||||
register: gce
|
||||
|
||||
- name: Template the inventory
|
||||
template:
|
||||
src: ../templates/inventory-gce.j2
|
||||
dest: "{{ inventory_path }}"
|
||||
|
||||
- name: Wait for SSH to come up
|
||||
wait_for: host={{item.public_ip}} port=22 delay=10 timeout=180 state=started
|
||||
with_items: gce.instance_data
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
---
|
||||
- hosts: kube-node
|
||||
sudo: False
|
||||
|
||||
tasks:
|
||||
- name: Gather EC2 facts
|
||||
action: ec2_facts
|
||||
|
||||
- name: Terminate EC2 instances
|
||||
local_action:
|
||||
module: ec2
|
||||
state: absent
|
||||
instance_ids: "{{ ansible_ec2_instance_id }}"
|
||||
region: "{{ ansible_ec2_placement_region }}"
|
||||
wait: True
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
---
|
||||
- hosts: localhost
|
||||
sudo: False
|
||||
gather_facts: no
|
||||
vars:
|
||||
cloud_machine_type: f1-micro
|
||||
|
||||
tasks:
|
||||
- name: replace_test_id
|
||||
set_fact:
|
||||
test_name: "{{test_id |regex_replace('\\.', '-')}}"
|
||||
|
||||
- name: delete gce instances
|
||||
gce:
|
||||
instance_names: "k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3"
|
||||
machine_type: "{{ cloud_machine_type }}"
|
||||
image: "{{ cloud_image }}"
|
||||
service_account_email: "{{ gce_service_account_email }}"
|
||||
pem_file: "{{ gce_pem_file }}"
|
||||
project_id: "{{ gce_project_id }}"
|
||||
zone: "{{cloud_region | default('europe-west1-b')}}"
|
||||
metadata: '{"test_id": "{{test_id}}", "network": "{{kube_network_plugin}}"}'
|
||||
state: 'absent'
|
||||
register: gce
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
aws:
|
||||
key_name: "{{ key_name | default('ansibl8s') }}"
|
||||
access_key: "{{ aws_access_key }}"
|
||||
secret_key: "{{ aws_secret_key }}"
|
||||
region: "{{ aws_region | default('eu-west-1') }}" # default to eu-west-1
|
||||
group: "{{ aws_security_group | default ('default')}}"
|
||||
instance_type: t2.micro
|
||||
ami_id: "{{ aws_ami_id | default('ami-02724d1f') }}" # default to Debian Jessie
|
||||
count: 3
|
||||
tags:
|
||||
test_id: "{{ test_id }}"
|
||||
network_plugin: "{{ kube_network_plugin }}"
|
||||
|
|
@ -0,0 +1 @@
|
|||
localhost ansible_connection=localx
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
#! /bin/bash
|
||||
|
||||
# curl -# -C - -o shebang-unit https://raw.github.com/arpinum-oss/shebang-unit/master/releases/shebang-unit
|
||||
# chmod +x shebang-unit
|
||||
|
||||
now=$(date +"%Y%m%d%H%M%S")
|
||||
mkdir -p ${PWD}/tests-results
|
||||
./shebang-unit --reporters=simple,junit --output-file=${PWD}/tests-results/junit_report-${now}.xml tests
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
#! /bin/bash
|
||||
|
||||
global_setup() {
|
||||
git clone https://github.com/ansibl8s/setup-kubernetes.git setup-kubernetes
|
||||
private_key=""
|
||||
if [ ! -z ${PRIVATE_KEY_FILE} ]
|
||||
then
|
||||
private_key="--private-key=${PRIVATE_KEY_FILE}"
|
||||
fi
|
||||
ansible-playbook create.yml -i hosts -u admin -s \
|
||||
-e test_id=${TEST_ID} \
|
||||
-e kube_network_plugin=${KUBE_NETWORK_PLUGIN} \
|
||||
-e aws_access_key=${AWS_ACCESS_KEY} \
|
||||
-e aws_secret_key=${AWS_SECRET_KEY} \
|
||||
-e aws_ami_id=${AWS_AMI_ID} \
|
||||
-e aws_security_group=${AWS_SECURITY_GROUP} \
|
||||
-e key_name=${AWS_KEY_PAIR_NAME} \
|
||||
-e inventory_path=${PWD}/inventory.ini \
|
||||
-e aws_region=${AWS_REGION}
|
||||
}
|
||||
|
||||
global_teardown() {
|
||||
if [ -f inventory.ini ];
|
||||
then
|
||||
ansible-playbook -i inventory.ini -u admin delete.yml
|
||||
fi
|
||||
rm -rf ${PWD}/setup-kubernetes
|
||||
}
|
||||
|
||||
should_deploy_cluster() {
|
||||
ansible-playbook -i inventory.ini -s ${private_key} -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml
|
||||
|
||||
assertion__status_code_is_success $?
|
||||
}
|
||||
|
||||
should_api_server_respond() {
|
||||
ansible-playbook -i inventory.ini ${private_key} testcases/check-apiserver.yml
|
||||
|
||||
assertion__status_code_is_success $?
|
||||
}
|
||||
|
||||
should_create_pod() {
|
||||
ansible-playbook -i inventory.ini -s ${private_key} testcases/check-create-pod.yml -vv
|
||||
|
||||
assertion__status_code_is_success $?
|
||||
}
|
||||
|
||||
should_pod_be_in_expected_subnet() {
|
||||
ansible-playbook -i inventory.ini -s ${private_key} testcases/check-network.yml -vv
|
||||
|
||||
assertion__status_code_is_success $?
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,20 @@
|
|||
node1 ansible_ssh_host={{gce.instance_data[0].public_ip}}
|
||||
node2 ansible_ssh_host={{gce.instance_data[1].public_ip}}
|
||||
node3 ansible_ssh_host={{gce.instance_data[2].public_ip}}
|
||||
|
||||
[kube-master]
|
||||
node1
|
||||
node2
|
||||
|
||||
[kube-node]
|
||||
node1
|
||||
node2
|
||||
node3
|
||||
|
||||
[etcd]
|
||||
node1
|
||||
node2
|
||||
|
||||
[k8s-cluster:children]
|
||||
kube-node
|
||||
kube-master
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
node1 ansible_ssh_host={{ec2.instances[0].public_ip}} ansible_ssh_user=admin
|
||||
node2 ansible_ssh_host={{ec2.instances[1].public_ip}} ansible_ssh_user=admin
|
||||
node3 ansible_ssh_host={{ec2.instances[2].public_ip}} ansible_ssh_user=admin
|
||||
|
||||
[kube-master]
|
||||
node1
|
||||
node2
|
||||
|
||||
[kube-node]
|
||||
node1
|
||||
node2
|
||||
node3
|
||||
|
||||
[etcd]
|
||||
node1
|
||||
node2
|
||||
|
||||
[k8s-cluster:children]
|
||||
kube-node
|
||||
kube-master
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
---
|
||||
- hosts: kube-master
|
||||
|
||||
tasks:
|
||||
- name: Check the API servers are responding
|
||||
local_action:
|
||||
module: uri
|
||||
url: https://{{ansible_ssh_host}}/api/v1
|
||||
user: kube
|
||||
password: changeme
|
||||
validate_certs: no
|
||||
status_code: 200
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
---
|
||||
- hosts: node1
|
||||
|
||||
tasks:
|
||||
- name: Run a replica controller composed of 2 pods
|
||||
shell: "kubectl run test --image=busybox --replicas=2 --command -- tail -f /dev/null"
|
||||
|
||||
- name: Pods are running
|
||||
shell: "kubectl get pods --no-headers -o json"
|
||||
register: run_pods_log
|
||||
until: (run_pods_log.stdout | from_json)['items'] | map(attribute = 'status.phase') | join(',') == "Running,Running"
|
||||
retries: 24
|
||||
delay: 5
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
---
|
||||
- hosts: node1
|
||||
|
||||
tasks:
|
||||
|
||||
- name: Get pod names
|
||||
shell: "kubectl get pods -o json"
|
||||
register: pods
|
||||
|
||||
- set_fact:
|
||||
pod_names: "{{ (pods.stdout | from_json)['items'] | map(attribute = 'metadata.name') | list }}"
|
||||
pod_ips: "{{ (pods.stdout | from_json)['items'] | map(attribute = 'status.podIP') | list }}"
|
||||
|
||||
- name: Check pods IP are in correct network
|
||||
assert:
|
||||
that: item | ipaddr(kube_pods_subnet)
|
||||
with_items: pod_ips
|
||||
|
||||
|
||||
- name: Ping between pods is working
|
||||
shell: "kubectl exec {{pod_names[0]}} -- ping -c 4 {{ pod_ips[1] }}"
|
||||
Loading…
Reference in New Issue