From c7b00caeaac992e8dbb623994515dd9affe1b1b3 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 21 Oct 2016 14:21:46 +0300 Subject: [PATCH 1/4] Use tar+register instead of copy/slurp for distributing tokens and certs Related bug: https://github.com/ansible/ansible/issues/15405 Uses tar and register because synchronize module cannot sudo on the remote side correctly and copy is too slow. This patch dramatically cuts down the number of tasks to process for cert synchronization. --- roles/kubernetes/secrets/tasks/gen_certs.yml | 33 +++++++++---------- roles/kubernetes/secrets/tasks/gen_tokens.yml | 17 ++++------ 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/roles/kubernetes/secrets/tasks/gen_certs.yml b/roles/kubernetes/secrets/tasks/gen_certs.yml index 6057c0676..bec1d9f16 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs.yml @@ -27,31 +27,30 @@ master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'apiserver-key.pem', 'apiserver.pem'] node_certs: ['ca.pem', 'node.pem', 'node-key.pem'] -- name: Gen_certs | Get the certs from first master - slurp: - src: "{{ kube_cert_dir }}/{{ item }}" +- name: Gen_certs | Gather master certs + shell: "tar cfz - -C {{ kube_cert_dir }} {{ master_certs|join(' ') }} {{ node_certs|join(' ') }} | base64 --wrap=0" + register: master_cert_data delegate_to: "{{groups['kube-master'][0]}}" - register: slurp_certs - with_items: '{{ master_certs + node_certs }}' - when: sync_certs|default(false) run_once: true - notify: set secret_changed + when: sync_certs|default(false) + +- name: Gen_certs | Gather node certs + shell: "tar cfz - -C {{ kube_cert_dir }} {{ node_certs|join(' ') }} | base64 --wrap=0" + register: node_cert_data + delegate_to: "{{groups['kube-master'][0]}}" + run_once: true + when: sync_certs|default(false) - name: Gen_certs | Copy certs on masters - copy: - content: "{{ item.content|b64decode }}" - dest: "{{ item.source }}" - with_items: '{{slurp_certs.results}}' + shell: "echo '{{master_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ kube_cert_dir }}" + changed_when: false when: inventory_hostname in groups['kube-master'] and sync_certs|default(false) and inventory_hostname != groups['kube-master'][0] - name: Gen_certs | Copy certs on nodes - copy: - content: "{{ item.content|b64decode }}" - dest: "{{ item.source }}" - with_items: '{{slurp_certs.results}}' - when: item.item in node_certs and - inventory_hostname in groups['kube-node'] and sync_certs|default(false) and + shell: "echo '{{node_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ kube_cert_dir }}" + changed_when: false + when: inventory_hostname in groups['kube-node'] and sync_certs|default(false) and inventory_hostname != groups['kube-master'][0] - name: Gen_certs | check certificate permissions diff --git a/roles/kubernetes/secrets/tasks/gen_tokens.yml b/roles/kubernetes/secrets/tasks/gen_tokens.yml index 796657f65..dbe35811b 100644 --- a/roles/kubernetes/secrets/tasks/gen_tokens.yml +++ b/roles/kubernetes/secrets/tasks/gen_tokens.yml @@ -43,20 +43,15 @@ delegate_to: "{{groups['kube-master'][0]}}" when: sync_tokens|default(false) -- name: Gen_tokens | Get the tokens from first master - slurp: - src: "{{ item }}" - register: slurp_tokens - with_items: '{{tokens_list.stdout_lines}}' - run_once: true +- name: Gen_tokens | Gather tokens + shell: "tar cfz - {{ tokens_list.stdout_lines | join(' ') }} | base64 --wrap=0" + register: tokens_data delegate_to: "{{groups['kube-master'][0]}}" + run_once: true when: sync_tokens|default(false) - notify: set secret_changed - name: Gen_tokens | Copy tokens on masters - copy: - content: "{{ item.content|b64decode }}" - dest: "{{ item.source }}" - with_items: '{{slurp_tokens.results}}' + shell: "echo '{{ tokens_data.stdout|quote }}' | base64 -d | tar xz -C /" + changed_when: false when: inventory_hostname in groups['kube-master'] and sync_tokens|default(false) and inventory_hostname != groups['kube-master'][0] From 2778ac61a432b016cc19f0671dae82f2e2d08fee Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Wed, 26 Oct 2016 17:56:15 +0300 Subject: [PATCH 2/4] Add new var skip_dnsmasq_k8s If skip_dnsmasq is set, it will still not set up dnsmasq k8s pod. This enables independent setup of resolvconf section before kubelet is up. --- roles/dnsmasq/defaults/main.yml | 8 +++++++- roles/dnsmasq/tasks/main.yml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/roles/dnsmasq/defaults/main.yml b/roles/dnsmasq/defaults/main.yml index 7a1e77023..89ab02ab8 100644 --- a/roles/dnsmasq/defaults/main.yml +++ b/roles/dnsmasq/defaults/main.yml @@ -16,4 +16,10 @@ dnsmasq_version: 2.72 # Images dnsmasq_image_repo: "andyshinn/dnsmasq" -dnsmasq_image_tag: "{{ dnsmasq_version }}" \ No newline at end of file +dnsmasq_image_tag: "{{ dnsmasq_version }}" + +# Skip dnsmasq setup +skip_dnsmasq: false + +# Skip setting up dnsmasq daemonset +skip_dnsmasq_k8s: "{{ skip_dnsmasq }}" diff --git a/roles/dnsmasq/tasks/main.yml b/roles/dnsmasq/tasks/main.yml index 46c1604f6..6b271a1e2 100644 --- a/roles/dnsmasq/tasks/main.yml +++ b/roles/dnsmasq/tasks/main.yml @@ -1,5 +1,5 @@ --- - include: dnsmasq.yml - when: "{{ not skip_dnsmasq|bool }}" + when: "{{ not skip_dnsmasq_k8s|bool }}" - include: resolvconf.yml From 03e162b342072b8229dc61ce569dcdd9e87848ae Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 28 Oct 2016 11:16:11 +0400 Subject: [PATCH 3/4] Update OWNERS --- OWNERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/OWNERS b/OWNERS index 583a0314b..6ecbee5c9 100644 --- a/OWNERS +++ b/OWNERS @@ -4,3 +4,6 @@ owners: - Smana - ant31 + - bogdando + - mattymo + - rsmitty From 50f77cca1d011e75c5b330cca3114ce54c13f502 Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Fri, 28 Oct 2016 14:56:48 +0200 Subject: [PATCH 4/4] Add CI test layouts * Drop Wily from test matrix * Replace the Wily cases dropped with extra cases to test separate roles deployment Signed-off-by: Bogdan Dobrelya --- .travis.yml | 34 ++++++++++++++++++++-------- tests/cloud_playbooks/create-gce.yml | 2 +- tests/cloud_playbooks/delete-gce.yml | 2 +- tests/templates/inventory-gce.j2 | 11 +++++++++ 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 13604765e..863374c26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,80 +11,92 @@ env: CONTAINER_ENGINE=docker PRIVATE_KEY=$GCE_PRIVATE_KEY ANSIBLE_KEEP_REMOTE_FILES=1 + CLUSTER_MODE=default matrix: # Debian Jessie - >- KUBE_NETWORK_PLUGIN=flannel CLOUD_IMAGE=debian-8-kubespray CLOUD_REGION=europe-west1-b + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=calico CLOUD_IMAGE=debian-8-kubespray CLOUD_REGION=us-central1-c + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=debian-8-kubespray CLOUD_REGION=us-east1-d + CLUSTER_MODE=default # Centos 7 - >- KUBE_NETWORK_PLUGIN=flannel CLOUD_IMAGE=centos-7-sudo CLOUD_REGION=asia-east1-c - + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=calico CLOUD_IMAGE=centos-7-sudo CLOUD_REGION=europe-west1-b - + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=centos-7-sudo CLOUD_REGION=us-central1-c + CLUSTER_MODE=default # Redhat 7 - >- KUBE_NETWORK_PLUGIN=flannel CLOUD_IMAGE=rhel-7-sudo CLOUD_REGION=us-east1-d - + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=calico CLOUD_IMAGE=rhel-7-sudo CLOUD_REGION=asia-east1-c - + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=rhel-7-sudo CLOUD_REGION=europe-west1-b + CLUSTER_MODE=default # Ubuntu 16.04 - >- KUBE_NETWORK_PLUGIN=flannel CLOUD_IMAGE=ubuntu-1604-xenial CLOUD_REGION=us-central1-c + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=calico CLOUD_IMAGE=ubuntu-1604-xenial CLOUD_REGION=us-east1-d + CLUSTER_MODE=default - >- KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=ubuntu-1604-xenial CLOUD_REGION=asia-east1-c + CLUSTER_MODE=default - # Ubuntu 15.10 + # Extra cases for separated roles - >- KUBE_NETWORK_PLUGIN=flannel - CLOUD_IMAGE=ubuntu-1510-wily + CLOUD_IMAGE=rhel-7-sudo CLOUD_REGION=europe-west1-b + CLUSTER_MODE=separate - >- KUBE_NETWORK_PLUGIN=calico - CLOUD_IMAGE=ubuntu-1510-wily + CLOUD_IMAGE=ubuntu-1604-xenial CLOUD_REGION=us-central1-a + CLUSTER_MODE=separate - >- KUBE_NETWORK_PLUGIN=weave - CLOUD_IMAGE=ubuntu-1510-wily + CLOUD_IMAGE=debian-8-kubespray CLOUD_REGION=us-east1-d + CLUSTER_MODE=separate before_install: @@ -92,7 +104,8 @@ before_install: - pip install --user boto -U - pip install --user ansible - pip install --user netaddr - - pip install --user apache-libcloud + # W/A https://github.com/ansible/ansible-modules-core/issues/5196#issuecomment-253766186 + - pip install --user apache-libcloud==0.20.1 cache: - directories: @@ -114,7 +127,8 @@ before_script: script: - > - $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts -c local $LOG_LEVEL + $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts.cfg -c local $LOG_LEVEL + -e mode=${CLUSTER_MODE} -e test_id=${TEST_ID} -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} -e gce_project_id=${GCE_PROJECT_ID} diff --git a/tests/cloud_playbooks/create-gce.yml b/tests/cloud_playbooks/create-gce.yml index 840cf2e7c..b2c3e3020 100644 --- a/tests/cloud_playbooks/create-gce.yml +++ b/tests/cloud_playbooks/create-gce.yml @@ -1,6 +1,6 @@ --- - hosts: localhost - sudo: False + become: false gather_facts: no vars: cloud_machine_type: g1-small diff --git a/tests/cloud_playbooks/delete-gce.yml b/tests/cloud_playbooks/delete-gce.yml index d42c6cc91..54902fb6f 100644 --- a/tests/cloud_playbooks/delete-gce.yml +++ b/tests/cloud_playbooks/delete-gce.yml @@ -1,6 +1,6 @@ --- - hosts: localhost - sudo: False + become: false gather_facts: no vars: cloud_machine_type: f1-micro diff --git a/tests/templates/inventory-gce.j2 b/tests/templates/inventory-gce.j2 index 72ad469de..418910771 100644 --- a/tests/templates/inventory-gce.j2 +++ b/tests/templates/inventory-gce.j2 @@ -2,6 +2,16 @@ node1 ansible_ssh_host={{gce.instance_data[0].public_ip}} node2 ansible_ssh_host={{gce.instance_data[1].public_ip}} node3 ansible_ssh_host={{gce.instance_data[2].public_ip}} +{% if mode is defined and mode == "separate" %} +[kube-master] +node1 + +[kube-node] +node2 + +[etcd] +node3 +{% else %} [kube-master] node1 node2 @@ -14,6 +24,7 @@ node3 [etcd] node1 node2 +{% endif %} [k8s-cluster:children] kube-node