C12s/c12s-kubespray
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Enable HA deploy of kubeadm (#1658)

Browse source 
* Enable HA deploy of kubeadm

* raise delay to 60s for starting gce hosts
This commit is contained in:
Matthew Mosesohn 2017-09-15 22:28:15 +01:00 committed by GitHub
parent b294db5aed
commit 8e731337ba
10 changed files with 88 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.gitlab-ci.yml
View file

@ -168,7 +168,9 @@ before_script:
# Tests Cases
## Test Master API
- ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL
- >
ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL
-e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
## Ping the between 2 pod
- ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $LOG_LEVEL
@ -301,7 +303,7 @@ before_script:
CLOUD_IMAGE: ubuntu-1604-xenial
CLOUD_MACHINE_TYPE: "n1-standard-2"
CLOUD_REGION: europe-west1-b
CLUSTER_MODE: default
CLUSTER_MODE: ha
KUBEADM_ENABLED: "true"
STARTUP_SCRIPT: ""

5
roles/download/defaults/main.yml
View file

@ -19,7 +19,8 @@ download_always_pull: False
# Versions
kube_version: v1.7.5
kubeadm_version: "{{ kube_version }}"
# Change to kube_version after v1.8.0 release
kubeadm_version: "v1.8.0-beta.1"
etcd_version: v3.2.4
# TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
# after migration to container download
@ -38,7 +39,7 @@ kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release
# Checksums
etcd_checksum: "274c46a7f8d26f7ae99d6880610f54933cbcf7f3beafa19236c52eb5df8c7a0b"
kubeadm_checksum: "378e6052f8b178f8e6a38e8637681c72d389443b66b78b51b8ddc9a162c655c3"
kubeadm_checksum: "ddd5949699d6bdbc0b90b379e7e534f137b1058db1acc8f26cc54843f017ffbf"
# Containers
# Possible values: host, docker

13
roles/kubernetes/kubeadm/tasks/main.yml
View file

@ -10,6 +10,11 @@
when: not is_kube_master
tags: facts
- name: Check if kubelet.conf exists
stat:
path: "{{ kube_config_dir }}/kubelet.conf"
register: kubelet_conf
- name: Create kubeadm client config
template:
src: kubeadm-client.conf.j2
@ -21,7 +26,13 @@
- name: Join to cluster if needed
command: kubeadm join --config {{ kube_config_dir}}/kubeadm-client.conf --skip-preflight-checks
register: kubeadm_join
when: not is_kube_master and kubeadm_client_conf.changed
when: not is_kube_master and (kubeadm_client_conf.changed or not kubelet_conf.stat.exists)
- name: Wait for kubelet bootstrap to create config
wait_for:
path: "{{ kube_config_dir }}/kubelet.conf"
delay: 1
timeout: 60
- name: Update server field in kubelet kubeconfig
replace:

53
roles/kubernetes/master/tasks/kubeadm-setup.yml
View file

@ -14,7 +14,7 @@
{{ apiserver_loadbalancer_domain_name }}
{%- endif %}
{%- for host in groups['kube-master'] -%}
{%- if hostvars[host]['access_ip'] is defined %}{{ hostvars[host]['access_ip'] }}{% endif -%}
{%- if hostvars[host]['access_ip'] is defined %}{{ hostvars[host]['access_ip'] }}{% endif %}
{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
{%- endfor %}
tags: facts
@ -29,7 +29,54 @@
dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
register: kubeadm_config
- name: kubeadm | Initialize cluster
- name: Check if kubeadm has already run
stat:
path: "{{ kube_config_dir }}/admin.conf"
register: admin_conf
- name: kubeadm | Initialize first master
command: timeout -k 240s 240s kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
register: kubeadm_init
when: kubeadm_config.changed
#Retry is because upload config sometimes fails
retries: 3
when: inventory_hostname == groups['kube-master']|first and (kubeadm_config.changed or not admin_conf.stat.exists)
- name: slurp kubeadm certs
slurp:
src: "{{ item }}"
with_items:
- "{{ kube_cert_dir }}/apiserver.crt"
- "{{ kube_cert_dir }}/apiserver.key"
- "{{ kube_cert_dir }}/apiserver-kubelet-client.crt"
- "{{ kube_cert_dir }}/apiserver-kubelet-client.key"
- "{{ kube_cert_dir }}/ca.crt"
- "{{ kube_cert_dir }}/ca.key"
- "{{ kube_cert_dir }}/front-proxy-ca.crt"
- "{{ kube_cert_dir }}/front-proxy-ca.key"
- "{{ kube_cert_dir }}/front-proxy-client.crt"
- "{{ kube_cert_dir }}/front-proxy-client.key"
- "{{ kube_cert_dir }}/sa.key"
- "{{ kube_cert_dir }}/sa.pub"
register: kubeadm_certs
delegate_to: "{{ groups['kube-master']|first }}"
run_once: true
- name: write out kubeadm certs
copy:
dest: "{{ item.item }}"
content: "{{ item.content | b64decode }}"
owner: root
group: root
mode: 0700
no_log: true
register: copy_kubeadm_certs
with_items: "{{ kubeadm_certs.results }}"
when: inventory_hostname != groups['kube-master']|first
- name: kubeadm | Initialize other masters
command: timeout -k 240s 240s kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
register: kubeadm_init
#Retry is because upload config sometimes fails
retries: 3
when: inventory_hostname != groups['kube-master']|first and (kubeadm_config.changed or not admin_conf.stat.exists or copy_kubeadm_certs.changed)

6
roles/kubernetes/master/tasks/main.yml
View file

@ -2,6 +2,12 @@
- include: pre-upgrade.yml
tags: k8s-pre-upgrade
# upstream bug: https://github.com/kubernetes/kubeadm/issues/441
- name: Disable kube_basic_auth until kubeadm/441 is fixed
set_fact:
kube_basic_auth: false
when: kubeadm_enabled|bool|default(false)
- include: users-file.yml
when: kube_basic_auth|default(true)

7
roles/kubernetes/master/templates/kubeadm-config.yaml.j2
View file

@ -2,7 +2,7 @@ apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
bindPort: "{{ kube_apiserver_port }}"
bindPort: {{ kube_apiserver_port }}
etcd:
endpoints:
{% for endpoint in etcd_access_endpoint.split(',') %}
@ -19,16 +19,18 @@ kubernetesVersion: {{ kube_version }}
cloudProvider: {{ cloud_provider|default('') }}
#TODO: cloud provider conf file
authorizationModes:
- Node
{% for mode in authorization_modes %}
- {{ mode }}
{% endfor %}
token: {{ kubeadm_token }}
tokenTTL: {{ kubeadm_token_ttl }}
tokenTTL: "{{ kubeadm_token_ttl }}"
selfHosted: false
apiServerExtraArgs:
insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
insecure-port: "{{ kube_apiserver_insecure_port }}"
admission-control: {{ kube_apiserver_admission_control | join(',') }}
apiserver-count: "{{ kube_apiserver_count }}"
service-node-port-range: {{ kube_apiserver_node_port_range }}
{% if kube_basic_auth|default(true) %}
basic-auth-file: {{ kube_users_dir }}/known_users.csv
@ -65,3 +67,4 @@ apiServerCertSANs:
- {{ san }}
{% endfor %}
certificatesDir: {{ kube_config_dir }}/ssl
unifiedControlPlaneImage: "{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}"

3
roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
View file

@ -15,8 +15,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
{# Base kubelet args #}
{% set kubelet_args_base -%}
{# start kubeadm specific settings #}
--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
--kubeconfig={{ kube_config_dir }}/kubelet.conf \
--require-kubeconfig=true \
--require-kubeconfig \
--authorization-mode=Webhook \
--client-ca-file={{ kube_cert_dir }}/ca.crt \
--pod-manifest-path={{ kube_manifest_dir }} \

1
roles/reset/tasks/main.yml
View file

@ -109,6 +109,7 @@
- "{{ bin_dir }}/etcdctl"
- "{{ bin_dir }}/kubernetes-scripts"
- "{{ bin_dir }}/kubectl"
- "{{ bin_dir }}/kubeadm"
- "{{ bin_dir }}/helm"
- "{{ bin_dir }}/calicoctl"
- "{{ bin_dir }}/weave"

4
tests/cloud_playbooks/create-gce.yml
View file

@ -35,7 +35,7 @@
register: gce
- name: Add instances to host group
add_host: hostname={{item.name}} ansible_host={{item.public_ip}} groupname="waitfor_hosts"
add_host: hostname={{item.public_ip}} groupname="waitfor_hosts"
with_items: '{{gce.instance_data}}'
- name: Template the inventory
@ -61,4 +61,4 @@
gather_facts: false
tasks:
- name: Wait for SSH to come up.
local_action: wait_for host={{inventory_hostname}} port=22 delay=5 timeout=240 state=started
local_action: wait_for host={{inventory_hostname}} port=22 delay=60 timeout=240 state=started

1
tests/testcases/010_check-apiserver.yml
View file

@ -9,3 +9,4 @@
password: "{{ lookup('password', '../../credentials/kube_user length=15 chars=ascii_letters,digits') }}"
validate_certs: no
status_code: 200
when: not kubeadm_enabled|default(false)