C12s/c12s-kubespray
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add etcd-events cluster for kube-apiserver (#2385)

Browse source 
Add etcd-events cluster for kube-apiserver
This commit is contained in:
RongZhang 2018-03-01 02:39:14 -06:00 committed by Matthew Mosesohn
parent af7edf4dff
commit 67ffd8e923
17 changed files with 309 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
roles/etcd/defaults/main.yml
View file

@ -4,6 +4,7 @@ etcd_cluster_setup: true
etcd_backup_prefix: "/var/backups"
etcd_data_dir: "/var/lib/etcd"
etcd_events_data_dir: "/var/lib/etcd-events"
etcd_config_dir: /etc/ssl/etcd
etcd_cert_dir: "{{ etcd_config_dir }}/ssl"

27
roles/etcd/handlers/main.yml
View file

@ -7,17 +7,33 @@
- reload etcd
- wait for etcd up
- name: restart etcd-events
command: /bin/true
notify:
- etcd-events | reload systemd
- reload etcd-events
- wait for etcd-events up
- import_tasks: backup.yml
- name: etcd | reload systemd
command: systemctl daemon-reload
- name: etcd-events | reload systemd
command: systemctl daemon-reload
- name: reload etcd
service:
name: etcd
state: restarted
when: is_etcd_master
- name: reload etcd-events
service:
name: etcd-events
state: restarted
when: is_etcd_master
- name: wait for etcd up
uri:
url: "https://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health"
@ -29,6 +45,17 @@
retries: 10
delay: 5
- name: wait for etcd-events up
uri:
url: "https://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2381/health"
validate_certs: no
client_cert: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem"
client_key: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem"
register: result
until: result.status is defined and result.status == 200
retries: 10
delay: 5
- name: set etcd_secret_changed
set_fact:
etcd_secret_changed: true

44
roles/etcd/tasks/configure.yml
View file

@ -1,5 +1,5 @@
---
- name: Configure | Check if member is in cluster
- name: Configure | Check if member is in etcd cluster
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
register: etcd_member_in_cluster
ignore_errors: true
@ -12,6 +12,19 @@
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
- name: Configure | Check if member is in etcd-events cluster
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_access_address }}"
register: etcd_events_member_in_cluster
ignore_errors: true
changed_when: false
check_mode: no
when: is_etcd_master and etcd_events_cluster_setup
tags:
- facts
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
- name: Configure | Copy etcd.service systemd file
template:
src: "etcd-{{ etcd_deployment_type }}.service.j2"
@ -20,11 +33,36 @@
when: is_etcd_master
notify: restart etcd
- name: Configure | Join member(s) to cluster one at a time
include_tasks: join_member.yml
- name: Configure | Copy etcd-events.service systemd file
template:
src: "etcd-events-host.service.j2"
dest: /etc/systemd/system/etcd-events.service
backup: yes
when: is_etcd_master and etcd_deployment_type == "host" and etcd_events_cluster_setup
notify: restart etcd-events
- name: Configure | Copy etcd-events.service systemd file
template:
src: "etcd-events-docker.service.j2"
dest: /etc/systemd/system/etcd-events.service
backup: yes
when: is_etcd_master and etcd_deployment_type == "docker" and etcd_events_cluster_setup
notify: restart etcd-events
- name: Configure | Join member(s) to etcd cluster one at a time
include_tasks: join_etcd_member.yml
vars:
target_node: "{{ item }}"
loop_control:
pause: 10
with_items: "{{ groups['etcd'] }}"
when: inventory_hostname == item and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0
- name: Configure | Join member(s) to etcd-events cluster one at a time
include_tasks: join_etcd-evetns_member.yml
vars:
target_node: "{{ item }}"
loop_control:
pause: 10
with_items: "{{ groups['etcd'] }}"
when: inventory_hostname == item and etcd_events_cluster_setup and etcd_events_member_in_cluster.rc != 0 and etcd_events_cluster_is_healthy.rc == 0

10
roles/etcd/tasks/install_docker.yml
View file

@ -18,3 +18,13 @@
mode: 0755
backup: yes
notify: restart etcd
- name: Install etcd-events launch script
template:
src: etcd-events.j2
dest: "{{ bin_dir }}/etcd-events"
owner: 'root'
mode: 0755
backup: yes
when: etcd_events_cluster_setup
notify: restart etcd-events

47
roles/etcd/tasks/join_etcd-events_member.yml Normal file
View file

@ -0,0 +1,47 @@
---
- name: Join Member | Add member to cluster
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} member add {{ etcd_member_name }} {{ etcd_events_peer_url }}"
register: member_add_result
until: member_add_result.rc == 0
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
when: target_node == inventory_hostname
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
- include_tasks: refresh_config.yml
vars:
etcd_events_peer_addresses: >-
{% for host in groups['etcd'] -%}
{%- if hostvars[host]['etcd_events_member_in_cluster'].rc == 0 -%}
{{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2382,
{%- endif -%}
{%- if loop.last -%}
{{ etcd_member_name }}={{ etcd_events_peer_url }}
{%- endif -%}
{%- endfor -%}
when: target_node == inventory_hostname
- name: Join Member | reload systemd
command: systemctl daemon-reload
when: target_node == inventory_hostname
- name: Join Member | Ensure etcd-events is running
service:
name: etcd-events
state: started
enabled: yes
when: target_node == inventory_hostname
- name: Join Member | Ensure member is in etcd-events cluster
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_access_addresses }} member list | grep -q {{ etcd_events_access_address }}"
register: etcd_events_member_in_cluster
changed_when: false
check_mode: no
tags:
- facts
when: target_node == inventory_hostname
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"

47
roles/etcd/tasks/join_etcd_member.yml Normal file
View file

@ -0,0 +1,47 @@
---
- name: Join Member | Add member to cluster
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}"
register: member_add_result
until: member_add_result.rc == 0
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
when: target_node == inventory_hostname
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
- include_tasks: refresh_config.yml
vars:
etcd_peer_addresses: >-
{% for host in groups['etcd'] -%}
{%- if hostvars[host]['etcd_member_in_cluster'].rc == 0 -%}
{{ "etcd"+loop.index|string }}=https://{{ hostvars[host].access_ip | default(hostvars[host].ip | default(hostvars[host].ansible_default_ipv4['address'])) }}:2380,
{%- endif -%}
{%- if loop.last -%}
{{ etcd_member_name }}={{ etcd_peer_url }}
{%- endif -%}
{%- endfor -%}
when: target_node == inventory_hostname
- name: Join Member | reload systemd
command: systemctl daemon-reload
when: target_node == inventory_hostname
- name: Join Member | Ensure etcd is running
service:
name: etcd
state: started
enabled: yes
when: target_node == inventory_hostname
- name: Join Member | Ensure member is in cluster
shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
register: etcd_member_in_cluster
changed_when: false
check_mode: no
tags:
- facts
when: target_node == inventory_hostname
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"

12
roles/etcd/tasks/main.yml
View file

@ -43,6 +43,11 @@
notify: restart etcd
when: is_etcd_master and etcd_secret_changed|default(false)
- name: Restart etcd-events if certs changed
command: /bin/true
notify: restart etcd
when: is_etcd_master and etcd_events_cluster_setup and etcd_secret_changed|default(false)
# reload-systemd
- meta: flush_handlers
@ -53,6 +58,13 @@
enabled: yes
when: is_etcd_master and etcd_cluster_setup
- name: Ensure etcd-events is running
service:
name: etcd-events
state: started
enabled: yes
when: is_etcd_master and etcd_events_cluster_setup
# After etcd cluster is assembled, make sure that
# initial state of the cluster is in `existing`
# state insted of `new`.

7
roles/etcd/tasks/refresh_config.yml
View file

@ -5,3 +5,10 @@
dest: /etc/etcd.env
notify: restart etcd
when: is_etcd_master
- name: Refresh config | Create etcd-events config file
template:
src: etcd-events.env.j2
dest: /etc/etcd-events.env
notify: restart etcd-events
when: is_etcd_master and etcd_events_cluster_setup

15
roles/etcd/tasks/set_cluster_health.yml
View file

@ -1,5 +1,5 @@
---
- name: Configure | Check if cluster is healthy
- name: Configure | Check if etcd cluster is healthy
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
register: etcd_cluster_is_healthy
ignore_errors: true
@ -11,3 +11,16 @@
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
- name: Configure | Check if etcd-events cluster is healthy
shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
register: etcd_events_cluster_is_healthy
ignore_errors: true
changed_when: false
check_mode: no
when: is_etcd_master and etcd_events_cluster_setup
tags:
- facts
environment:
ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"

18
roles/etcd/templates/etcd-events-docker.service.j2 Normal file
View file

@ -0,0 +1,18 @@
[Unit]
Description=etcd docker wrapper
Wants=docker.socket
After=docker.service
[Service]
User=root
PermissionsStartOnly=true
EnvironmentFile=-/etc/etcd-events.env
ExecStart={{ bin_dir }}/etcd-events
ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ etcd_member_name }}-events
ExecStop={{ docker_bin_dir }}/docker stop {{ etcd_member_name }}-events
Restart=always
RestartSec=15s
TimeoutStartSec=30s
[Install]
WantedBy=multi-user.target

16
roles/etcd/templates/etcd-events-host.service.j2 Normal file
View file

@ -0,0 +1,16 @@
[Unit]
Description=etcd
After=network.target
[Service]
Type=notify
User=root
EnvironmentFile=/etc/etcd-events.env
ExecStart={{ bin_dir }}/etcd
NotifyAccess=all
Restart=always
RestartSec=10s
LimitNOFILE=40000
[Install]
WantedBy=multi-user.target

26
roles/etcd/templates/etcd-events.env.j2 Normal file
View file

@ -0,0 +1,26 @@
ETCD_DATA_DIR={{ etcd_events_data_dir }}
ETCD_ADVERTISE_CLIENT_URLS={{ etcd_events_client_url }}
ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_events_peer_url }}
ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}
ETCD_METRICS={{ etcd_metrics }}
ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2381,https://127.0.0.1:2381
ETCD_ELECTION_TIMEOUT={{ etcd_election_timeout }}
ETCD_HEARTBEAT_INTERVAL={{ etcd_heartbeat_interval }}
ETCD_INITIAL_CLUSTER_TOKEN=k8s_events_etcd
ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2382
ETCD_NAME={{ etcd_member_name }}-events
ETCD_PROXY=off
ETCD_INITIAL_CLUSTER={{ etcd_events_peer_addresses }}
ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }}
# TLS settings
ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
ETCD_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
ETCD_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
ETCD_CLIENT_CERT_AUTH={{ etcd_secure_client | lower}}
ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
ETCD_PEER_CLIENT_CERT_AUTH={{ etcd_peer_client_auth }}

22
roles/etcd/templates/etcd-events.j2 Normal file
View file

@ -0,0 +1,22 @@
#!/bin/bash
{{ docker_bin_dir }}/docker run \
--restart=on-failure:5 \
--env-file=/etc/etcd-events.env \
--net=host \
-v /etc/ssl/certs:/etc/ssl/certs:ro \
-v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
-v {{ etcd_events_data_dir }}:{{ etcd_events_data_dir }}:rw \
{% if etcd_memory_limit is defined %}
--memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} \
{% endif %}
--oom-kill-disable \
{% if etcd_cpu_limit is defined %}
--cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \
{% endif %}
{% if etcd_blkio_weight is defined %}
--blkio-weight={{ etcd_blkio_weight }} \
{% endif %}
--name={{ etcd_member_name }}-events \
{{ etcd_image_repo }}:{{ etcd_image_tag }} \
/usr/local/bin/etcd \
"$@"

3
roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
View file

@ -30,6 +30,9 @@ spec:
- apiserver
- --advertise-address={{ ip | default(ansible_default_ipv4.address) }}
- --etcd-servers={{ etcd_access_addresses }}
{% if etcd_events_cluster_setup %}
- --etcd-servers-overrides=/events#{{ etcd_events_access_addresses }}
{% endif %}
{% if kube_version | version_compare('v1.9', '<') %}
- --etcd-quorum-read=true
{% endif %}

13
roles/kubespray-defaults/defaults/main.yaml
View file

@ -289,16 +289,25 @@ kube_apiserver_client_key: |-
{{ kube_cert_dir }}/apiserver-key.pem
{%- endif %}
# Set to true to deploy etcd-events cluster
etcd_events_cluster_setup: false
# Vars for pointing to etcd endpoints
is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}"
etcd_address: "{{ ip | default(ansible_default_ipv4['address']) }}"
etcd_access_address: "{{ access_ip | default(etcd_address) }}"
etcd_peer_url: "https://{{ etcd_access_address }}:2380"
etcd_client_url: "https://{{ etcd_access_address }}:2379"
etcd_events_peer_url: "https://{{ etcd_access_address }}:2382"
etcd_events_client_url: "https://{{ etcd_access_address }}:2381"
etcd_access_addresses: |-
{% for item in groups['etcd'] -%}
https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}
{%- endfor %}
etcd_events_access_addresses: |-
{% for item in groups['etcd'] -%}
https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}:2381{% if not loop.last %},{% endif %}
{%- endfor %}
etcd_member_name: |-
{% for host in groups['etcd'] %}
{% if inventory_hostname == host %}{{"etcd"+loop.index|string }}{% endif %}
@ -307,3 +316,7 @@ etcd_peer_addresses: |-
{% for item in groups['etcd'] -%}
{{ "etcd"+loop.index|string }}=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %}
{%- endfor %}
etcd_events_peer_addresses: |-
{% for item in groups['etcd'] -%}
{{ "etcd"+loop.index|string }}-events=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2382{% if not loop.last %},{% endif %}
{%- endfor %}

4
roles/reset/tasks/main.yml
View file

@ -8,6 +8,7 @@
- kubelet
- vault
- etcd
- etcd-events
failed_when: false
tags:
- services
@ -19,6 +20,7 @@
with_items:
- kubelet
- etcd
- etcd-events
- vault
- calico-node
register: services_removed
@ -95,6 +97,7 @@
- /root/.kube
- /root/.helm
- "{{ etcd_data_dir }}"
- /var/lib/etcd-events
- /etc/ssl/etcd
- /var/log/calico
- /etc/cni
@ -125,6 +128,7 @@
- "{{ bin_dir }}/kubelet"
- "{{ bin_dir }}/etcd-scripts"
- "{{ bin_dir }}/etcd"
- "{{ bin_dir }}/etcd-events"
- "{{ bin_dir }}/etcdctl"
- "{{ bin_dir }}/kubernetes-scripts"
- "{{ bin_dir }}/kubectl"

1
tests/files/gce_centos7-flannel-addons.yml
View file

@ -9,6 +9,7 @@ kube_network_plugin: flannel
helm_enabled: true
istio_enabled: true
efk_enabled: true
etcd_events_cluster_setup: true
local_volume_provisioner_enabled: true
etcd_deployment_type: host
deploy_netchecker: true