From eaed0050454c439e0a6820d6e534f3c09400f073 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 24 Jun 2016 15:25:16 +0300 Subject: [PATCH] Add optional deployment mode for Docker etcd_deployment_type Running etcd in Docker reduces the number of individual file downloads and services running on the host. Note: etcd container v3.0.1 moves bindir to /usr/local/bin Fixes: #298 --- roles/etcd/defaults/main.yml | 7 + roles/etcd/handlers/main.yml | 9 +- roles/etcd/meta/main.yml | 3 + roles/etcd/tasks/configure.yml | 4 +- roles/etcd/tasks/install.yml | 35 ++++- roles/etcd/tasks/main.yml | 2 +- roles/etcd/tasks/set_facts.yml | 4 + roles/etcd/templates/deb-etcd-docker.initd.j2 | 126 ++++++++++++++++++ ...b-etcd.initd.j2 => deb-etcd-host.initd.j2} | 5 +- roles/etcd/templates/etcd-docker.service.j2 | 33 +++++ .../{etcd.service.j2 => etcd-host.service.j2} | 0 roles/etcd/templates/etcd.j2 | 24 ++-- 12 files changed, 232 insertions(+), 20 deletions(-) create mode 100644 roles/etcd/templates/deb-etcd-docker.initd.j2 rename roles/etcd/templates/{deb-etcd.initd.j2 => deb-etcd-host.initd.j2} (97%) create mode 100644 roles/etcd/templates/etcd-docker.service.j2 rename roles/etcd/templates/{etcd.service.j2 => etcd-host.service.j2} (100%) diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index e599619ca..63f485719 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -1,3 +1,10 @@ --- etcd_version: v3.0.1 etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/" + +# Possible values: host, docker +etcd_deployment_type: "host" + + +etcd_image_repo: "quay.io/coreos/etcd" +etcd_image_tag: "{{ etcd_version }}" diff --git a/roles/etcd/handlers/main.yml b/roles/etcd/handlers/main.yml index a2044fb08..334a4c396 100644 --- a/roles/etcd/handlers/main.yml +++ b/roles/etcd/handlers/main.yml @@ -3,13 +3,20 @@ command: /bin/true notify: - reload systemd + - start etcd - reload etcd - name: reload systemd command: systemctl daemon-reload when: ansible_service_mgr == "systemd" +- name: start etcd + service: + name: etcd + enabled: yes + state: started + - name: reload etcd service: name: etcd - state: restarted + state: "{{ 'restarted' if etcd_deployment_type == 'host' else 'reloaded' }}" diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml index 5ea32c371..84d029d58 100644 --- a/roles/etcd/meta/main.yml +++ b/roles/etcd/meta/main.yml @@ -5,3 +5,6 @@ dependencies: when: ansible_os_family != 'CoreOS' - role: download file: "{{ downloads.etcd }}" + when: etcd_deployment_type == "host" + - role: docker + when: (ansible_os_family != "CoreOS" and etcd_deployment_type == "docker") diff --git a/roles/etcd/tasks/configure.yml b/roles/etcd/tasks/configure.yml index 5f8ab0622..818559239 100644 --- a/roles/etcd/tasks/configure.yml +++ b/roles/etcd/tasks/configure.yml @@ -12,7 +12,7 @@ - name: Configure | Copy etcd.service systemd file template: - src: etcd.service.j2 + src: "etcd-{{ etcd_deployment_type }}.service.j2" dest: /etc/systemd/system/etcd.service backup: yes when: ansible_service_mgr == "systemd" @@ -20,7 +20,7 @@ - name: Configure | Write etcd initd script template: - src: deb-etcd.initd.j2 + src: "deb-etcd-{{ etcd_deployment_type }}.initd.j2" dest: /etc/init.d/etcd owner: root mode: 0755 diff --git a/roles/etcd/tasks/install.yml b/roles/etcd/tasks/install.yml index 56af3b098..09bee1452 100644 --- a/roles/etcd/tasks/install.yml +++ b/roles/etcd/tasks/install.yml @@ -1,9 +1,38 @@ --- -- name: Install | Copy etcd binary +- name: Install | Copy etcd binary from downloaddir command: rsync -piu "{{ etcd_bin_dir }}/etcd" "{{ bin_dir }}/etcd" + when: etcd_deployment_type == "host" register: etcd_copy + +- name: Install | Copy etcdctl binary from downloaddir + command: rsync -piu "{{ etcd_bin_dir }}/etcdctl" "{{ bin_dir }}/etcdctl" + when: etcd_deployment_type == "host" changed_when: false -- name: Install | Copy etcdctl binary - command: rsync -piu "{{ etcd_bin_dir }}/etcdctl" "{{ bin_dir }}/etcdctl" +#Plan A: no docker-py deps +- name: Install | Copy etcdctl binary from container + command: sh -c "/usr/bin/docker rm -f etcdctl-binarycopy; + /usr/bin/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} && + /usr/bin/docker cp etcdctl-binarycopy:{{ etcd_container_bin_dir }}etcdctl {{ bin_dir }}/etcdctl && + /usr/bin/docker rm -f etcdctl-binarycopy" + when: etcd_deployment_type == "docker" changed_when: false + +#Plan B: looks nicer, but requires docker-py on all hosts: +#- name: Install | Set up etcd-binarycopy container +# docker: +# name: etcd-binarycopy +# state: present +# image: "{{ etcd_image_repo }}:{{ etcd_image_tag }}" +# when: etcd_deployment_type == "docker" +# +#- name: Install | Copy etcdctl from etcd-binarycopy container +# command: /usr/bin/docker cp "etcd-binarycopy:{{ etcd_container_bin_dir }}etcdctl" "{{ bin_dir }}/etcdctl" +# when: etcd_deployment_type == "docker" +# +#- name: Install | Clean up etcd-binarycopy container +# docker: +# name: etcd-binarycopy +# state: absent +# image: "{{ etcd_image_repo }}:{{ etcd_image_tag }}" +# when: etcd_deployment_type == "docker" diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index a5ac9da96..060d3708b 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -8,7 +8,7 @@ - name: Restart etcd if binary changed command: /bin/true notify: restart etcd - when: etcd_copy.stdout_lines + when: etcd_deployment_type == "host" and etcd_copy.stdout_lines # Reload systemd before starting service - meta: flush_handlers diff --git a/roles/etcd/tasks/set_facts.yml b/roles/etcd/tasks/set_facts.yml index a7e91a84e..415ed85c6 100644 --- a/roles/etcd/tasks/set_facts.yml +++ b/roles/etcd/tasks/set_facts.yml @@ -14,3 +14,7 @@ {% endfor %} - set_fact: is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}" +- set_fact: + etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=") +- set_fact: + etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}" diff --git a/roles/etcd/templates/deb-etcd-docker.initd.j2 b/roles/etcd/templates/deb-etcd-docker.initd.j2 new file mode 100644 index 000000000..6e5a8cb03 --- /dev/null +++ b/roles/etcd/templates/deb-etcd-docker.initd.j2 @@ -0,0 +1,126 @@ +#!/bin/sh +set -a + +### BEGIN INIT INFO +# Provides: etcd +# Required-Start: $local_fs $network $syslog +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: etcd distributed k/v store +# Description: +# etcd is a distributed, consistent key-value store for shared configuration and service discovery +### END INIT INFO + +PATH=/sbin:/usr/sbin:/bin/:/usr/bin +DESC="etcd k/v store" +NAME=etcd +DAEMON={{ docker_bin_dir | default("/usr/bin") }}/docker +{% if is_etcd_master %} +DAEMON_ARGS='--restart=always --env-file=/etc/etcd.env \ +--net=host \ +-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \ +-v /var/lib/etcd:/var/lib/etcd:rw \ +--name={{ etcd_member_name | default("etcd-proxy") }} \ +{{ etcd_image_repo }}:{{ etcd_image_tag }} \ +{% if etcd_after_v3 %} +{{ etcd_container_bin_dir }}etcd \ +{% endif %} +{% if is_etcd_master %} +--proxy off +{% else %} +--proxy on +{% endif %}' + + +SCRIPTNAME=/etc/init.d/$NAME +DAEMON_USER=etcd +STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}" +PID=/var/run/etcd.pid + +# Exit if the binary is not present +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -f /etc/etcd.env ] && . /etc/etcd.env + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +do_status() +{ + status_of_proc -p $PID "$DAEMON" "$NAME" && exit 0 || exit $? +} + +# Function that starts the daemon/service +# +do_start() +{ + start-stop-daemon --background --start --quiet --make-pidfile --pidfile $PID --user $DAEMON_USER --exec $DAEMON -- \ + $DAEMON_ARGS \ + || return 2 +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + start-stop-daemon --stop --quiet --retry=$STOP_SCHEDULE --pidfile $PID --name $NAME + RETVAL="$?" + + sleep 1 + return "$RETVAL" +} + + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + if do_stop; then + log_end_msg 0 + else + log_failure_msg "Can't stop etcd" + log_end_msg 1 + fi + ;; + status) + if do_status; then + log_end_msg 0 + else + log_failure_msg "etcd is not running" + log_end_msg 1 + fi + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + if do_stop; then + if do_start; then + log_end_msg 0 + exit 0 + else + rc="$?" + fi + else + rc="$?" + fi + log_failure_msg "Can't restart etcd" + log_end_msg ${rc} + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + diff --git a/roles/etcd/templates/deb-etcd.initd.j2 b/roles/etcd/templates/deb-etcd-host.initd.j2 similarity index 97% rename from roles/etcd/templates/deb-etcd.initd.j2 rename to roles/etcd/templates/deb-etcd-host.initd.j2 index ec3d9373d..ccecdce4e 100644 --- a/roles/etcd/templates/deb-etcd.initd.j2 +++ b/roles/etcd/templates/deb-etcd-host.initd.j2 @@ -16,10 +16,10 @@ PATH=/sbin:/usr/sbin:/bin:/usr/bin DESC="etcd k/v store" NAME=etcd DAEMON={{ bin_dir }}/etcd -{% if inventory_hostname in groups['etcd'] %} +{% if is_etcd_master %} DAEMON_ARGS="" {% else %} -DAEMON_ARGS="-proxy on" +DAEMON_ARGS="--proxy on" {% endif %} SCRIPTNAME=/etc/init.d/$NAME DAEMON_USER=etcd @@ -111,3 +111,4 @@ case "$1" in exit 3 ;; esac + diff --git a/roles/etcd/templates/etcd-docker.service.j2 b/roles/etcd/templates/etcd-docker.service.j2 new file mode 100644 index 000000000..623ec70b9 --- /dev/null +++ b/roles/etcd/templates/etcd-docker.service.j2 @@ -0,0 +1,33 @@ +[Unit] +Description=etcd docker wrapper +Requires=docker.service +After=docker.service + +[Service] +User=root +PermissionsStartOnly=true +ExecStart={{ docker_bin_dir | default("/usr/bin") }}/docker run --restart=always \ +--env-file=/etc/etcd.env \ +{# TODO(mattymo): Allow docker IP binding and disable in envfile + -p 2380:2380 -p 2379:2379 #} +--net=host \ +-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \ +-v /var/lib/etcd:/var/lib/etcd:rw \ +--name={{ etcd_member_name | default("etcd-proxy") }} \ +{{ etcd_image_repo }}:{{ etcd_image_tag }} \ +{% if etcd_after_v3 %} +{{ etcd_container_bin_dir }}etcd \ +{% endif %} +{% if is_etcd_master %} +--proxy off +{% else %} +--proxy on +{% endif %} +ExecStartPre=-{{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_member_name | default("etcd-proxy") }} +ExecReload={{ docker_bin_dir | default("/usr/bin") }}/docker restart {{ etcd_member_name | default("etcd-proxy") }} +ExecStop={{ docker_bin_dir | default("/usr/bin") }}/docker stop {{ etcd_member_name | default("etcd-proxy") }} +Restart=always +RestartSec=10s + +[Install] +WantedBy=multi-user.target diff --git a/roles/etcd/templates/etcd.service.j2 b/roles/etcd/templates/etcd-host.service.j2 similarity index 100% rename from roles/etcd/templates/etcd.service.j2 rename to roles/etcd/templates/etcd-host.service.j2 diff --git a/roles/etcd/templates/etcd.j2 b/roles/etcd/templates/etcd.j2 index 49f0c83a0..94d9e8121 100644 --- a/roles/etcd/templates/etcd.j2 +++ b/roles/etcd/templates/etcd.j2 @@ -1,12 +1,14 @@ -ETCD_DATA_DIR="/var/lib/etcd" -{% if inventory_hostname in groups['etcd'] %} -ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])) }}:2379" -ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])) }}:2380" -ETCD_INITIAL_CLUSTER_STATE="{% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}" -ETCD_INITIAL_CLUSTER_TOKEN="k8s_etcd" -ETCD_LISTEN_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}:2380" -ETCD_NAME="{{ etcd_member_name }}" -{% endif %} +ETCD_DATA_DIR=/var/lib/etcd +{% if is_etcd_master %} +ETCD_ADVERTISE_CLIENT_URLS=http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])) }}:2379 +ETCD_INITIAL_ADVERTISE_PEER_URLS=http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])) }}:2380 +ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %} -ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}={{ hostvars[host]['etcd_peer_url'] }}{% if not loop.last %},{% endif %}{% endfor %}" -ETCD_LISTEN_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}:2379,http://127.0.0.1:2379" +ETCD_ELECTION_TIMEOUT=10000 +ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd +ETCD_LISTEN_PEER_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}:2380 +ETCD_NAME={{ etcd_member_name }} +{% endif %} +ETCD_INITIAL_CLUSTER={% for host in groups['etcd'] %}etcd{{ loop.index|string }}={{ hostvars[host]['etcd_peer_url'] }}{% if not loop.last %},{% endif %}{% endfor %} + +ETCD_LISTEN_CLIENT_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}:2379,http://127.0.0.1:2379