Add optional deployment mode for Docker etcd_deployment_type

Running etcd in Docker reduces the number of individual file
downloads and services running on the host.

Note: etcd container v3.0.1 moves bindir to /usr/local/bin

Fixes: #298

roles/etcd/defaults/main.yml

@@ -1,3 +1,10 @@
 ---
 etcd_version: v3.0.1
 etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"
+
+# Possible values: host, docker
+etcd_deployment_type: "host"
+
+
+etcd_image_repo: "quay.io/coreos/etcd"
+etcd_image_tag: "{{ etcd_version }}"

roles/etcd/handlers/main.yml

@@ -3,13 +3,20 @@
   command: /bin/true
   notify:
     - reload systemd
+    - start etcd
     - reload etcd
 
 - name: reload systemd
   command: systemctl daemon-reload
   when: ansible_service_mgr == "systemd"
 
+- name: start etcd
+  service:
+    name: etcd
+    enabled: yes
+    state: started
+
 - name: reload etcd
   service:
     name: etcd
-    state: restarted
+    state: "{{ 'restarted' if etcd_deployment_type == 'host' else 'reloaded' }}"

roles/etcd/meta/main.yml

@@ -5,3 +5,6 @@ dependencies:
     when: ansible_os_family != 'CoreOS'
   - role: download
     file: "{{ downloads.etcd }}"
+    when: etcd_deployment_type == "host"
+  - role: docker
+    when: (ansible_os_family != "CoreOS" and etcd_deployment_type == "docker")

roles/etcd/tasks/configure.yml

@@ -12,7 +12,7 @@
 
 - name: Configure | Copy etcd.service systemd file
   template:
-    src: etcd.service.j2
+    src: "etcd-{{ etcd_deployment_type }}.service.j2"
     dest: /etc/systemd/system/etcd.service
     backup: yes
   when: ansible_service_mgr == "systemd"
@@ -20,7 +20,7 @@
 
 - name: Configure | Write etcd initd script
   template:
-    src: deb-etcd.initd.j2
+    src: "deb-etcd-{{ etcd_deployment_type }}.initd.j2"
     dest: /etc/init.d/etcd
     owner: root
     mode: 0755

roles/etcd/tasks/install.yml

@@ -1,9 +1,38 @@
 ---
-- name: Install | Copy etcd binary
+- name: Install | Copy etcd binary from downloaddir
   command: rsync -piu "{{ etcd_bin_dir }}/etcd" "{{ bin_dir }}/etcd"
+  when: etcd_deployment_type == "host"
   register: etcd_copy
+
+- name: Install | Copy etcdctl binary from downloaddir
+  command: rsync -piu "{{ etcd_bin_dir }}/etcdctl" "{{ bin_dir }}/etcdctl"
+  when: etcd_deployment_type == "host"
   changed_when: false
 
-- name: Install | Copy etcdctl binary
+#Plan A: no docker-py deps
-  command: rsync -piu "{{ etcd_bin_dir }}/etcdctl" "{{ bin_dir }}/etcdctl"
+- name: Install | Copy etcdctl binary from container
+  command: sh -c "/usr/bin/docker rm -f etcdctl-binarycopy;
+           /usr/bin/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} &&
+           /usr/bin/docker cp etcdctl-binarycopy:{{ etcd_container_bin_dir }}etcdctl {{ bin_dir }}/etcdctl &&
+           /usr/bin/docker rm -f etcdctl-binarycopy"
+  when: etcd_deployment_type == "docker"
   changed_when: false
+
+#Plan B: looks nicer, but requires docker-py on all hosts:
+#- name: Install | Set up etcd-binarycopy container
+#  docker:
+#    name: etcd-binarycopy
+#    state: present
+#    image: "{{ etcd_image_repo }}:{{ etcd_image_tag }}"
+#  when: etcd_deployment_type == "docker"
+#
+#- name: Install | Copy etcdctl from etcd-binarycopy container
+#  command: /usr/bin/docker cp "etcd-binarycopy:{{ etcd_container_bin_dir }}etcdctl" "{{ bin_dir }}/etcdctl"
+#  when: etcd_deployment_type == "docker"
+#
+#- name: Install | Clean up etcd-binarycopy container
+#  docker:
+#    name: etcd-binarycopy
+#    state: absent
+#    image: "{{ etcd_image_repo }}:{{ etcd_image_tag }}"
+#  when: etcd_deployment_type == "docker"

roles/etcd/tasks/main.yml

@@ -8,7 +8,7 @@
 - name: Restart etcd if binary changed
   command: /bin/true
   notify: restart etcd
-  when: etcd_copy.stdout_lines
+  when: etcd_deployment_type == "host" and etcd_copy.stdout_lines
 
 # Reload systemd before starting service
 - meta: flush_handlers

roles/etcd/tasks/set_facts.yml

@@ -14,3 +14,7 @@
       {% endfor %}
 - set_fact:
     is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}"
+- set_fact:
+    etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=")
+- set_fact:
+    etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}"

roles/etcd/templates/deb-etcd-docker.initd.j2

@@ -0,0 +1,126 @@
+#!/bin/sh
+set -a
+
+### BEGIN INIT INFO
+# Provides:   etcd
+# Required-Start:    $local_fs $network $syslog
+# Required-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: etcd distributed k/v store
+# Description:
+#   etcd is a distributed, consistent key-value store for shared configuration and service discovery
+### END INIT INFO
+
+PATH=/sbin:/usr/sbin:/bin/:/usr/bin
+DESC="etcd k/v store"
+NAME=etcd
+DAEMON={{ docker_bin_dir | default("/usr/bin") }}/docker
+{% if is_etcd_master %}
+DAEMON_ARGS='--restart=always --env-file=/etc/etcd.env \
+--net=host \
+-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
+-v /var/lib/etcd:/var/lib/etcd:rw \
+--name={{ etcd_member_name | default("etcd-proxy") }} \
+{{ etcd_image_repo }}:{{ etcd_image_tag }} \
+{% if etcd_after_v3 %}
+{{ etcd_container_bin_dir }}etcd \
+{% endif %}
+{% if is_etcd_master %}
+--proxy off
+{% else %}
+--proxy on
+{% endif %}'
+
+
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMON_USER=etcd
+STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}"
+PID=/var/run/etcd.pid
+
+# Exit if the binary is not present
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -f /etc/etcd.env ] && . /etc/etcd.env
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
+# and status_of_proc is working.
+. /lib/lsb/init-functions
+
+do_status()
+{
+    status_of_proc -p $PID "$DAEMON" "$NAME" && exit 0 || exit $?
+}
+
+# Function that starts the daemon/service
+#
+do_start()
+{
+    start-stop-daemon --background --start --quiet --make-pidfile --pidfile $PID --user $DAEMON_USER --exec $DAEMON -- \
+        $DAEMON_ARGS \
+        || return 2
+}
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+    start-stop-daemon --stop --quiet --retry=$STOP_SCHEDULE --pidfile $PID --name $NAME
+    RETVAL="$?"
+
+    sleep 1
+    return "$RETVAL"
+}
+
+
+case "$1" in
+  start)
+        log_daemon_msg "Starting $DESC" "$NAME"
+        do_start
+        case "$?" in
+                0|1) log_end_msg 0 || exit 0 ;;
+                2) log_end_msg 1 || exit 1 ;;
+        esac
+        ;;
+  stop)
+        log_daemon_msg "Stopping $DESC" "$NAME"
+        if do_stop; then
+            log_end_msg 0
+        else
+            log_failure_msg "Can't stop etcd"
+            log_end_msg 1
+        fi
+        ;;
+  status)
+        if do_status; then
+            log_end_msg 0
+        else
+            log_failure_msg "etcd is not running"
+            log_end_msg 1
+        fi
+        ;;
+
+  restart|force-reload)
+        log_daemon_msg "Restarting $DESC" "$NAME"
+        if do_stop; then
+            if do_start; then
+                log_end_msg 0
+                exit 0
+            else
+                rc="$?"
+            fi
+        else
+           rc="$?"
+        fi
+        log_failure_msg "Can't restart etcd"
+        log_end_msg ${rc}
+        ;;
+  *)
+        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
+        exit 3
+        ;;
+esac
+

roles/etcd/templates/deb-etcd-host.initd.j2

@@ -16,10 +16,10 @@ PATH=/sbin:/usr/sbin:/bin:/usr/bin
 DESC="etcd k/v store"
 NAME=etcd
 DAEMON={{ bin_dir }}/etcd
-{% if inventory_hostname in groups['etcd'] %}
+{% if is_etcd_master %}
 DAEMON_ARGS=""
 {% else %}
-DAEMON_ARGS="-proxy on"
+DAEMON_ARGS="--proxy on"
 {% endif %}
 SCRIPTNAME=/etc/init.d/$NAME
 DAEMON_USER=etcd
@@ -111,3 +111,4 @@ case "$1" in
         exit 3
         ;;
 esac
+

roles/etcd/templates/etcd-docker.service.j2

@@ -0,0 +1,33 @@
+[Unit]
+Description=etcd docker wrapper
+Requires=docker.service
+After=docker.service
+
+[Service]
+User=root
+PermissionsStartOnly=true
+ExecStart={{ docker_bin_dir | default("/usr/bin") }}/docker run --restart=always \
+--env-file=/etc/etcd.env \
+{# TODO(mattymo): Allow docker IP binding and disable in envfile
+   -p 2380:2380 -p 2379:2379 #}
+--net=host \
+-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
+-v /var/lib/etcd:/var/lib/etcd:rw \
+--name={{ etcd_member_name | default("etcd-proxy") }} \
+{{ etcd_image_repo }}:{{ etcd_image_tag }} \
+{% if etcd_after_v3 %}
+{{ etcd_container_bin_dir }}etcd \
+{% endif %}
+{% if is_etcd_master %}
+--proxy off
+{% else %}
+--proxy on
+{% endif %}
+ExecStartPre=-{{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_member_name | default("etcd-proxy") }}
+ExecReload={{ docker_bin_dir | default("/usr/bin") }}/docker restart {{ etcd_member_name | default("etcd-proxy") }}
+ExecStop={{ docker_bin_dir | default("/usr/bin") }}/docker stop {{ etcd_member_name | default("etcd-proxy") }}
+Restart=always
+RestartSec=10s
+
+[Install]
+WantedBy=multi-user.target

roles/etcd/templates/etcd.j2

@@ -1,12 +1,14 @@
-ETCD_DATA_DIR="/var/lib/etcd"
+ETCD_DATA_DIR=/var/lib/etcd
-{% if inventory_hostname in groups['etcd'] %}
+{% if is_etcd_master %}
-ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])) }}:2379"
+ETCD_ADVERTISE_CLIENT_URLS=http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])) }}:2379
-ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address']))  }}:2380"
+ETCD_INITIAL_ADVERTISE_PEER_URLS=http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address']))  }}:2380
-ETCD_INITIAL_CLUSTER_STATE="{% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}"
+ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}
-ETCD_INITIAL_CLUSTER_TOKEN="k8s_etcd"
-ETCD_LISTEN_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])  }}:2380"
-ETCD_NAME="{{ etcd_member_name }}"
-{% endif %}
 
-ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}={{ hostvars[host]['etcd_peer_url'] }}{% if not loop.last %},{% endif %}{% endfor %}"
+ETCD_ELECTION_TIMEOUT=10000
-ETCD_LISTEN_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])  }}:2379,http://127.0.0.1:2379"
+ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
+ETCD_LISTEN_PEER_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])  }}:2380
+ETCD_NAME={{ etcd_member_name }}
+{% endif %}
+ETCD_INITIAL_CLUSTER={% for host in groups['etcd'] %}etcd{{ loop.index|string }}={{ hostvars[host]['etcd_peer_url'] }}{% if not loop.last %},{% endif %}{% endfor %}
+
+ETCD_LISTEN_CLIENT_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])  }}:2379,http://127.0.0.1:2379