etcd directly in host

fix etcd configuration for nodes

fix wrong calico checksums

using a var name etcd_bin_dir

fix etcd handlers for sysvinit

using a var name etcd_bin_dir

sysvinit script

review etcd configuration

README.md

@@ -24,7 +24,7 @@ in order to avoid any issue during deployment you should **disable your firewall
 
 ### Components
 * [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.1.4
-* [etcd](https://github.com/coreos/etcd/releases) v2.2.2
+* [etcd](https://github.com/coreos/etcd/releases) v2.2.4
 * [calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.14.0
 * [flanneld](https://github.com/coreos/flannel/releases) v0.5.5
 * [docker](https://www.docker.com/) v1.9.1
@@ -255,7 +255,7 @@ That way is easier if you want to do some changes and commit them.
 
 ### Networking
 
-#### Calico networking
+#### Calico
 Check if the calico-node container is running
 ```
 docker ps | grep calico
@@ -277,7 +277,7 @@ calicoctl pool show
 calicoctl endpoint show --detail
 ```
 
-#### Flannel networking
+#### Flannel
 
 * Flannel configuration file should have been created there
 ```

roles/download/defaults/main.yml

@@ -1,27 +1,41 @@
 ---
 local_release_dir: /tmp
 
+# Versions
+kube_version: v1.1.4
+etcd_version: v2.2.4
 calico_version: v0.14.0
 calico_plugin_version: v0.7.0
-kube_version: v1.1.4
 
+# Download URL's
+kube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64"
+etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
+calico_download_url: "https://github.com/Metaswitch/calico-docker/releases/download/{{calico_version}}/calicoctl"
+calico_plugin_download_url: "https://github.com/projectcalico/calico-kubernetes/releases/download/{{calico_plugin_version}}/calico_kubernetes"
+
+# Checksums
+calico_checksum: "f251d7a8583233906aa6d059447c1e4fb32bf1369a51fdf96a68d50466d6a69c"
+calico_plugin_checksum: "032f582f5eeec6fb26191d2fbcbf8bca4da3b14abb579db7baa7b3504d4dffec"
+etcd_checksum: "6c4e5cdeaaac1a70b8f06b5dd6b82c37ff19993c9bca81248975610e555c4b9b"
 kubectl_checksum: "873ba19926d17a3287dc8639ea1434fe3cd0cb4e61d82101ba754922cfc7a633"
 kubelet_checksum: "f2d1eae3fa6e304f6cbc9b2621e4b86fc3bcb4e74a15d35f58bf00e45c706e0a"
 
-kube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64"
-
-calico_download_url: "https://github.com/Metaswitch/calico-docker/releases/download/{{calico_version}}/calicoctl"
-
-calico_plugin_download_url: "https://github.com/projectcalico/calico-kubernetes/releases/download/{{calico_plugin_version}}/calico_kubernetes"
-
 downloads:
   - name: calico
     dest: calico/bin/calicoctl
-    url: "{{calico_download_url}}"
+    sha256: "{{ calico_checksum }}"
+    url: "{{ calico_download_url }}"
 
   - name: calico-plugin
     dest: calico/bin/calico
-    url: "{{calico_plugin_download_url}}"
+    sha256: "{{ calico_plugin_checksum }}"
+    url: "{{ calico_plugin_download_url }}"
+
+  - name: etcd
+    dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
+    sha256: "{{ etcd_checksum }}"
+    url: "{{ etcd_download_url }}"
+    unarchive: true
 
   - name: kubernetes-kubelet
     dest: kubernetes/bin/kubelet

roles/etcd/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+etcd_version: v2.2.4
+etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"

roles/etcd/handlers/main.yml

@@ -0,0 +1,14 @@
+---
+- name: restart systemd-etcd
+  command: /bin/true
+  notify:
+    - reload systemd
+    - restart etcd
+
+- name: reload systemd
+  command: systemctl daemon-reload
+
+- name: restart etcd
+  service:
+    name: etcd
+    state: restarted

roles/etcd/tasks/configure.yml

@@ -0,0 +1,27 @@
+---
+- name: Copy etcd.service systemd file
+  template:
+    src: etcd.service.j2
+    dest: /lib/systemd/system/etcd.service
+    backup: yes
+  when: init_system == "systemd"
+  notify:
+    - restart systemd-etcd
+
+- name: Write calico-node initd script
+  template:
+    src: deb-etcd.initd.j2
+    dest: /etc/init.d/etcd
+    owner: root
+    mode: 755
+  when: init_system == "sysvinit" and ansible_os_family == "Debian"
+  notify: restart etcd
+
+- name: Create etcd environment vars file
+  template:
+    src: etcd.j2
+    dest: /etc/etcd.env
+  notify: restart etcd
+
+- name: Ensure etcd is running
+  service: name=etcd state=started enabled=yes

roles/etcd/tasks/install.yml

@@ -0,0 +1,14 @@
+---
+- name: Create etcd user
+  user: name=etcd shell=/bin/nologin home=/var/lib/etcd
+
+- name: Install etcd binaries
+  copy:
+     src={{ etcd_bin_dir }}/{{ item }}
+     dest={{ bin_dir }}
+     owner=etcd
+     mode=0755
+  with_items:
+    - etcdctl
+    - etcd
+  notify: restart etcd

roles/etcd/tasks/main.yml

@@ -1,12 +1,3 @@
 ---
-- name: Stop etcd2 service
-  service: name=etcd state=stopped
-  ignore_errors: yes
-
-- name: Create etcd pod manifest
-  template: src=etcd-pod.yml dest=/etc/kubernetes/manifests/etcd-pod.manifest
-
-- name: Check for etcd2 port (pulling image and running container)
-  wait_for:
-    port: 2379
-    delay: 5
+- include: install.yml
+- include: configure.yml

roles/etcd/templates/deb-etcd.initd.j2

@@ -0,0 +1,113 @@
+#!/bin/sh
+set -a
+
+### BEGIN INIT INFO
+# Provides:   etcd
+# Required-Start:    $local_fs $network $syslog
+# Required-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: etcd distributed k/v store
+# Description:
+#   etcd is a distributed, consistent key-value store for shared configuration and service discovery
+### END INIT INFO
+
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="etcd k/v store"
+NAME=etcd
+DAEMON={{ bin_dir }}/etcd
+{% if inventory_hostname in groups['etcd'] %}
+DAEMON_ARGS=""
+{% else %}
+DAEMON_ARGS="-proxy on"
+{% endif %}
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMON_USER=etcd
+STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}"
+PID=/var/run/etcd.pid
+
+# Exit if the binary is not present
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -f /etc/etcd.env ] && . /etc/etcd.env
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
+# and status_of_proc is working.
+. /lib/lsb/init-functions
+
+do_status()
+{
+    status_of_proc -p $PID "$DAEMON" "$NAME" && exit 0 || exit $?
+}
+
+# Function that starts the daemon/service
+#
+do_start()
+{
+    start-stop-daemon --background --start --quiet --make-pidfile --pidfile $PID --user $DAEMON_USER --exec $DAEMON \
+        $DAEMON_OPTS \
+        || return 2
+}
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+    start-stop-daemon --stop --quiet --retry=$STOP_SCHEDULE --pidfile $PID --name $NAME
+    RETVAL="$?"
+
+    sleep 1
+    return "$RETVAL"
+}
+
+
+case "$1" in
+  start)
+        log_daemon_msg "Starting $DESC" "$NAME"
+        do_start
+        case "$?" in
+                0|1) log_end_msg 0 || exit 0 ;;
+                2) log_end_msg 1 || exit 1 ;;
+        esac
+        ;;
+  stop)
+        log_daemon_msg "Stopping $DESC" "$NAME"
+        if do_stop; then
+            log_end_msg 0
+        else
+            log_failure_msg "Can't stop etcd"
+            log_end_msg 1
+        fi
+        ;;
+  status)
+        if do_status; then
+            log_end_msg 0
+        else
+            log_failure_msg "etcd is not running"
+            log_end_msg 1
+        fi
+        ;;
+
+  restart|force-reload)
+        log_daemon_msg "Restarting $DESC" "$NAME"
+        if do_stop; then
+            if do_start; then
+                log_end_msg 0
+                exit 0
+            else
+                rc="$?"
+            fi
+        else
+           rc="$?"
+        fi
+        log_failure_msg "Can't restart etcd"
+        log_end_msg ${rc}
+        ;;
+  *)
+        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
+        exit 3
+        ;;
+esac

roles/etcd/templates/etcd-pod.yml

@@ -1,54 +0,0 @@
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: etcd
-  namespace: kube-system
-spec:
-  hostNetwork: true
-  containers:
-    - name: etcd
-      image: quay.io/coreos/etcd:v2.2.2
-      resources:
-        limits:
-          cpu: 100m
-          memory: 256M
-      args:
-{% if inventory_hostname in groups['etcd'] %}
-        - --name
-        - etcd-{{inventory_hostname}}-master
-        - --advertise-client-urls
-        - "http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379"
-        - --listen-peer-urls
-        - http://0.0.0.0:2380
-        - --initial-advertise-peer-urls
-        - http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380
-        - --data-dir
-        - /var/etcd/data
-        - --initial-cluster-state
-        - new
-{% else %}
-        - --proxy
-        - 'on'
-{% endif %}
-        - --listen-client-urls
-        - "http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2379,http://127.0.0.1:2379"
-        - --initial-cluster
-        - "{% for host in groups['etcd'] %}etcd-{{host}}-master=http://{{ hostvars[host]['ip'] | default( hostvars[host]['ansible_default_ipv4']['address'])   }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
-        - --initial-cluster-token
-        - etcd-k8s-cluster
-      ports:
-        - name: etcd-client
-          containerPort: 2379
-          hostPort: 2379
-        - name: etcd-peer
-          containerPort: 2380
-          hostPort: 2380
-      volumeMounts:
-        - name: varetcd
-          mountPath: /var/etcd
-          readOnly: false
-  volumes:
-    - name: varetcd
-      hostPath:
-        path: /containers/pods/etcd-{{inventory_hostname}}/rootfs/var/etcd

roles/etcd/templates/etcd.j2

@@ -0,0 +1,17 @@
+ETCD_DATA_DIR="/var/lib/etcd"
+{% if inventory_hostname in groups['etcd'] %}
+{% set etcd = {} %}
+{%     for host in groups['etcd'] %}
+{%         if inventory_hostname == host %}
+{%             set _dummy = etcd.update({'name':"etcd"+loop.index|string}) %}
+{%         endif %}
+{%     endfor %}
+ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379"
+ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2380"
+ETCD_INITIAL_CLUSTER_STATE="new"
+ETCD_INITIAL_CLUSTER_TOKEN="k8s_etcd"
+ETCD_LISTEN_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2380"
+ETCD_NAME="{{ etcd.name }}"
+{% endif %}
+ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}=http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
+ETCD_LISTEN_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2379,http://127.0.0.1:2379"

roles/etcd/templates/etcd.service.j2

@@ -0,0 +1,18 @@
+[Unit]
+Description=etcd
+Conflicts=etcd.service
+
+[Service]
+User=etcd
+EnvironmentFile=/etc/etcd.env
+{% if inventory_hostname in groups['etcd'] %}
+ExecStart={{ bin_dir }}/etcd
+{% else %}
+ExecStart={{ bin_dir }}/etcd -proxy on
+{% endif %}
+Restart=always
+RestartSec=10s
+LimitNOFILE=40000
+
+[Install]
+WantedBy=multi-user.target