From e2984b4fdb7d69571346663c965bc1686a050493 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Tue, 15 Dec 2015 11:49:11 +0100 Subject: [PATCH] ha etcd with calico --- cluster.yml | 6 +----- roles/etcd/handlers/main.yml | 19 +++++++++---------- roles/etcd/tasks/configure.yml | 11 +++++++++-- roles/etcd/tasks/install.yml | 10 +--------- roles/etcd/templates/etcd2.j2 | 11 ++++++++--- roles/etcd/templates/systemd-etcd2.service.j2 | 4 ++++ roles/network_plugin/tasks/calico.yml | 2 +- .../templates/network-environment.j2 | 8 +++++--- 8 files changed, 38 insertions(+), 33 deletions(-) diff --git a/cluster.yml b/cluster.yml index d411f381c..ef91f27ac 100644 --- a/cluster.yml +++ b/cluster.yml @@ -4,13 +4,9 @@ roles: - { role: download, tags: download } -# etcd must be running on master(s) before going on -- hosts: etcd - roles: - - { role: etcd, tags: etcd } - - hosts: k8s-cluster roles: + - { role: etcd, tags: etcd } - { role: docker, tags: docker } - { role: dnsmasq, tags: dnsmasq } - { role: network_plugin, tags: ['calico', 'flannel', 'network'] } diff --git a/roles/etcd/handlers/main.yml b/roles/etcd/handlers/main.yml index af2442abf..67334a353 100644 --- a/roles/etcd/handlers/main.yml +++ b/roles/etcd/handlers/main.yml @@ -1,15 +1,14 @@ --- -- name: restart daemons - command: /bin/true - notify: - - reload systemd - - restart etcd2 - - name: reload systemd command: systemctl daemon-reload -- name: restart etcd2 - service: name=etcd2 state=restarted +- name: restart reloaded-etcd2 + service: + name: etcd2 + state: restarted -- name: Save iptables rules - command: service iptables save +- name: restart etcd2 + command: /bin/true + notify: + - reload systemd + - restart reloaded-etcd2 diff --git a/roles/etcd/tasks/configure.yml b/roles/etcd/tasks/configure.yml index 18a2cc882..5aea90f44 100644 --- a/roles/etcd/tasks/configure.yml +++ b/roles/etcd/tasks/configure.yml @@ -1,11 +1,18 @@ --- +- name: Copy etcd2.service systemd file + template: + src: systemd-etcd2.service.j2 + dest: /lib/systemd/system/etcd2.service + backup: yes + notify: + - restart etcd2 + - name: Create etcd2 environment vars dir file: path=/etc/systemd/system/etcd2.service.d state=directory - name: Write etcd2 config file - template: src=etcd2.j2 dest=/etc/systemd/system/etcd2.service.d/10-etcd2-cluster.conf backup=yes + template: src=etcd2.j2 dest=/etc/systemd/system/etcd2.service.d/10-etcd2.conf backup=yes notify: - - reload systemd - restart etcd2 - name: Ensure etcd2 is running diff --git a/roles/etcd/tasks/install.yml b/roles/etcd/tasks/install.yml index b500d88ed..f02dc93db 100644 --- a/roles/etcd/tasks/install.yml +++ b/roles/etcd/tasks/install.yml @@ -11,15 +11,7 @@ with_items: - etcdctl - etcd - notify: - - restart daemons + notify: restart etcd2 - name: Create etcd2 binary symlink file: src=/usr/local/bin/etcd dest=/usr/local/bin/etcd2 state=link - -- name: Copy etcd2.service systemd file - template: - src: systemd-etcd2.service.j2 - dest: /lib/systemd/system/etcd2.service - backup: yes - notify: restart daemons diff --git a/roles/etcd/templates/etcd2.j2 b/roles/etcd/templates/etcd2.j2 index cb3305287..a00fb72e2 100644 --- a/roles/etcd/templates/etcd2.j2 +++ b/roles/etcd/templates/etcd2.j2 @@ -1,16 +1,21 @@ # etcd2.0 +[Service] +{% if inventory_hostname in groups['etcd'] %} {% set etcd = {} %} -{% for srv in groups['kube-master'] %} +{% for srv in groups['etcd'] %} {% if inventory_hostname == srv %} {% set _dummy = etcd.update({'name':"master"+loop.index|string}) %} {% endif %} {% endfor %} -[Service] Environment="ETCD_ADVERTISE_CLIENT_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379" Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380" -Environment="ETCD_INITIAL_CLUSTER={% for srv in groups['kube-master'] %}master{{ loop.index|string }}=http://{{ srv }}:2380{% if not loop.last %},{% endif %}{% endfor %}" +Environment="ETCD_INITIAL_CLUSTER={% for srv in groups['etcd'] %}master{{ loop.index|string }}=http://{{ srv }}:2380{% if not loop.last %},{% endif %}{% endfor %}" Environment="ETCD_INITIAL_CLUSTER_STATE=new" Environment="ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd" Environment="ETCD_LISTEN_CLIENT_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379,http://127.0.0.1:2379" Environment="ETCD_LISTEN_PEER_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380" Environment="ETCD_NAME={{ etcd.name }}" +{% else %} +Environment="ETCD_INITIAL_CLUSTER={% for srv in groups['etcd'] %}master{{ loop.index|string }}=http://{{ srv }}:2380{% if not loop.last %},{% endif %}{% endfor %}" +Environment="ETCD_LISTEN_CLIENT_URLS=http://127.0.0.1:23799" +{% endif %} diff --git a/roles/etcd/templates/systemd-etcd2.service.j2 b/roles/etcd/templates/systemd-etcd2.service.j2 index 26cda24eb..84a527d04 100644 --- a/roles/etcd/templates/systemd-etcd2.service.j2 +++ b/roles/etcd/templates/systemd-etcd2.service.j2 @@ -6,7 +6,11 @@ Conflicts=etcd.service User=etcd Environment=ETCD_DATA_DIR=/var/lib/etcd2 Environment=ETCD_NAME=%m +{% if inventory_hostname in groups['etcd'] %} ExecStart={{ bin_dir }}/etcd2 +{% else %} +ExecStart={{ bin_dir }}/etcd2 -proxy on +{% endif %} Restart=always RestartSec=10s LimitNOFILE=40000 diff --git a/roles/network_plugin/tasks/calico.yml b/roles/network_plugin/tasks/calico.yml index c507d66e1..eba8967d1 100644 --- a/roles/network_plugin/tasks/calico.yml +++ b/roles/network_plugin/tasks/calico.yml @@ -12,7 +12,7 @@ - name: Calico | Configure calico-node desired pool shell: calicoctl pool add {{ kube_pods_subnet }} environment: - ETCD_AUTHORITY: "{{ loadbalancer_address | default(groups['kube-master'][0]) }}:2379" + ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379" run_once: true - name: Calico | Write calico-node systemd init file diff --git a/roles/network_plugin/templates/network-environment.j2 b/roles/network_plugin/templates/network-environment.j2 index 53ab5f15a..5793e8818 100755 --- a/roles/network_plugin/templates/network-environment.j2 +++ b/roles/network_plugin/templates/network-environment.j2 @@ -4,20 +4,22 @@ CALICO_IPAM=true DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }} -{% if inventory_hostname in groups['kube-node'] %} # The kubernetes master IP KUBERNETES_MASTER={{ groups['kube-master'][0] }} # Location of etcd cluster used by Calico. By default, this uses the etcd # instance running on the Kubernetes Master -ETCD_AUTHORITY={{ loadbalancer_address | default(groups['kube-master'][0]) }}:2379 +{% if inventory_hostname in groups['etcd'] %} +ETCD_AUTHORITY="127.0.0.1:2379" +{% else %} +ETCD_AUTHORITY="127.0.0.1:23799" +{% endif %} # The kubernetes-apiserver location - used by the calico plugin KUBE_API_ROOT=http://{{ groups['kube-master'][0] }}:{{kube_apiserver_insecure_port}}/api/v1/ # Location of the calicoctl binary - used by the calico plugin CALICOCTL_PATH="{{ bin_dir }}/calicoctl" -{% endif %} {% else %} FLANNEL_ETCD_PREFIX="--etcd-prefix=/{{ cluster_name }}/network" {% endif %}