From e7e03bae9fba2e6bd633b7fdfb1542eb34f6286a Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Fri, 18 Dec 2015 22:22:52 +0100 Subject: [PATCH 01/13] calico talks to apiserver with https --- .../manifests/kube-apiserver.manifest.j2 | 1 - roles/kubernetes/node/tasks/secrets.yml | 26 +++++++++++++++++++ .../templates/network-environment.j2 | 2 +- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 940ec1ace..0d8cfb026 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -10,7 +10,6 @@ spec: command: - /hyperkube - apiserver - - --insecure-bind-address=0.0.0.0 - --etcd-servers={% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %} - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota diff --git a/roles/kubernetes/node/tasks/secrets.yml b/roles/kubernetes/node/tasks/secrets.yml index 3d0c76734..5154b9b59 100644 --- a/roles/kubernetes/node/tasks/secrets.yml +++ b/roles/kubernetes/node/tasks/secrets.yml @@ -21,6 +21,32 @@ run_once: true when: inventory_hostname == groups['kube-master'][0] +- name: tokens | generate tokens for calico + command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}" + environment: + TOKEN_DIR: "{{ kube_token_dir }}" + with_nested: + - [ "system:calico" ] + - "{{ groups['k8s-cluster'] }}" + register: gentoken + changed_when: "'Added' in gentoken.stdout" + when: kube_network_plugin == "calico" + delegate_to: "{{ groups['kube-master'][0] }}" + +- name: tokens | get the calico token values + slurp: + src: "{{ kube_token_dir }}/system:calico-{{ inventory_hostname }}.token" + register: calico_token + when: kube_network_plugin == "calico" + delegate_to: "{{ groups['kube-master'][0] }}" + +- name: tokens | Add KUBE_AUTH_TOKEN for calico + lineinfile: + regexp: "^KUBE_AUTH_TOKEN=.*$" + line: "KUBE_AUTH_TOKEN={{ calico_token.content|b64decode }}" + dest: "/etc/network-environment" + when: kube_network_plugin == "calico" + # Sync certs between nodes - user: name: '{{ansible_user_id}}' diff --git a/roles/network_plugin/templates/network-environment.j2 b/roles/network_plugin/templates/network-environment.j2 index b926c8cf2..0aaf4bb69 100755 --- a/roles/network_plugin/templates/network-environment.j2 +++ b/roles/network_plugin/templates/network-environment.j2 @@ -16,7 +16,7 @@ ETCD_AUTHORITY="127.0.0.1:23799" {% endif %} # The kubernetes-apiserver location - used by the calico plugin -KUBE_API_ROOT=http://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{kube_apiserver_insecure_port}}/api/v1/ +KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{kube_apiserver_port}}/api/v1/ {% else %} FLANNEL_ETCD_PREFIX="--etcd-prefix=/{{ cluster_name }}/network" {% endif %} From fec1dc9041b18d2faad797a71ba9b605423d0014 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Sat, 19 Dec 2015 11:00:22 +0100 Subject: [PATCH 02/13] A single file for tokens tasks --- roles/kubernetes/node/tasks/gen_tokens.yml | 29 ++++++++++++++++++++++ roles/kubernetes/node/tasks/secrets.yml | 28 --------------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/roles/kubernetes/node/tasks/gen_tokens.yml b/roles/kubernetes/node/tasks/gen_tokens.yml index f2e5625f9..7d1ce0156 100644 --- a/roles/kubernetes/node/tasks/gen_tokens.yml +++ b/roles/kubernetes/node/tasks/gen_tokens.yml @@ -4,6 +4,7 @@ src=kube-gen-token.sh dest={{ kube_script_dir }} mode=u+x + when: inventory_hostname == groups['kube-master'][0] - name: tokens | generate tokens for master components command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}" @@ -14,6 +15,7 @@ - "{{ groups['kube-master'] }}" register: gentoken changed_when: "'Added' in gentoken.stdout" + when: inventory_hostname == groups['kube-master'][0] - name: tokens | generate tokens for node components command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}" @@ -24,3 +26,30 @@ - "{{ groups['kube-node'] }}" register: gentoken changed_when: "'Added' in gentoken.stdout" + when: inventory_hostname == groups['kube-master'][0] + +- name: tokens | generate tokens for calico + command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}" + environment: + TOKEN_DIR: "{{ kube_token_dir }}" + with_nested: + - [ "system:calico" ] + - "{{ groups['k8s-cluster'] }}" + register: gentoken + changed_when: "'Added' in gentoken.stdout" + when: kube_network_plugin == "calico" + delegate_to: "{{ groups['kube-master'][0] }}" + +- name: tokens | get the calico token values + slurp: + src: "{{ kube_token_dir }}/system:calico-{{ inventory_hostname }}.token" + register: calico_token + when: kube_network_plugin == "calico" + delegate_to: "{{ groups['kube-master'][0] }}" + +- name: tokens | Add KUBE_AUTH_TOKEN for calico + lineinfile: + regexp: "^KUBE_AUTH_TOKEN=.*$" + line: "KUBE_AUTH_TOKEN={{ calico_token.content|b64decode }}" + dest: "/etc/network-environment" + when: kube_network_plugin == "calico" diff --git a/roles/kubernetes/node/tasks/secrets.yml b/roles/kubernetes/node/tasks/secrets.yml index 5154b9b59..4d6a2dcc3 100644 --- a/roles/kubernetes/node/tasks/secrets.yml +++ b/roles/kubernetes/node/tasks/secrets.yml @@ -18,34 +18,6 @@ when: inventory_hostname == groups['kube-master'][0] - include: gen_tokens.yml - run_once: true - when: inventory_hostname == groups['kube-master'][0] - -- name: tokens | generate tokens for calico - command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}" - environment: - TOKEN_DIR: "{{ kube_token_dir }}" - with_nested: - - [ "system:calico" ] - - "{{ groups['k8s-cluster'] }}" - register: gentoken - changed_when: "'Added' in gentoken.stdout" - when: kube_network_plugin == "calico" - delegate_to: "{{ groups['kube-master'][0] }}" - -- name: tokens | get the calico token values - slurp: - src: "{{ kube_token_dir }}/system:calico-{{ inventory_hostname }}.token" - register: calico_token - when: kube_network_plugin == "calico" - delegate_to: "{{ groups['kube-master'][0] }}" - -- name: tokens | Add KUBE_AUTH_TOKEN for calico - lineinfile: - regexp: "^KUBE_AUTH_TOKEN=.*$" - line: "KUBE_AUTH_TOKEN={{ calico_token.content|b64decode }}" - dest: "/etc/network-environment" - when: kube_network_plugin == "calico" # Sync certs between nodes - user: From d585ceaf3b9713a35a41a024640d98234a4a7b30 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Sat, 19 Dec 2015 12:32:06 +0100 Subject: [PATCH 03/13] set permissions on network-environment file --- roles/network_plugin/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/network_plugin/tasks/main.yml b/roles/network_plugin/tasks/main.yml index e3ebf305f..68363b53c 100644 --- a/roles/network_plugin/tasks/main.yml +++ b/roles/network_plugin/tasks/main.yml @@ -5,7 +5,7 @@ kube_network_plugin is not defined - name: Write network-environment - template: src=network-environment.j2 dest=/etc/network-environment mode=u+x + template: src=network-environment.j2 dest=/etc/network-environment mode=640 - include: flannel.yml when: kube_network_plugin == "flannel" From 2c816f66a3a94784a701522ab34aad0edd530242 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Sun, 20 Dec 2015 16:51:14 +0100 Subject: [PATCH 04/13] Check calico network pool --- roles/network_plugin/tasks/calico.yml | 40 +++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/roles/network_plugin/tasks/calico.yml b/roles/network_plugin/tasks/calico.yml index 1ba00f6fe..1d455ef8f 100644 --- a/roles/network_plugin/tasks/calico.yml +++ b/roles/network_plugin/tasks/calico.yml @@ -1,19 +1,49 @@ --- - name: Calico | Install calicoctl bin copy: - src={{ local_release_dir }}/calico/bin/calicoctl - dest={{ bin_dir }} - mode=0755 + src: "{{ local_release_dir }}/calico/bin/calicoctl" + dest: "{{ bin_dir }}" + mode: 0755 notify: restart calico-node - name: Calico | Create calicoctl symlink (needed by kubelet) - file: src=/usr/local/bin/calicoctl dest=/usr/bin/calicoctl state=link + file: + src: /usr/local/bin/calicoctl + dest: /usr/bin/calicoctl + state: link -- name: Calico | Configure calico-node desired pool +- name: Calico | Check if calico network pool has already been configured + uri: + url: "http://127.0.0.1:2379/v2/keys/calico/v1/ipam/v4/pool" + return_content: yes + status_code: 200,404 + register: calico_conf + run_once: true + delegate_to: "{{ groups['etcd'][0] }}" + +- name: Calico | Configure calico network pool shell: calicoctl pool add {{ kube_pods_subnet }} environment: ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379" run_once: true + when: calico_conf.status == 404 + delegate_to: "{{ groups['etcd'][0] }}" + +- name: Calico | Get calico configuration from etcd + uri: + url: "http://127.0.0.1:2379/v2/keys/calico/v1/ipam/v4/pool" + return_content: yes + register: calico_pools + run_once: true + delegate_to: "{{ groups['etcd'][0] }}" + +- name: Calico | Check if calico pool is properly configured + fail: + msg: 'Only one network pool must be configured and it must be the subnet {{ kube_pods_subnet }}. + Please erase calico configuration and run the playbook again ("etcdctl rm --recursive /calico/v1/ipam/v4/pool")' + when: ( calico_pools.json['node']['nodes'] | length > 1 ) or + ( not calico_pools.json['node']['nodes'][0]['key'] | search(".*{{ kube_pods_subnet | ipaddr('network') }}.*") ) + run_once: true delegate_to: "{{ groups['etcd'][0] }}" - name: Calico | Write calico-node systemd init file From bba3525cd8fcd2069c23bfe119a3e26ce682ff47 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Mon, 21 Dec 2015 09:13:48 +0100 Subject: [PATCH 05/13] use loadbalancer when that's possible --- roles/network_plugin/templates/network-environment.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/network_plugin/templates/network-environment.j2 b/roles/network_plugin/templates/network-environment.j2 index 0aaf4bb69..6b8330a64 100755 --- a/roles/network_plugin/templates/network-environment.j2 +++ b/roles/network_plugin/templates/network-environment.j2 @@ -5,7 +5,11 @@ CALICO_IPAM=true DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }} # The kubernetes master IP +{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} +KUBERNETES_MASTER=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }} +{% else %} KUBERNETES_MASTER={{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }} +{% endif %} # Location of etcd cluster used by Calico. By default, this uses the etcd # instance running on the Kubernetes Master From ab694ee29139708431fda78de2812ea0a5def65c Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Mon, 21 Dec 2015 12:00:42 +0100 Subject: [PATCH 06/13] Install python-httplib2 required packaged --- roles/etcd/tasks/install.yml | 6 ++++++ roles/kubernetes/master/tasks/main.yml | 7 +------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/roles/etcd/tasks/install.yml b/roles/etcd/tasks/install.yml index ac3522f4d..8d442e6b3 100644 --- a/roles/etcd/tasks/install.yml +++ b/roles/etcd/tasks/install.yml @@ -15,3 +15,9 @@ - name: Create etcd2 binary symlink file: src=/usr/local/bin/etcd dest=/usr/local/bin/etcd2 state=link + +- name: install required python module 'httplib2' + apt: + name: "python-httplib2" + state: present + when: inventory_hostname == groups['kube-master'][0] or inventory_hostname == groups['etcd'][0] diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index 987c41c6e..ad2739d00 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -32,6 +32,7 @@ - "{{ kube_cert_dir }}" - "{{ kube_users_dir }}" delegate_to: "{{ groups['kube-master'][0] }}" + when: inventory_hostname != "{{ groups['kube-master'][0] }}" # Write manifests - name: Write kube-apiserver manifest @@ -48,12 +49,6 @@ port: "{{kube_apiserver_insecure_port}}" delay: 10 -- name: install required python module 'httplib2' - apt: - name: "python-httplib2" - state: present - when: inventory_hostname == groups['kube-master'][0] - - name: Create 'kube-system' namespace uri: url: http://127.0.0.1:{{ kube_apiserver_insecure_port }}/api/v1/namespaces From b2afbfd4fb81ed035ffc9275de87fa6577b56d5a Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Mon, 21 Dec 2015 14:23:33 +0100 Subject: [PATCH 07/13] don't touch if the file exists --- roles/kubernetes/node/files/kube-gen-token.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/kubernetes/node/files/kube-gen-token.sh b/roles/kubernetes/node/files/kube-gen-token.sh index fa6a5ddc7..121b52263 100644 --- a/roles/kubernetes/node/files/kube-gen-token.sh +++ b/roles/kubernetes/node/files/kube-gen-token.sh @@ -19,7 +19,10 @@ token_file="${token_dir}/known_tokens.csv" create_accounts=($@) -touch "${token_file}" +if [ ! -e "${token_file}" ]; then + touch "${token_file}" +fi + for account in "${create_accounts[@]}"; do if grep ",${account}," "${token_file}" ; then continue From 7315d33e3ce1ab6daccbf3c6e5c57d6f68460937 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Mon, 21 Dec 2015 14:24:10 +0100 Subject: [PATCH 08/13] use ip for etcd proxies even when hostnames are used in the inventory --- roles/etcd/templates/etcd2-environment.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/etcd/templates/etcd2-environment.j2 b/roles/etcd/templates/etcd2-environment.j2 index e9546be5e..2c0760388 100644 --- a/roles/etcd/templates/etcd2-environment.j2 +++ b/roles/etcd/templates/etcd2-environment.j2 @@ -15,6 +15,6 @@ ETCD_LISTEN_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ETCD_LISTEN_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380" ETCD_NAME="{{ etcd.name }}" {% else %} -ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}master{{ loop.index|string }}=http://{{ host }}:2380{% if not loop.last %},{% endif %}{% endfor %}" +ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}master{{ loop.index|string }}=http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2380{% if not loop.last %},{% endif %}{% endfor %}" ETCD_LISTEN_CLIENT_URLS="http://127.0.0.1:23799" {% endif %} From 680864f95c037d2b19c7b5a92ba8a8a60c0851f2 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Mon, 21 Dec 2015 14:24:57 +0100 Subject: [PATCH 09/13] don't sync certs on masters, already done in another task --- roles/kubernetes/node/tasks/secrets.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/kubernetes/node/tasks/secrets.yml b/roles/kubernetes/node/tasks/secrets.yml index 4d6a2dcc3..4b2c20802 100644 --- a/roles/kubernetes/node/tasks/secrets.yml +++ b/roles/kubernetes/node/tasks/secrets.yml @@ -48,3 +48,4 @@ - "{{ kube_cert_dir}}/node.pem" - "{{ kube_cert_dir}}/node-key.pem" delegate_to: "{{ groups['kube-master'][0] }}" + when: inventory_hostname not in "{{ groups['kube-master'] }}" From 7c9c609ac420073cae01d7246e7cdc32973480a1 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Tue, 22 Dec 2015 08:45:14 +0100 Subject: [PATCH 10/13] calico uses loadbalancer address for apiserver --- roles/network_plugin/templates/network-environment.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/network_plugin/templates/network-environment.j2 b/roles/network_plugin/templates/network-environment.j2 index 6b8330a64..6173a7a97 100755 --- a/roles/network_plugin/templates/network-environment.j2 +++ b/roles/network_plugin/templates/network-environment.j2 @@ -20,7 +20,11 @@ ETCD_AUTHORITY="127.0.0.1:23799" {% endif %} # The kubernetes-apiserver location - used by the calico plugin +{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} +KUBE_API_ROOT=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}/api/v1/ +{% else %} KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{kube_apiserver_port}}/api/v1/ +{% endif %} {% else %} FLANNEL_ETCD_PREFIX="--etcd-prefix=/{{ cluster_name }}/network" {% endif %} From 5f4e01cec5c5bb99d70cc10772887af4c68fafda Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Tue, 22 Dec 2015 16:38:40 +0100 Subject: [PATCH 11/13] new version of logstash submodule --- roles/apps/k8s-kube-logstash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/apps/k8s-kube-logstash b/roles/apps/k8s-kube-logstash index 256fa156e..340d1a5ec 160000 --- a/roles/apps/k8s-kube-logstash +++ b/roles/apps/k8s-kube-logstash @@ -1 +1 @@ -Subproject commit 256fa156e46d623ab0a7a60efdc7bac535cea8d7 +Subproject commit 340d1a5ec75e7b7c43783dc7a1c02aa7d5991dbe From 595e93e6da63482e82ce1bde941ee3b3e9005c71 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Thu, 24 Dec 2015 13:56:53 +0100 Subject: [PATCH 12/13] Peer with router configuration is made on the first etcd node --- roles/network_plugin/tasks/calico.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/roles/network_plugin/tasks/calico.yml b/roles/network_plugin/tasks/calico.yml index 1d455ef8f..2cf3e2b2c 100644 --- a/roles/network_plugin/tasks/calico.yml +++ b/roles/network_plugin/tasks/calico.yml @@ -23,8 +23,6 @@ - name: Calico | Configure calico network pool shell: calicoctl pool add {{ kube_pods_subnet }} - environment: - ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379" run_once: true when: calico_conf.status == 404 delegate_to: "{{ groups['etcd'][0] }}" @@ -63,9 +61,13 @@ - name: Calico | Disable node mesh shell: calicoctl bgp node-mesh off + environment: + ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379" when: peer_with_router|default(false) and inventory_hostname in groups['kube-node'] - name: Calico | Configure peering with router(s) shell: calicoctl node bgp peer add {{ item.router_id }} as {{ item.as }} + environment: + ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379" with_items: peers when: peer_with_router|default(false) and inventory_hostname in groups['kube-node'] From 4388cab8d679d88eb5d4f626f6c008889a350c2a Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Thu, 24 Dec 2015 13:58:04 +0100 Subject: [PATCH 13/13] Use second ip address in order to avoid any ip range problem --- inventory/group_vars/all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 41c87a57d..0fe0dd5e5 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -68,7 +68,7 @@ dns_setup: true dns_domain: "{{ cluster_name }}" # # # Ip address of the kubernetes dns service -dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}" +dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}" # For multi masters architecture: # kube-proxy doesn't support multiple apiservers for the time being so you'll need to configure your own loadbalancer