From 953f482585181bde926f9e9a99cf25ae48919032 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Tue, 15 Dec 2015 15:20:08 +0100 Subject: [PATCH 1/3] kube-proxy loadbalancing, need an external loadbalancer --- environments/test/group_vars/all.yml | 15 +++++++++++++++ environments/test/inventory.example | 10 +--------- roles/dnsmasq/tasks/main.yml | 8 ++++++++ .../templates/manifests/kube-proxy.manifest.j2 | 6 +++++- roles/kubernetes/node/templates/openssl.conf.j2 | 5 +---- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/environments/test/group_vars/all.yml b/environments/test/group_vars/all.yml index 5dc380898..35ae21a0a 100644 --- a/environments/test/group_vars/all.yml +++ b/environments/test/group_vars/all.yml @@ -69,3 +69,18 @@ dns_domain: "{{ cluster_name }}" # # # Ip address of the kubernetes dns service dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}" + +# For multi masters architecture: +# kube-proxy doesn't support multiple apiservers for the time being so you'll need to configure your own loadbalancer +# This domain name will be inserted into the /etc/hosts file of all servers +# configurationexample with haproxy : +# lissten kubernetes-apiserver-https +# bind 10.99.0.21:8383 +# option ssl-hello-chk +# mode tcp +# timeout client 3h +# timeout server 3h +# server master1 10.99.0.26:443 +# server master2 10.99.0.27:443 +# balance roundrobin +apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local" diff --git a/environments/test/inventory.example b/environments/test/inventory.example index 35730d191..a811b084a 100644 --- a/environments/test/inventory.example +++ b/environments/test/inventory.example @@ -17,18 +17,10 @@ 10.99.0.4 local_as=xxxxxxxx 10.99.0.5 local_as=xxxxxxxx -[usa] +[new-york] 10.99.0.36 local_as=xxxxxxxx 10.99.0.37 local_as=xxxxxxxx [k8s-cluster:children] kube-node kube-master - -[paris:vars] -peers=[{"router_id": "10.99.0.2", "as": "65xxx"}, {"router_id": "10.99.0.3", "as": "65xxx"}] -loadbalancer_address="10.99.0.24" - -[usa:vars] -peers=[{"router_id": "10.99.0.34", "as": "65xxx"}, {"router_id": "10.99.0.35", "as": "65xxx"}] -loadbalancer_address="10.99.0.44" diff --git a/roles/dnsmasq/tasks/main.yml b/roles/dnsmasq/tasks/main.yml index 7ba02c36c..87c056167 100644 --- a/roles/dnsmasq/tasks/main.yml +++ b/roles/dnsmasq/tasks/main.yml @@ -8,6 +8,14 @@ when: hostvars[item].ansible_default_ipv4.address is defined with_items: groups['all'] +- name: populate kubernetes loadbalancer address into hosts file + lineinfile: + dest: /etc/hosts + regexp: ".*{{ apiserver_loadbalancer_domain_name }}$" + line: "{{ loadbalancer_apiserver.address }} lb-apiserver.kubernetes.local" + state: present + when: loadbalancer_apiserver is defined + - name: clean hosts file lineinfile: dest: /etc/hosts diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index 32c52e7d9..923c29764 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -15,7 +15,11 @@ spec: {% if inventory_hostname in groups['kube-master'] %} - --master=http://127.0.0.1:8080 {% else %} - - --master=https://{{ groups['kube-master'][0] }}:{{kube_apiserver_port }} +{% if loadbalancer_apiserver.address is defined | default('') %} + - --master=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }} +{% else %} + - --master=https://{{ groups['kube-master'][0] }}:{{ kube_apiserver_port }} +{% endif%} - --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml {% endif %} securityContext: diff --git a/roles/kubernetes/node/templates/openssl.conf.j2 b/roles/kubernetes/node/templates/openssl.conf.j2 index ce2b88e9c..3715d97b8 100644 --- a/roles/kubernetes/node/templates/openssl.conf.j2 +++ b/roles/kubernetes/node/templates/openssl.conf.j2 @@ -10,12 +10,9 @@ subjectAltName = @alt_names DNS.1 = kubernetes DNS.2 = kubernetes.default DNS.3 = kubernetes.default.svc.{{ dns_domain }} +DNS.4 = {{ apiserver_loadbalancer_domain_name }} {% for host in groups['kube-master'] %} IP.{{ loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} {% endfor %} {% set idx = groups['kube-master'] | length | int + 1 %} IP.{{ idx | string }} = {{ kube_apiserver_ip }} -{% if loadbalancer_address is defined | default('') %} -{% set idx = idx | int + 1 %} -IP.{{ idx | string }} = {{ loadbalancer_address }} -{% endif %} From 4d1828c724e11da05f3e4364adf11e7ff4aac745 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Tue, 15 Dec 2015 15:25:24 +0100 Subject: [PATCH 2/3] group vars per location --- environments/test/group_vars/new-york.yml | 10 ++++++++++ environments/test/group_vars/paris.yml | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 environments/test/group_vars/new-york.yml create mode 100644 environments/test/group_vars/paris.yml diff --git a/environments/test/group_vars/new-york.yml b/environments/test/group_vars/new-york.yml new file mode 100644 index 000000000..5b51961db --- /dev/null +++ b/environments/test/group_vars/new-york.yml @@ -0,0 +1,10 @@ +--- +peers: + -router_id: "10.99.0.34" + as: "65xxx" + - router_id: "10.99.0.35" + as: "65xxx" + +loadbalancer_apiserver: + address: "10.99.0.44" + port: "8383" diff --git a/environments/test/group_vars/paris.yml b/environments/test/group_vars/paris.yml new file mode 100644 index 000000000..052200ba6 --- /dev/null +++ b/environments/test/group_vars/paris.yml @@ -0,0 +1,10 @@ +--- +peers: + -router_id: "10.99.0.2" + as: "65xxx" + - router_id: "10.99.0.3" + as: "65xxx" + +loadbalancer_apiserver: + address: "10.99.0.21" + port: "8383" From 43afd42f596e582908e1aa18403097b10afe72be Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Tue, 15 Dec 2015 15:27:12 +0100 Subject: [PATCH 3/3] use 3 members for etcd clustering --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index fd8f41668..c0f380039 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Edit the inventory according to the number of servers [etcd] 10.115.99.31 10.115.99.32 +10.115.99.33 [kube-node] 10.115.99.32 @@ -82,6 +83,7 @@ In node-mesh mode the nodes peers with all the nodes in order to exchange routes [etcd] 10.99.0.26 +10.99.0.4 10.99.0.59 [kube-node]