From 0b4168cad4cd046211138bbc2251415ec4af19e6 Mon Sep 17 00:00:00 2001 From: woopstar Date: Mon, 5 Feb 2018 10:37:36 +0100 Subject: [PATCH 1/7] WIP. Adding metrics-server support for K8s version 1.9 --- inventory/group_vars/k8s-cluster.yml | 11 +++++++++++ roles/kubernetes/secrets/files/make-ssl.sh | 2 ++ roles/kubernetes/secrets/tasks/check-certs.yml | 6 +++++- roles/kubernetes/secrets/tasks/gen_certs_script.yml | 2 ++ .../secrets/tasks/sync_kube_master_certs.yml | 2 +- 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index 8f79f3297..e2fe06149 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -192,3 +192,14 @@ persistent_volumes_enabled: false ## See https://github.com/kubernetes-incubator/kubespray/issues/2141 ## Set this variable to true to get rid of this issue volume_cross_zone_attachment: false + +## Add options for metrics-server +#apiserver_custom_flags: +# - --requestheader-client-ca-file=/etc/kubernetes/ssl/ca.pem +# - --requestheader-allowed-names=aggregator +# - "--requestheader-extra-headers-prefix=X-Remote-Extra-" +# - --requestheader-group-headers=X-Remote-Group +# - --requestheader-username-headers=X-Remote-User +# - --enable-aggregator-routing=true +# - --proxy-client-cert-file=/etc/kubernetes/ssl/aggregator-proxy-client.pem +# - --proxy-client-key-file=/etc/kubernetes/ssl/aggregator-proxy-client-key.pem diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh index 9139ce98f..8cfc0728a 100755 --- a/roles/kubernetes/secrets/files/make-ssl.sh +++ b/roles/kubernetes/secrets/files/make-ssl.sh @@ -93,6 +93,8 @@ if [ -n "$MASTERS" ]; then gen_key_and_cert "kube-scheduler" "/CN=system:kube-scheduler" # kube-controller-manager gen_key_and_cert "kube-controller-manager" "/CN=system:kube-controller-manager" + # metrics aggregator + gen_key_and_cert "aggregator-proxy-client" "/CN=system:aggregator-proxy-client" for host in $MASTERS; do cn="${host%%.*}" diff --git a/roles/kubernetes/secrets/tasks/check-certs.yml b/roles/kubernetes/secrets/tasks/check-certs.yml index 3870a3e96..782da6863 100644 --- a/roles/kubernetes/secrets/tasks/check-certs.yml +++ b/roles/kubernetes/secrets/tasks/check-certs.yml @@ -26,6 +26,8 @@ - kube-scheduler-key.pem - kube-controller-manager.pem - kube-controller-manager-key.pem + - aggregator-proxy-client.pem + - aggregator-proxy-client-key.pem - admin-{{ inventory_hostname }}.pem - admin-{{ inventory_hostname }}-key.pem - node-{{ inventory_hostname }}.pem @@ -46,6 +48,8 @@ '{{ kube_cert_dir }}/kube-scheduler-key.pem', '{{ kube_cert_dir }}/kube-controller-manager.pem', '{{ kube_cert_dir }}/kube-controller-manager-key.pem', + '{{ kube_cert_dir }}/aggregator-proxy-client.pem', + '{{ kube_cert_dir }}/aggregator-proxy-client-key.pem', {% for host in groups['kube-master'] %} '{{ kube_cert_dir }}/admin-{{ host }}.pem' '{{ kube_cert_dir }}/admin-{{ host }}-key.pem' @@ -66,7 +70,7 @@ {% set existing_certs = kubecert_master.files|map(attribute='path')|list|sort %} {% for cert in ['apiserver.pem', 'apiserver-key.pem', 'kube-scheduler.pem', 'kube-scheduler-key.pem', 'kube-controller-manager.pem', - 'kube-controller-manager-key.pem'] -%} + 'kube-controller-manager-key.pem','aggregator-proxy-client.pem','aggregator-proxy-client-key.pem'] -%} {% set cert_file = "%s/%s.pem"|format(kube_cert_dir, cert) %} {% if not cert_file in existing_certs -%} {%- set gen = True -%} diff --git a/roles/kubernetes/secrets/tasks/gen_certs_script.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml index 619bbe445..9be59fb7b 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs_script.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml @@ -73,6 +73,8 @@ 'kube-scheduler-key.pem', 'kube-controller-manager.pem', 'kube-controller-manager-key.pem', + 'aggregator-proxy-client.pem', + 'aggregator-proxy-client-key.pem', {% for node in groups['kube-master'] %} 'admin-{{ node }}.pem', 'admin-{{ node }}-key.pem', diff --git a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml index d54bf2b67..f488cc61b 100644 --- a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml +++ b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml @@ -32,7 +32,7 @@ sync_file_hosts: "{{ groups['kube-master'] }}" sync_file_is_cert: true sync_file_owner: kube - with_items: ["apiserver.pem", "kube-scheduler.pem", "kube-controller-manager.pem"] + with_items: ["apiserver.pem", "kube-scheduler.pem", "kube-controller-manager.pem", "aggregator-proxy-client.pem"] - name: sync_kube_master_certs | Set facts for kube master components sync_file results set_fact: From 82d10b882cfc39f96c4d9d828d9548ab6e6d0a29 Mon Sep 17 00:00:00 2001 From: woopstar Date: Mon, 5 Feb 2018 20:06:55 +0100 Subject: [PATCH 2/7] Added fixes from whereismyjetpack --- roles/kubernetes/secrets/files/make-ssl.sh | 2 +- roles/kubernetes/secrets/tasks/check-certs.yml | 5 +++-- roles/kubernetes/secrets/tasks/gen_certs_script.yml | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh index 8cfc0728a..61668992d 100755 --- a/roles/kubernetes/secrets/files/make-ssl.sh +++ b/roles/kubernetes/secrets/files/make-ssl.sh @@ -94,7 +94,7 @@ if [ -n "$MASTERS" ]; then # kube-controller-manager gen_key_and_cert "kube-controller-manager" "/CN=system:kube-controller-manager" # metrics aggregator - gen_key_and_cert "aggregator-proxy-client" "/CN=system:aggregator-proxy-client" + gen_key_and_cert "aggregator-proxy-client" "/CN=aggregator" for host in $MASTERS; do cn="${host%%.*}" diff --git a/roles/kubernetes/secrets/tasks/check-certs.yml b/roles/kubernetes/secrets/tasks/check-certs.yml index 782da6863..3b3b20300 100644 --- a/roles/kubernetes/secrets/tasks/check-certs.yml +++ b/roles/kubernetes/secrets/tasks/check-certs.yml @@ -105,7 +105,8 @@ {% if gen_node_certs[inventory_hostname] or (not kubecert_node.results[0].stat.exists|default(False)) or (not kubecert_node.results[10].stat.exists|default(False)) or - (kubecert_node.results[10].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[10].stat.path)|map(attribute="checksum")|first|default('')) -%} - {%- set _ = certs.update({'sync': True}) -%} + (not kubecert_node.results[7].stat.exists|default(False)) or + (kubecert_node.results[10].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[10].stat.path)|map(attribute="checksum")|first|default('')) -%} + {%- set _ = certs.update({'sync': True}) -%} {% endif %} {{ certs.sync }} diff --git a/roles/kubernetes/secrets/tasks/gen_certs_script.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml index 9be59fb7b..0b88e0f14 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs_script.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml @@ -84,6 +84,8 @@ 'admin-{{ inventory_hostname }}-key.pem', 'apiserver.pem', 'apiserver-key.pem', + 'aggregator-proxy-client.pem', + 'aggregator-proxy-client-key.pem', 'kube-scheduler.pem', 'kube-scheduler-key.pem', 'kube-controller-manager.pem', From b2d30d68e7458fdc59f7d0faf33a040d89fa734f Mon Sep 17 00:00:00 2001 From: woopstar Date: Mon, 5 Feb 2018 20:37:06 +0100 Subject: [PATCH 3/7] Rename CN for aggreator back. Add flags to apiserver when version is >= 1.9 --- inventory/group_vars/k8s-cluster.yml | 11 ----------- .../templates/manifests/kube-apiserver.manifest.j2 | 10 ++++++++++ roles/kubernetes/secrets/files/make-ssl.sh | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index e2fe06149..8f79f3297 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -192,14 +192,3 @@ persistent_volumes_enabled: false ## See https://github.com/kubernetes-incubator/kubespray/issues/2141 ## Set this variable to true to get rid of this issue volume_cross_zone_attachment: false - -## Add options for metrics-server -#apiserver_custom_flags: -# - --requestheader-client-ca-file=/etc/kubernetes/ssl/ca.pem -# - --requestheader-allowed-names=aggregator -# - "--requestheader-extra-headers-prefix=X-Remote-Extra-" -# - --requestheader-group-headers=X-Remote-Group -# - --requestheader-username-headers=X-Remote-User -# - --enable-aggregator-routing=true -# - --proxy-client-cert-file=/etc/kubernetes/ssl/aggregator-proxy-client.pem -# - --proxy-client-key-file=/etc/kubernetes/ssl/aggregator-proxy-client-key.pem diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 39974846d..751ce9392 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -100,6 +100,16 @@ spec: {% if kube_feature_gates %} - --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} +{% if kube_version | version_compare('1.9', '>=') %} + - --requestheader-client-ca-file=/etc/kubernetes/ssl/ca.pem + - --requestheader-allowed-names=system:aggregator-proxy-client + - "--requestheader-extra-headers-prefix=X-Remote-Extra-" + - --requestheader-group-headers=X-Remote-Group + - --requestheader-username-headers=X-Remote-User + - --enable-aggregator-routing=true + - --proxy-client-cert-file=/etc/kubernetes/ssl/aggregator-proxy-client.pem + - --proxy-client-key-file=/etc/kubernetes/ssl/aggregator-proxy-client-key.pem +{% endif %} {% if apiserver_custom_flags is string %} - {{ apiserver_custom_flags }} {% else %} diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh index 61668992d..8cfc0728a 100755 --- a/roles/kubernetes/secrets/files/make-ssl.sh +++ b/roles/kubernetes/secrets/files/make-ssl.sh @@ -94,7 +94,7 @@ if [ -n "$MASTERS" ]; then # kube-controller-manager gen_key_and_cert "kube-controller-manager" "/CN=system:kube-controller-manager" # metrics aggregator - gen_key_and_cert "aggregator-proxy-client" "/CN=aggregator" + gen_key_and_cert "aggregator-proxy-client" "/CN=system:aggregator-proxy-client" for host in $MASTERS; do cn="${host%%.*}" From 4dab92ce69805dc607220e7d9f07d58ae3936270 Mon Sep 17 00:00:00 2001 From: woopstar Date: Wed, 7 Feb 2018 09:50:08 +0100 Subject: [PATCH 4/7] Rename from aggregator-proxy-client to front-proxy-client to match kubeadm design. Added kubeadm support too. Changed to use variables set and not hardcode paths. Still missing cert generation for Vault --- .../master/templates/kubeadm-config.yaml.j2 | 10 ++++++++++ .../manifests/kube-apiserver.manifest.j2 | 12 ++++++------ roles/kubernetes/secrets/files/make-ssl.sh | 2 +- roles/kubernetes/secrets/tasks/check-certs.yml | 15 ++++++++------- .../kubernetes/secrets/tasks/gen_certs_script.yml | 8 ++++---- .../secrets/tasks/sync_kube_master_certs.yml | 2 +- roles/kubespray-defaults/defaults/main.yaml | 4 ++++ 7 files changed, 34 insertions(+), 19 deletions(-) diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index 26e3b46a4..e25804e66 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -54,6 +54,16 @@ apiServerExtraArgs: runtime-config: {{ kube_api_runtime_config | join(',') }} {% endif %} allow-privileged: "true" +{% if kube_version | version_compare('1.9', '>=') %} + requestheader-client-ca-file: "{{ kube_cert_dir }}/ca.pem" + requestheader-allowed-names: "{{ kube_api_requestheader_allowed_names }}" + requestheader-extra-headers-prefix: "X-Remote-Extra-" + requestheader-group-headers: "X-Remote-Group" + requestheader-username-headers: "X-Remote-User" + enable-aggregator-routing: "{{ kube_api_aggregator_routing }}" + proxy-client-cert-file: "{{ kube_cert_dir }}/front-proxy-client.pem" + proxy-client-key-file: "{{ kube_cert_dir }}/front-proxy-client-key.pem" +{% endif %} controllerManagerExtraArgs: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} node-monitor-period: {{ kube_controller_node_monitor_period }} diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 751ce9392..d6f065ea5 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -101,14 +101,14 @@ spec: - --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kube_version | version_compare('1.9', '>=') %} - - --requestheader-client-ca-file=/etc/kubernetes/ssl/ca.pem - - --requestheader-allowed-names=system:aggregator-proxy-client - - "--requestheader-extra-headers-prefix=X-Remote-Extra-" + - --requestheader-client-ca-file={{ kube_cert_dir }}/ca.pem + - --requestheader-allowed-names={{ kube_api_requestheader_allowed_names }} + - --requestheader-extra-headers-prefix=X-Remote-Extra- - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User - - --enable-aggregator-routing=true - - --proxy-client-cert-file=/etc/kubernetes/ssl/aggregator-proxy-client.pem - - --proxy-client-key-file=/etc/kubernetes/ssl/aggregator-proxy-client-key.pem + - --enable-aggregator-routing={{ kube_api_aggregator_routing }} + - --proxy-client-cert-file={{ kube_cert_dir }}/front-proxy-client.pem + - --proxy-client-key-file={{ kube_cert_dir }}/front-proxy-client-key.pem {% endif %} {% if apiserver_custom_flags is string %} - {{ apiserver_custom_flags }} diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh index 8cfc0728a..750e9c4fe 100755 --- a/roles/kubernetes/secrets/files/make-ssl.sh +++ b/roles/kubernetes/secrets/files/make-ssl.sh @@ -94,7 +94,7 @@ if [ -n "$MASTERS" ]; then # kube-controller-manager gen_key_and_cert "kube-controller-manager" "/CN=system:kube-controller-manager" # metrics aggregator - gen_key_and_cert "aggregator-proxy-client" "/CN=system:aggregator-proxy-client" + gen_key_and_cert "front-proxy-client" "/CN=front-proxy-client" for host in $MASTERS; do cn="${host%%.*}" diff --git a/roles/kubernetes/secrets/tasks/check-certs.yml b/roles/kubernetes/secrets/tasks/check-certs.yml index 3b3b20300..627889771 100644 --- a/roles/kubernetes/secrets/tasks/check-certs.yml +++ b/roles/kubernetes/secrets/tasks/check-certs.yml @@ -26,8 +26,8 @@ - kube-scheduler-key.pem - kube-controller-manager.pem - kube-controller-manager-key.pem - - aggregator-proxy-client.pem - - aggregator-proxy-client-key.pem + - front-proxy-client.pem + - front-proxy-client-key.pem - admin-{{ inventory_hostname }}.pem - admin-{{ inventory_hostname }}-key.pem - node-{{ inventory_hostname }}.pem @@ -48,8 +48,8 @@ '{{ kube_cert_dir }}/kube-scheduler-key.pem', '{{ kube_cert_dir }}/kube-controller-manager.pem', '{{ kube_cert_dir }}/kube-controller-manager-key.pem', - '{{ kube_cert_dir }}/aggregator-proxy-client.pem', - '{{ kube_cert_dir }}/aggregator-proxy-client-key.pem', + '{{ kube_cert_dir }}/front-proxy-client.pem', + '{{ kube_cert_dir }}/front-proxy-client-key.pem', {% for host in groups['kube-master'] %} '{{ kube_cert_dir }}/admin-{{ host }}.pem' '{{ kube_cert_dir }}/admin-{{ host }}-key.pem' @@ -68,9 +68,10 @@ gen_master_certs: |- {%- set gen = False -%} {% set existing_certs = kubecert_master.files|map(attribute='path')|list|sort %} - {% for cert in ['apiserver.pem', 'apiserver-key.pem', 'kube-scheduler.pem', - 'kube-scheduler-key.pem', 'kube-controller-manager.pem', - 'kube-controller-manager-key.pem','aggregator-proxy-client.pem','aggregator-proxy-client-key.pem'] -%} + {% for cert in ['apiserver.pem', 'apiserver-key.pem', + 'kube-scheduler.pem','kube-scheduler-key.pem', + 'kube-controller-manager.pem','kube-controller-manager-key.pem', + 'front-proxy-client.pem','front-proxy-client-key.pem'] -%} {% set cert_file = "%s/%s.pem"|format(kube_cert_dir, cert) %} {% if not cert_file in existing_certs -%} {%- set gen = True -%} diff --git a/roles/kubernetes/secrets/tasks/gen_certs_script.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml index 0b88e0f14..c1dfeb394 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs_script.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml @@ -73,8 +73,8 @@ 'kube-scheduler-key.pem', 'kube-controller-manager.pem', 'kube-controller-manager-key.pem', - 'aggregator-proxy-client.pem', - 'aggregator-proxy-client-key.pem', + 'front-proxy-client.pem', + 'front-proxy-client-key.pem', {% for node in groups['kube-master'] %} 'admin-{{ node }}.pem', 'admin-{{ node }}-key.pem', @@ -84,8 +84,8 @@ 'admin-{{ inventory_hostname }}-key.pem', 'apiserver.pem', 'apiserver-key.pem', - 'aggregator-proxy-client.pem', - 'aggregator-proxy-client-key.pem', + 'front-proxy-client.pem', + 'front-proxy-client-key.pem', 'kube-scheduler.pem', 'kube-scheduler-key.pem', 'kube-controller-manager.pem', diff --git a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml index f488cc61b..f675f6eca 100644 --- a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml +++ b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml @@ -32,7 +32,7 @@ sync_file_hosts: "{{ groups['kube-master'] }}" sync_file_is_cert: true sync_file_owner: kube - with_items: ["apiserver.pem", "kube-scheduler.pem", "kube-controller-manager.pem", "aggregator-proxy-client.pem"] + with_items: ["apiserver.pem", "kube-scheduler.pem", "kube-controller-manager.pem", "front-proxy-client.pem"] - name: sync_kube_master_certs | Set facts for kube master components sync_file results set_fact: diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 498b14365..efec7bd3d 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -122,6 +122,10 @@ kube_apiserver_port: 6443 kube_apiserver_insecure_bind_address: 127.0.0.1 kube_apiserver_insecure_port: 8080 +# Metrics server +kube_api_requestheader_allowed_names: "front-proxy-client" +kube_api_aggregator_routing: true + # Path used to store Docker data docker_daemon_graph: "/var/lib/docker" From 2cd254954cc3b78f2fa8cbc516e0ea7e2be377a8 Mon Sep 17 00:00:00 2001 From: woopstar Date: Wed, 7 Feb 2018 10:07:46 +0100 Subject: [PATCH 5/7] Remove defaults of allowed names. Updated kubeadm --- roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 4 ---- .../master/templates/manifests/kube-apiserver.manifest.j2 | 2 +- roles/kubespray-defaults/defaults/main.yaml | 3 +-- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index e25804e66..e489bb115 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -56,10 +56,6 @@ apiServerExtraArgs: allow-privileged: "true" {% if kube_version | version_compare('1.9', '>=') %} requestheader-client-ca-file: "{{ kube_cert_dir }}/ca.pem" - requestheader-allowed-names: "{{ kube_api_requestheader_allowed_names }}" - requestheader-extra-headers-prefix: "X-Remote-Extra-" - requestheader-group-headers: "X-Remote-Group" - requestheader-username-headers: "X-Remote-User" enable-aggregator-routing: "{{ kube_api_aggregator_routing }}" proxy-client-cert-file: "{{ kube_cert_dir }}/front-proxy-client.pem" proxy-client-key-file: "{{ kube_cert_dir }}/front-proxy-client-key.pem" diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index d6f065ea5..f499e1a7d 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -102,7 +102,7 @@ spec: {% endif %} {% if kube_version | version_compare('1.9', '>=') %} - --requestheader-client-ca-file={{ kube_cert_dir }}/ca.pem - - --requestheader-allowed-names={{ kube_api_requestheader_allowed_names }} + - --requestheader-allowed-names=front-proxy-client - --requestheader-extra-headers-prefix=X-Remote-Extra- - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index efec7bd3d..a76bfcc9f 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -122,8 +122,7 @@ kube_apiserver_port: 6443 kube_apiserver_insecure_bind_address: 127.0.0.1 kube_apiserver_insecure_port: 8080 -# Metrics server -kube_api_requestheader_allowed_names: "front-proxy-client" +# Aggregator kube_api_aggregator_routing: true # Path used to store Docker data From f193b120592f8aeac003085497c1efd06da5f582 Mon Sep 17 00:00:00 2001 From: woopstar Date: Wed, 7 Feb 2018 10:50:30 +0100 Subject: [PATCH 6/7] Kubeadm auto creates this --- roles/kubernetes/master/templates/kubeadm-config.yaml.j2 | 6 ------ 1 file changed, 6 deletions(-) diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index e489bb115..26e3b46a4 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -54,12 +54,6 @@ apiServerExtraArgs: runtime-config: {{ kube_api_runtime_config | join(',') }} {% endif %} allow-privileged: "true" -{% if kube_version | version_compare('1.9', '>=') %} - requestheader-client-ca-file: "{{ kube_cert_dir }}/ca.pem" - enable-aggregator-routing: "{{ kube_api_aggregator_routing }}" - proxy-client-cert-file: "{{ kube_cert_dir }}/front-proxy-client.pem" - proxy-client-key-file: "{{ kube_cert_dir }}/front-proxy-client-key.pem" -{% endif %} controllerManagerExtraArgs: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} node-monitor-period: {{ kube_controller_node_monitor_period }} From f9df692056e6222ab6908ea1e1f26eb3ff8d75a0 Mon Sep 17 00:00:00 2001 From: woopstar Date: Wed, 7 Feb 2018 11:03:07 +0100 Subject: [PATCH 7/7] Issue front proxy certs for vault --- .../secrets/tasks/gen_certs_vault.yml | 26 +++++++++++++++++++ roles/vault/defaults/main.yml | 8 ++++++ 2 files changed, 34 insertions(+) diff --git a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml index ca28b537f..5b3b46edc 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml @@ -93,3 +93,29 @@ issue_cert_mount_path: "{{ kube_vault_mount_path }}" with_items: "{{ kube_proxy_certs_needed|d([]) }}" when: inventory_hostname in groups['k8s-cluster'] + +# Issue front proxy cert to kube-master hosts +- include_tasks: ../../../vault/tasks/shared/issue_cert.yml + vars: + issue_cert_common_name: "front-proxy-client" + issue_cert_alt_names: "{{ kube_cert_alt_names }}" + issue_cert_file_group: "{{ kube_cert_group }}" + issue_cert_file_owner: kube + issue_cert_hosts: "{{ groups['kube-master'] }}" + issue_cert_ip_sans: >- + [ + {%- for host in groups['kube-master'] -%} + "{{ hostvars[host]['ansible_default_ipv4']['address'] }}", + {%- if hostvars[host]['ip'] is defined -%} + "{{ hostvars[host]['ip'] }}", + {%- endif -%} + {%- endfor -%} + "127.0.0.1","::1","{{ kube_apiserver_ip }}" + ] + issue_cert_path: "{{ item }}" + issue_cert_role: front-proxy-client + issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + issue_cert_mount_path: "{{ kube_vault_mount_path }}" + with_items: "{{ kube_master_components_certs_needed|d([]) }}" + when: inventory_hostname in groups['kube-master'] + notify: set secret_changed diff --git a/roles/vault/defaults/main.yml b/roles/vault/defaults/main.yml index 3e41cb00c..5c44dd80b 100644 --- a/roles/vault/defaults/main.yml +++ b/roles/vault/defaults/main.yml @@ -162,3 +162,11 @@ vault_pki_mounts: allow_any_name: true enforce_hostnames: false organization: "system:node-proxier" + - name: front-proxy-client + group: k8s-cluster + password: "{{ lookup('password', 'credentials/vault/kube-proxy length=15') }}" + policy_rules: default + role_options: + allow_any_name: true + enforce_hostnames: false + organization: "system:front-proxy"