From 0c7e1889e4c3c1a9f5757cf2503cdbd96c0d9484 Mon Sep 17 00:00:00 2001 From: Chad Swenson Date: Mon, 6 Nov 2017 14:01:10 -0600 Subject: [PATCH] Support for disabling apiserver insecure port This allows `kube_apiserver_insecure_port` to be set to 0 (disabled). It's working, but so far I have had to: 1. Make the `uri` module "Wait for apiserver up" checks use `kube_apiserver_port` (HTTPS) 2. Add apiserver client cert/key to the "Wait for apiserver up" checks 3. Update apiserver liveness probe to use HTTPS ports 4. Set `kube_api_anonymous_auth` to true to allow liveness probe to hit apiserver's /healthz over HTTPS (livenessProbes can't use client cert/key unfortunately) 5. RBAC has to be enabled. Anonymous requests are in the `system:unauthenticated` group which is granted access to /healthz by one of RBAC's default ClusterRoleBindings. An equivalent ABAC rule could allow this as well. Changes 1 and 2 should work for everyone, but 3, 4, and 5 require new coupling of currently independent configuration settings. So I also added a new settings check. Options: 1. The problem goes away if you have both anonymous-auth and RBAC enabled. This is how kubeadm does it. This may be the best way to go since RBAC is already on by default but anonymous auth is not. 2. Include conditional templates to set a different liveness probe for possible combinations of `kube_apiserver_insecure_port = 0`, RBAC, and `kube_api_anonymous_auth` (won't be possible to cover every case without a guaranteed authorizer for the secure port) 3. Use basic auth headers for the liveness probe (I really don't like this, it adds a new dependency on basic auth which I'd also like to leave independently configurable, and it requires encoded passwords in the apiserver manifest) Option 1 seems like the clear winner to me, but is there a reason we wouldn't want anonymous-auth on by default? The apiserver binary defaults anonymous-auth to true, but kubespray's default was false. --- inventory/group_vars/k8s-cluster.yml | 4 +++- roles/kubernetes-apps/ansible/tasks/main.yml | 5 ++++- roles/kubernetes-apps/cluster_roles/tasks/main.yml | 5 ++++- roles/kubernetes/master/handlers/main.yml | 5 ++++- .../templates/manifests/kube-apiserver.manifest.j2 | 12 ++++++++++-- .../kubernetes/preinstall/tasks/verify-settings.yml | 6 ++++++ 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index a400d05f9..f3830a521 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -20,7 +20,7 @@ kube_token_dir: "{{ kube_config_dir }}/tokens" # This is where to save basic auth file kube_users_dir: "{{ kube_config_dir }}/users" -kube_api_anonymous_auth: false +kube_api_anonymous_auth: true ## Change this to use another Kubernetes version, e.g. a current beta release kube_version: v1.8.2 @@ -106,6 +106,8 @@ kube_network_node_prefix: 24 kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}" kube_apiserver_port: 6443 # (https) kube_apiserver_insecure_port: 8080 # (http) +# Set to 0 to disable insecure port - Requires RBAC in authorization_modes and kube_api_anonymous_auth: true +#kube_apiserver_insecure_port: 0 # (disabled) # DNS configuration. # Kubernetes cluster name, also will be used as DNS domain diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index 025b4fab6..f4349669a 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -1,7 +1,10 @@ --- - name: Kubernetes Apps | Wait for kube-apiserver uri: - url: "{{ kube_apiserver_insecure_endpoint }}/healthz" + url: "{{ kube_apiserver_endpoint }}/healthz" + validate_certs: no + client_cert: "{{ kube_cert_dir }}/apiserver.pem" + client_key: "{{ kube_cert_dir }}/apiserver-key.pem" register: result until: result.status == 200 retries: 10 diff --git a/roles/kubernetes-apps/cluster_roles/tasks/main.yml b/roles/kubernetes-apps/cluster_roles/tasks/main.yml index 24f94aac5..75be11d4f 100644 --- a/roles/kubernetes-apps/cluster_roles/tasks/main.yml +++ b/roles/kubernetes-apps/cluster_roles/tasks/main.yml @@ -1,7 +1,10 @@ --- - name: Kubernetes Apps | Wait for kube-apiserver uri: - url: "{{ kube_apiserver_insecure_endpoint }}/healthz" + url: "{{ kube_apiserver_endpoint }}/healthz" + validate_certs: no + client_cert: "{{ kube_cert_dir }}/apiserver.pem" + client_key: "{{ kube_cert_dir }}/apiserver-key.pem" register: result until: result.status == 200 retries: 10 diff --git a/roles/kubernetes/master/handlers/main.yml b/roles/kubernetes/master/handlers/main.yml index 1c6dc956c..02f0b62b9 100644 --- a/roles/kubernetes/master/handlers/main.yml +++ b/roles/kubernetes/master/handlers/main.yml @@ -39,7 +39,10 @@ - name: Master | wait for the apiserver to be running uri: - url: "{{ kube_apiserver_insecure_endpoint }}/healthz" + url: "{{ kube_apiserver_endpoint }}/healthz" + validate_certs: no + client_cert: "{{ kube_cert_dir }}/apiserver.pem" + client_key: "{{ kube_cert_dir }}/apiserver-key.pem" register: result until: result.status == 200 retries: 20 diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 5d4f6cf47..2d0f0c9fb 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -110,9 +110,17 @@ spec: httpGet: host: 127.0.0.1 path: /healthz +{% if kube_apiserver_insecure_port == 0 %} + port: {{ kube_apiserver_port }} + scheme: HTTPS +{% else %} port: {{ kube_apiserver_insecure_port }} - initialDelaySeconds: 30 - timeoutSeconds: 10 +{% endif %} + failureThreshold: 8 + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 15 volumeMounts: - mountPath: {{ kube_config_dir }} name: kubernetes-config diff --git a/roles/kubernetes/preinstall/tasks/verify-settings.yml b/roles/kubernetes/preinstall/tasks/verify-settings.yml index 9dbd7ab8c..b7bf2d664 100644 --- a/roles/kubernetes/preinstall/tasks/verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/verify-settings.yml @@ -78,3 +78,9 @@ that: ansible_swaptotal_mb == 0 when: kubelet_fail_swap_on|default(true) ignore_errors: "{{ ignore_assert_errors }}" + +- name: Stop if RBAC and anonymous-auth are not enabled when insecure port is disabled + assert: + that: rbac_enabled and kube_api_anonymous_auth + when: kube_apiserver_insecure_port == 0 + ignore_errors: "{{ ignore_assert_errors }}" \ No newline at end of file