From 0c6f172e75da39a8676a8fdd9cc40d439396e02d Mon Sep 17 00:00:00 2001 From: Chad Swenson Date: Thu, 9 Nov 2017 15:59:30 -0600 Subject: [PATCH 1/2] Kubernetes Dashboard v1.7.1 Refactor This version required changing the previous access model for dashboard completely but it's a change for the better. Docs were updated. * New login/auth options that use apiserver auth proxying by default * Requires RBAC in `authorization_modes` * Only serves over https * No longer available at https://first_master:6443/ui until apiserver is updated with the https proxy URL: * Can access from https://first_master:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login you will be prompted for credentials * Or you can run 'kubectl proxy' from your local machine to access dashboard in your browser from: http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/ * It is recommended to access dashboard from behind a gateway that enforces an authentication token, details and other access options here: https://github.com/kubernetes/dashboard/wiki/Accessing-Dashboard---1.7.X-and-above --- docs/getting-started.md | 19 +-- inventory/group_vars/k8s-cluster.yml | 3 +- .../kubernetes-apps/ansible/defaults/main.yml | 4 +- .../ansible/tasks/dashboard.yml | 12 +- .../ansible/templates/dashboard.yml.j2 | 129 +++++++++++++++--- .../preinstall/tasks/verify-settings.yml | 7 + 6 files changed, 142 insertions(+), 32 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index e9e6d2f91..85ae0ae72 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -93,18 +93,19 @@ the Kubernetes [documentation](https://kubernetes.io/docs/tasks/access-applicati Accessing Kubernetes Dashboard ------------------------------ -If the variable `dashboard_enabled` is set (default is true) as well as -kube_basic_auth (default is false), then you can -access the Kubernetes Dashboard at the following URL: +As of kubernetes-dashboard v1.7.x: +* New login options that use apiserver auth proxying of token/basic/kubeconfig by default +* Requires RBAC in authorization_modes +* Only serves over https +* No longer available at https://first_master:6443/ui until apiserver is updated with the https proxy URL - https://kube:_kube-password_@_host_:6443/ui/ +If the variable `dashboard_enabled` is set (default is true), then you can access the Kubernetes Dashboard at the following URL, You will be prompted for credentials: +https://first_master:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login -To see the password, refer to the section above, titled *Connecting to -Kubernetes*. The host can be any kube-master or kube-node or loadbalancer -(when enabled). +Or you can run 'kubectl proxy' from your local machine to access dashboard in your browser from: +http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login -To access the Dashboard with basic auth disabled, follow the instructions here: -https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/#command-line-proxy +It is recommended to access dashboard from behind a gateway (like Ingress Controller) that enforces an authentication token. Details and other access options here: https://github.com/kubernetes/dashboard/wiki/Accessing-Dashboard---1.7.X-and-above Accessing Kubernetes API ------------------------ diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index a400d05f9..2ec789440 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -143,7 +143,8 @@ helm_deployment_type: docker # K8s image pull policy (imagePullPolicy) k8s_image_pull_policy: IfNotPresent -# Kubernetes dashboard (available at http://first_master:6443/ui by default) +# Kubernetes dashboard +# RBAC required. see docs/getting-started.md for access details. dashboard_enabled: true # Monitoring apps for k8s diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index e40fcceb6..b8f9cc206 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -41,7 +41,9 @@ netchecker_server_memory_requests: 64M # Dashboard dashboard_enabled: false dashboard_image_repo: gcr.io/google_containers/kubernetes-dashboard-amd64 -dashboard_image_tag: v1.6.3 +dashboard_image_tag: v1.7.1 +dashboard_init_image_repo: gcr.io/google_containers/kubernetes-dashboard-init-amd64 +dashboard_init_image_tag: v1.0.1 # Limits for dashboard dashboard_cpu_limit: 100m diff --git a/roles/kubernetes-apps/ansible/tasks/dashboard.yml b/roles/kubernetes-apps/ansible/tasks/dashboard.yml index 63ea3cf70..530796c21 100644 --- a/roles/kubernetes-apps/ansible/tasks/dashboard.yml +++ b/roles/kubernetes-apps/ansible/tasks/dashboard.yml @@ -1,10 +1,20 @@ --- +- name: Kubernetes Apps | Delete old kubernetes-dashboard resources + kube: + name: "kubernetes-dashboard" + kubectl: "{{bin_dir}}/kubectl" + resource: "{{ item }}" + state: absent + with_items: ['ClusterRoleBinding'] + tags: + - upgrade + - name: Kubernetes Apps | Lay down dashboard template template: src: "{{item.file}}" dest: "{{kube_config_dir}}/{{item.file}}" with_items: - - {file: dashboard.yml.j2, type: deploy, name: netchecker-agent} + - {file: dashboard.yml.j2, type: deploy, name: kubernetes-dashboard} register: manifests when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 b/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 index ac32b1c7f..35415326e 100644 --- a/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 @@ -1,4 +1,4 @@ -# Copyright 2015 Google Inc. All Rights Reserved. +# Copyright 2017 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,12 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Configuration to deploy head version of the Dashboard UI compatible with -# Kubernetes 1.6 (RBAC enabled). +# Configuration to deploy release version of the Dashboard UI compatible with +# Kubernetes 1.7. # # Example usage: kubectl create -f -{% if rbac_enabled %} +# ------------------- Dashboard Secret ------------------- # + +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-certs + namespace: {{ system_namespace }} +type: Opaque + +--- +# ------------------- Dashboard Service Account ------------------- # + apiVersion: v1 kind: ServiceAccount metadata: @@ -25,23 +38,77 @@ metadata: k8s-app: kubernetes-dashboard name: kubernetes-dashboard namespace: {{ system_namespace }} + +--- +# ------------------- Dashboard Role & Role Binding ------------------- # + +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: kubernetes-dashboard-minimal + namespace: {{ system_namespace }} +rules: + # Allow Dashboard to create and watch for changes of 'kubernetes-dashboard-key-holder' secret. +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "watch"] +- apiGroups: [""] + resources: ["secrets"] + # Allow Dashboard to get, update and delete 'kubernetes-dashboard-key-holder' secret. + resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"] + verbs: ["get", "update", "delete"] + # Allow Dashboard to get metrics from heapster. +- apiGroups: [""] + resources: ["services"] + resourceNames: ["heapster"] + verbs: ["proxy"] + --- apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding +kind: RoleBinding metadata: - name: kubernetes-dashboard - labels: - k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-minimal + namespace: {{ system_namespace }} roleRef: apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin + kind: Role + name: kubernetes-dashboard-minimal subjects: - kind: ServiceAccount name: kubernetes-dashboard namespace: {{ system_namespace }} -{% endif %} + --- +# ------------------- Gross Hack For anonymous auth through api proxy ------------------- # +# Allows users to reach login page and other proxied dashboard URLs +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kubernetes-dashboard-anonymous +rules: +- apiGroups: [""] + resources: ["services/proxy"] + resourceNames: ["https:kubernetes-dashboard:"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- nonResourceURLs: ["/ui", "/ui/*", "/api/v1/namespaces/{{ system_namespace }}/services/https:kubernetes-dashboard:/proxy/*"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-dashboard-anonymous +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubernetes-dashboard-anonymous +subjects: +- kind: User + name: system:anonymous + +--- +# ------------------- Dashboard Deployment ------------------- # + kind: Deployment apiVersion: extensions/v1beta1 metadata: @@ -60,10 +127,15 @@ spec: labels: k8s-app: kubernetes-dashboard spec: + initContainers: + - name: kubernetes-dashboard-init + image: {{ dashboard_init_image_repo }}:{{ dashboard_init_image_tag }} + volumeMounts: + - name: kubernetes-dashboard-certs + mountPath: /certs containers: - name: kubernetes-dashboard image: {{ dashboard_image_repo }}:{{ dashboard_image_tag }} - # Image is tagged and updated with :head, so always pull it. imagePullPolicy: Always resources: limits: @@ -73,27 +145,45 @@ spec: cpu: {{ dashboard_cpu_requests }} memory: {{ dashboard_memory_requests }} ports: - - containerPort: 9090 + - containerPort: 8443 protocol: TCP args: + - --tls-key-file=/certs/dashboard.key + - --tls-cert-file=/certs/dashboard.crt + - --authentication-mode=token{% if kube_basic_auth|default(false) %},basic{% endif %} # Uncomment the following line to manually specify Kubernetes API server Host # If not specified, Dashboard will attempt to auto discover the API server and connect # to it. Uncomment only if the default does not work. # - --apiserver-host=http://my-address:port + volumeMounts: + - name: kubernetes-dashboard-certs + mountPath: /certs + readOnly: true + # Create on-disk volume to store exec logs + - mountPath: /tmp + name: tmp-volume livenessProbe: httpGet: + scheme: HTTPS path: / - port: 9090 + port: 8443 initialDelaySeconds: 30 timeoutSeconds: 30 -{% if rbac_enabled %} + volumes: + - name: kubernetes-dashboard-certs + secret: + secretName: kubernetes-dashboard-certs + - name: tmp-volume + emptyDir: {} serviceAccountName: kubernetes-dashboard -{% endif %} # Comment the following tolerations if Dashboard must not be deployed on master tolerations: - key: node-role.kubernetes.io/master effect: NoSchedule + --- +# ------------------- Dashboard Service ------------------- # + kind: Service apiVersion: v1 metadata: @@ -103,8 +193,7 @@ metadata: namespace: {{ system_namespace }} spec: ports: - - port: 80 - targetPort: 9090 + - port: 443 + targetPort: 8443 selector: - k8s-app: kubernetes-dashboard - + k8s-app: kubernetes-dashboard \ No newline at end of file diff --git a/roles/kubernetes/preinstall/tasks/verify-settings.yml b/roles/kubernetes/preinstall/tasks/verify-settings.yml index 9dbd7ab8c..9ae665287 100644 --- a/roles/kubernetes/preinstall/tasks/verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/verify-settings.yml @@ -78,3 +78,10 @@ that: ansible_swaptotal_mb == 0 when: kubelet_fail_swap_on|default(true) ignore_errors: "{{ ignore_assert_errors }}" + + +- name: Stop if RBAC is not enabled when dashboard is enabled + assert: + that: rbac_enabled + when: dashboard_enabled + ignore_errors: "{{ ignore_assert_errors }}" \ No newline at end of file From a89ee8c4067c1a695b430ff3a731e6a4a17098c2 Mon Sep 17 00:00:00 2001 From: Chad Swenson Date: Mon, 13 Nov 2017 13:59:31 -0600 Subject: [PATCH 2/2] Add ability to use custom cert secret instead of init container provisioned self-signed certs --- roles/kubernetes-apps/ansible/defaults/main.yml | 6 ++++++ roles/kubernetes-apps/ansible/tasks/dashboard.yml | 3 ++- roles/kubernetes-apps/ansible/tasks/main.yml | 4 +++- roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 | 8 +++++--- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index b8f9cc206..5951086e9 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -54,6 +54,12 @@ dashboard_memory_requests: 64M # SSL etcd_cert_dir: "/etc/ssl/etcd/ssl" canal_cert_dir: "/etc/canal/certs" +# Set dashboard_use_custom_certs to true if overriding dashboard_certs_secret_name with a secret that +# contains dashboard_tls_key_file and dashboard_tls_cert_file instead of using the initContainer provisioned certs +dashboard_use_custom_certs: false +dashboard_certs_secret_name: kubernetes-dashboard-certs +dashboard_tls_key_file: dashboard.key +dashboard_tls_cert_file: dashboard.crt rbac_resources: - sa diff --git a/roles/kubernetes-apps/ansible/tasks/dashboard.yml b/roles/kubernetes-apps/ansible/tasks/dashboard.yml index 530796c21..84816127e 100644 --- a/roles/kubernetes-apps/ansible/tasks/dashboard.yml +++ b/roles/kubernetes-apps/ansible/tasks/dashboard.yml @@ -5,7 +5,8 @@ kubectl: "{{bin_dir}}/kubectl" resource: "{{ item }}" state: absent - with_items: ['ClusterRoleBinding'] + with_items: + - 'ClusterRoleBinding' tags: - upgrade diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index 025b4fab6..7b36d4536 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -15,7 +15,9 @@ kubectl: "{{bin_dir}}/kubectl" resource: "{{ item }}" state: absent - with_items: ['deploy', 'svc'] + with_items: + - 'deploy' + - 'svc' tags: - upgrade diff --git a/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 b/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 index 35415326e..b16ddd467 100644 --- a/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 @@ -127,12 +127,14 @@ spec: labels: k8s-app: kubernetes-dashboard spec: +{% if not dashboard_use_custom_certs %} initContainers: - name: kubernetes-dashboard-init image: {{ dashboard_init_image_repo }}:{{ dashboard_init_image_tag }} volumeMounts: - name: kubernetes-dashboard-certs mountPath: /certs +{% endif %} containers: - name: kubernetes-dashboard image: {{ dashboard_image_repo }}:{{ dashboard_image_tag }} @@ -148,8 +150,8 @@ spec: - containerPort: 8443 protocol: TCP args: - - --tls-key-file=/certs/dashboard.key - - --tls-cert-file=/certs/dashboard.crt + - --tls-key-file=/certs/{{ dashboard_tls_key_file }} + - --tls-cert-file=/certs/{{ dashboard_tls_cert_file }} - --authentication-mode=token{% if kube_basic_auth|default(false) %},basic{% endif %} # Uncomment the following line to manually specify Kubernetes API server Host # If not specified, Dashboard will attempt to auto discover the API server and connect @@ -172,7 +174,7 @@ spec: volumes: - name: kubernetes-dashboard-certs secret: - secretName: kubernetes-dashboard-certs + secretName: {{ dashboard_certs_secret_name }} - name: tmp-volume emptyDir: {} serviceAccountName: kubernetes-dashboard