From c8a2fe321b32497c8a2f8e2b502e0137b725dfc9 Mon Sep 17 00:00:00 2001 From: Raj Perera Date: Fri, 16 Jun 2017 10:28:23 -0400 Subject: [PATCH] Basic RBAC functionality. (Based from work done by @jwfang (#1351)) * Add a flag "authorization_method", when set to "RBAC" enables role based access control. * Add required cluster roles and bindings for kube-dns * Patch tiller deployment to use a service account with proper credentials. * Add a flag to regenerate kubernetes certs on the nodes. --- docs/vars.md | 7 +++- .../kubernetes-apps/ansible/defaults/main.yml | 8 +++++ roles/kubernetes-apps/ansible/tasks/main.yml | 9 ++++-- .../kubedns-autoscaler-clusterrole.yml | 32 +++++++++++++++++++ .../kubedns-autoscaler-clusterrolebinding.yml | 27 ++++++++++++++++ .../templates/kubedns-autoscaler-sa.yml | 19 +++++++++++ .../ansible/templates/kubedns-autoscaler.yml | 3 +- .../ansible/templates/kubedns-deploy.yml | 4 ++- roles/kubernetes-apps/helm/defaults/main.yml | 2 ++ roles/kubernetes-apps/helm/tasks/main.yml | 18 +++++++++++ roles/kubernetes/master/defaults/main.yml | 4 ++- .../manifests/kube-apiserver.manifest.j2 | 3 ++ .../kube-controller-manager.manifest.j2 | 3 ++ roles/kubernetes/node/handlers/main.yml | 5 +++ roles/kubernetes/node/tasks/pre_upgrade.yml | 4 +++ .../manifests/kube-proxy.manifest.j2 | 8 ++++- roles/kubernetes/secrets/defaults/main.yml | 2 ++ .../kubernetes/secrets/tasks/check-certs.yml | 10 +++--- .../secrets/tasks/gen_certs_script.yml | 2 -- 19 files changed, 156 insertions(+), 14 deletions(-) create mode 100644 roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrole.yml create mode 100644 roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrolebinding.yml create mode 100644 roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-sa.yml diff --git a/docs/vars.md b/docs/vars.md index 603a614b2..b3aee535a 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -67,7 +67,11 @@ following default cluster paramters: OpenStack (default is unset) * *kube_hostpath_dynamic_provisioner* - Required for use of PetSets type in Kubernetes - +* *authorization_mode* - Set this to "RBAC" (upper-case, no quotes) +[to enable Role Based Access Control](https://kubernetes.io/docs/admin/authorization/rbac/) +* *rotate_kubernetes_certs* - Set this to true to regenerate kubernetes Node certificates. *Warning: Will overwrite old certs.* + + Note, if cloud providers have any use of the ``10.233.0.0/16``, like instances' private addresses, make sure to pick another values for ``kube_service_addresses`` and ``kube_pods_subnet``, for example from the ``172.18.0.0/16``. @@ -116,3 +120,4 @@ The possible vars are: Kargo sets up two Kubernetes accounts by default: ``root`` and ``kube``. Their passwords default to changeme. You can set this by changing ``kube_api_pwd``. + diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 89bdd4277..a5329c635 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -40,3 +40,11 @@ netchecker_server_memory_requests: 64M # SSL etcd_cert_dir: "/etc/ssl/etcd/ssl" canal_cert_dir: "/etc/canal/certs" + +# RBAC +rbac_resources: + - clusterrole, + - clusterrolebinding, + - sa + +rbac_enabled: "{{ authorization_mode == 'RBAC' }}" \ No newline at end of file diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index 5225bbda3..2c4b14ad8 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -13,12 +13,15 @@ src: "{{item.file}}" dest: "{{kube_config_dir}}/{{item.file}}" with_items: - - {name: kube-dns, file: kubedns-serviceaccount.yml, type: serviceaccount} + - {name: kubedns, file: kubedns-sa.yml, type: sa} - {name: kubedns, file: kubedns-deploy.yml, type: deployment} - {name: kubedns, file: kubedns-svc.yml, type: svc} + - {name: kubedns-autoscaler, file: kubedns-autoscaler-sa.yml, type: sa} + - {name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrole.yml, type: clusterrole} + - {name: kubedns-autoscaler, file: kubedns-autoscaler-clusterrolebinding.yml, type: clusterrolebinding} - {name: kubedns-autoscaler, file: kubedns-autoscaler.yml, type: deployment} register: manifests - when: dns_mode != 'none' and inventory_hostname == groups['kube-master'][0] + when: dns_mode != 'none' and inventory_hostname == groups['kube-master'][0] and (item.type not in rbac_resources or rbac_enabled) tags: dnsmasq - name: Kubernetes Apps | Start Resources @@ -30,6 +33,7 @@ filename: "{{kube_config_dir}}/{{item.item.file}}" state: "{{item.changed | ternary('latest','present') }}" with_items: "{{ manifests.results }}" + failed_when: manifests|failed and "Error from server (AlreadyExists)" not in manifests.msg when: dns_mode != 'none' and inventory_hostname == groups['kube-master'][0] tags: dnsmasq @@ -37,3 +41,4 @@ include: tasks/netchecker.yml when: deploy_netchecker tags: netchecker + diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrole.yml b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrole.yml new file mode 100644 index 000000000..b3324abf2 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrole.yml @@ -0,0 +1,32 @@ +# Copyright 2016 The Kubernetes Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: cluster-proportional-autoscaler + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] + - apiGroups: [""] + resources: ["replicationcontrollers/scale"] + verbs: ["get", "update"] + - apiGroups: ["extensions"] + resources: ["deployments/scale", "replicasets/scale"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create"] diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrolebinding.yml b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrolebinding.yml new file mode 100644 index 000000000..c4cdda9f1 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-clusterrolebinding.yml @@ -0,0 +1,27 @@ +# Copyright 2016 The Kubernetes Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: cluster-proportional-autoscaler + namespace: kube-system +subjects: + - kind: ServiceAccount + name: cluster-proportional-autoscaler + namespace: kube-system +roleRef: + kind: ClusterRole + name: cluster-proportional-autoscaler + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-sa.yml b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-sa.yml new file mode 100644 index 000000000..521f12ad9 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler-sa.yml @@ -0,0 +1,19 @@ +# Copyright 2016 The Kubernetes Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: ServiceAccount +apiVersion: v1 +metadata: + name: cluster-proportional-autoscaler + namespace: kube-system diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml index c0f519e2c..829df512e 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml +++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml @@ -46,4 +46,5 @@ spec: - --default-params={"linear":{"nodesPerReplica":{{ kubedns_nodes_per_replica }},"min":{{ kubedns_min_replicas }}}} - --logtostderr=true - --v=2 - + serviceAccountName: cluster-proportional-autoscaler + serviceAccount: cluster-proportional-autoscaler diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml index 4c7a7eec7..32d307d76 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml +++ b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml @@ -114,4 +114,6 @@ spec: - containerPort: 8080 protocol: TCP dnsPolicy: Default # Don't use cluster DNS. - serviceAccountName: kube-dns +{% if authorization_mode is defined and authorization_mode == "RBAC" %} + serviceAccount: kube-dns +{% endif %} diff --git a/roles/kubernetes-apps/helm/defaults/main.yml b/roles/kubernetes-apps/helm/defaults/main.yml index b1b2dfca9..5cb600439 100644 --- a/roles/kubernetes-apps/helm/defaults/main.yml +++ b/roles/kubernetes-apps/helm/defaults/main.yml @@ -2,3 +2,5 @@ helm_enabled: false # specify a dir and attach it to helm for HELM_HOME. helm_home_dir: "/root/.helm" + +rbac_enabled: "{{ authorization_mode == 'RBAC' }}" \ No newline at end of file diff --git a/roles/kubernetes-apps/helm/tasks/main.yml b/roles/kubernetes-apps/helm/tasks/main.yml index f12875da2..38aeed669 100644 --- a/roles/kubernetes-apps/helm/tasks/main.yml +++ b/roles/kubernetes-apps/helm/tasks/main.yml @@ -10,10 +10,28 @@ mode: 0755 register: helm_container +- name: Helm | Configure tiller service account for RBAC + command: kubectl create serviceaccount tiller --namespace=kube-system + ignore_errors: yes + when: rbac_enabled + +- name: Helm | Configure tiller rolebindings for RBAC + command: kubectl create clusterrolebinding tiller --clusterrole=cluster-admin --serviceaccount=kube-system:tiller + ignore_errors: yes + when: rbac_enabled + - name: Helm | Install/upgrade helm command: "{{ bin_dir }}/helm init --upgrade --tiller-image={{ tiller_image_repo }}:{{ tiller_image_tag }}" when: helm_container.changed +- name: Helm | Patch tiller deployment for RBAC + shell: > + kubectl --namespace=kube-system get deployment tiller-deploy -o json | \ + python -c 'import sys,json;a=json.load(sys.stdin);a["spec"]["template"]["spec"]["serviceAccount"]="tiller";json.dump(a,sys.stdout)' | \ + kubectl apply -f - + when: rbac_enabled + - name: Helm | Set up bash completion shell: "umask 022 && {{ bin_dir }}/helm completion bash >/etc/bash_completion.d/helm.sh" when: ( helm_container.changed and not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] ) + diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 785ef43af..96901e235 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -64,4 +64,6 @@ apiserver_custom_flags: [] controller_mgr_custom_flags: [] -scheduler_custom_flags: [] \ No newline at end of file +scheduler_custom_flags: [] + +authorization_mode: RBAC \ No newline at end of file diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 851cca060..5a2101b73 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -81,6 +81,9 @@ spec: {% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=') %} - --anonymous-auth={{ kube_api_anonymous_auth }} {% endif %} +{% if authorization_mode %} + - --authorization-mode={{ authorization_mode }} +{% endif %} {% if apiserver_custom_flags is string %} - {{ apiserver_custom_flags }} {% else %} diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index f65bb004c..a5171f82f 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -35,6 +35,9 @@ spec: - --node-monitor-period={{ kube_controller_node_monitor_period }} - --pod-eviction-timeout={{ kube_controller_pod_eviction_timeout }} - --v={{ kube_log_level }} +{% if authorization_mode is defined and authorization_mode == "RBAC" %} + - --use-service-account-credentials +{% endif %} {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %} - --cloud-provider={{cloud_provider}} - --cloud-config={{ kube_config_dir }}/cloud_config diff --git a/roles/kubernetes/node/handlers/main.yml b/roles/kubernetes/node/handlers/main.yml index 00525b995..079cbd3da 100644 --- a/roles/kubernetes/node/handlers/main.yml +++ b/roles/kubernetes/node/handlers/main.yml @@ -1,4 +1,9 @@ --- +- name: restart kubelet if secrets changed + command: /bin/true + when: secret_changed|d(False) + notify: restart kubelet + - name: restart kubelet command: /bin/true notify: diff --git a/roles/kubernetes/node/tasks/pre_upgrade.yml b/roles/kubernetes/node/tasks/pre_upgrade.yml index 612dd3e6f..291817562 100644 --- a/roles/kubernetes/node/tasks/pre_upgrade.yml +++ b/roles/kubernetes/node/tasks/pre_upgrade.yml @@ -4,3 +4,7 @@ args: creates: "/var/lib/cni" failed_when: false + +- name: "Pre-upgrade | Make sure to restart kubelet if certificates changed" + command: /bin/true + notify: restart kubelet if secrets changed \ No newline at end of file diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index bbb13bc9d..d28621cfd 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -44,6 +44,9 @@ spec: - mountPath: "{{ kube_config_dir }}/ssl" name: etc-kube-ssl readOnly: true + - mountPath: {{kube_config_dir}}/kube-proxy-kubeconfig.yaml + name: "kubeconfig" + readOnly: true - mountPath: "{{ kube_config_dir }}/kube-proxy-kubeconfig.yaml" name: kubeconfig readOnly: true @@ -63,7 +66,10 @@ spec: path: "{{ kube_config_dir }}/ssl" - name: kubeconfig hostPath: - path: "{{ kube_config_dir }}/kube-proxy-kubeconfig.yaml" + path: "{{kube_config_dir}}/kube-proxy-kubeconfig.yaml" + - name: "etc-kube-ssl" + hostPath: + path: "{{kube_config_dir}}/ssl" - name: var-run-dbus hostPath: path: /var/run/dbus diff --git a/roles/kubernetes/secrets/defaults/main.yml b/roles/kubernetes/secrets/defaults/main.yml index e6177857e..3b65c23a9 100644 --- a/roles/kubernetes/secrets/defaults/main.yml +++ b/roles/kubernetes/secrets/defaults/main.yml @@ -1,2 +1,4 @@ --- kube_cert_group: kube-cert + +rotate_kubernetes_certs: false # set this to true to regenerate certificates diff --git a/roles/kubernetes/secrets/tasks/check-certs.yml b/roles/kubernetes/secrets/tasks/check-certs.yml index 061b04b04..fafc20c1e 100644 --- a/roles/kubernetes/secrets/tasks/check-certs.yml +++ b/roles/kubernetes/secrets/tasks/check-certs.yml @@ -10,8 +10,8 @@ - name: "Check_certs | Set default value for 'sync_certs', 'gen_certs', and 'secret_changed' to false" set_fact: - sync_certs: false - gen_certs: false + sync_certs: true + gen_certs: true secret_changed: false - name: "Check certs | check if a cert already exists on node" @@ -25,7 +25,7 @@ - name: "Check_certs | Set 'gen_certs' to true" set_fact: gen_certs: true - when: "not item in kubecert_master.files|map(attribute='path') | list" + when: "rotate_kubernetes_certs or item not in (kubecert_master.files|map(attribute='path')|list)" run_once: true with_items: >- ['{{ kube_cert_dir }}/ca.pem', @@ -41,7 +41,7 @@ {% set existing_certs = kubecert_master.files|map(attribute='path')|list|sort %} {% for host in groups['k8s-cluster'] -%} {% set host_cert = "%s/node-%s-key.pem"|format(kube_cert_dir, host) %} - {% if host_cert in existing_certs -%} + {% if host_cert in existing_certs and not rotate_kubernetes_certs -%} "{{ host }}": False, {% else -%} "{{ host }}": True, @@ -62,5 +62,5 @@ (kubecert_node.results[1].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[1].stat.path)|map(attribute="checksum")|first|default('')) -%} {%- set _ = certs.update({'sync': True}) -%} {% endif %} - {{ certs.sync }} + {{ rotate_kubernetes_certs or certs.sync }} diff --git a/roles/kubernetes/secrets/tasks/gen_certs_script.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml index 61d9c7826..54afed35d 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs_script.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml @@ -136,7 +136,6 @@ - name: Gen_certs | Unpack certs on masters shell: "base64 -d < {{ cert_tempfile.stdout }} | tar xz -C {{ kube_cert_dir }}" no_log: true - changed_when: false check_mode: no when: inventory_hostname in groups['kube-master'] and sync_certs|default(false) and inventory_hostname != groups['kube-master'][0] @@ -154,7 +153,6 @@ args: executable: /bin/bash no_log: true - changed_when: false check_mode: no when: inventory_hostname in groups['kube-node'] and sync_certs|default(false) and