From c1fd46868739ae765fee3d970e9599df177e636e Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Thu, 31 Aug 2017 07:16:57 +0100 Subject: [PATCH] Add RBAC support for canal Refactored how rbac_enabled is set Added RBAC to ubuntu-canal-ha CI job --- .gitlab-ci.yml | 11 +-- .../network_plugin/canal/tasks/main.yml | 25 ++---- roles/network_plugin/canal/defaults/main.yml | 5 ++ roles/network_plugin/canal/tasks/main.yml | 23 +++--- .../canal/templates/canal-cr-calico.yml.j2 | 80 +++++++++++++++++++ .../canal/templates/canal-cr-flannel.yml.j2 | 25 ++++++ .../canal/templates/canal-crb-calico.yml.j2 | 14 ++++ .../canal/templates/canal-crb-flannel.yml.j2 | 14 ++++ .../canal/templates/canal-node-sa.yml.j2 | 9 +++ .../canal/templates/canal-node.yml.j2 | 1 + tests/cloud_playbooks/create-gce.yml | 2 +- 11 files changed, 177 insertions(+), 32 deletions(-) create mode 100644 roles/network_plugin/canal/templates/canal-cr-calico.yml.j2 create mode 100644 roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2 create mode 100644 roles/network_plugin/canal/templates/canal-crb-calico.yml.j2 create mode 100644 roles/network_plugin/canal/templates/canal-crb-flannel.yml.j2 create mode 100644 roles/network_plugin/canal/templates/canal-node-sa.yml.j2 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a456f9df..17851b19c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -269,9 +269,10 @@ before_script: ##User-data to simply turn off coreos upgrades STARTUP_SCRIPT: 'systemctl disable locksmithd && systemctl stop locksmithd' -.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables +.ubuntu_canal_ha_rbac_variables: &ubuntu_canal_ha_rbac_variables # stage: deploy-gce-part1 KUBE_NETWORK_PLUGIN: canal + AUTHORIZATION_MODES: "{ 'authorization_modes': [ 'RBAC' ] }" CLOUD_IMAGE: ubuntu-1604-xenial CLOUD_REGION: europe-west1-b CLUSTER_MODE: ha @@ -445,24 +446,24 @@ ubuntu-weave-sep-triggers: only: ['triggers'] # More builds for PRs/merges (manual) and triggers (auto) -ubuntu-canal-ha: +ubuntu-canal-ha-rbac: stage: deploy-gce-part1 <<: *job <<: *gce variables: <<: *gce_variables - <<: *ubuntu_canal_ha_variables + <<: *ubuntu_canal_ha_rbac_variables when: manual except: ['triggers'] only: ['master', /^pr-.*$/] -ubuntu-canal-ha-triggers: +ubuntu-canal-ha-rbac-triggers: stage: deploy-gce-part1 <<: *job <<: *gce variables: <<: *gce_variables - <<: *ubuntu_canal_ha_variables + <<: *ubuntu_canal_ha_rbac_variables when: on_success only: ['triggers'] diff --git a/roles/kubernetes-apps/network_plugin/canal/tasks/main.yml b/roles/kubernetes-apps/network_plugin/canal/tasks/main.yml index 72956dac9..fe820bd11 100644 --- a/roles/kubernetes-apps/network_plugin/canal/tasks/main.yml +++ b/roles/kubernetes-apps/network_plugin/canal/tasks/main.yml @@ -1,20 +1,11 @@ --- -- name: Create canal ConfigMap - run_once: true +- name: Canal | Start Resources kube: - name: "canal-config" + name: "{{item.item.name}}" + namespace: "{{ system_namespace }}" kubectl: "{{bin_dir}}/kubectl" - filename: "{{kube_config_dir}}/canal-config.yaml" - resource: "configmap" - namespace: "{{system_namespace}}" - -- name: Start flannel and calico-node - run_once: true - kube: - name: "canal-node" - kubectl: "{{bin_dir}}/kubectl" - filename: "{{kube_config_dir}}/canal-node.yaml" - resource: "ds" - namespace: "{{system_namespace}}" - state: "{{ item | ternary('latest','present') }}" - with_items: "{{ canal_node_manifest.changed }}" + resource: "{{item.item.type}}" + filename: "{{kube_config_dir}}/{{item.item.file}}" + state: "{{item.changed | ternary('latest','present') }}" + with_items: "{{ canal_manifests.results }}" + failed_when: canal_manifests|failed and "Error from server (AlreadyExists)" not in canal_manifests.msg diff --git a/roles/network_plugin/canal/defaults/main.yml b/roles/network_plugin/canal/defaults/main.yml index 38696b87a..bf74653c7 100644 --- a/roles/network_plugin/canal/defaults/main.yml +++ b/roles/network_plugin/canal/defaults/main.yml @@ -31,3 +31,8 @@ calicoctl_memory_limit: 170M calicoctl_cpu_limit: 100m calicoctl_memory_requests: 32M calicoctl_cpu_requests: 25m + +rbac_resources: + - sa + - clusterrole + - clusterrolebinding diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml index ea67e20cd..5283b9b41 100644 --- a/roles/network_plugin/canal/tasks/main.yml +++ b/roles/network_plugin/canal/tasks/main.yml @@ -32,16 +32,21 @@ delegate_to: "{{groups['etcd'][0]}}" run_once: true -- name: Canal | Write canal configmap +- name: Canal | Create canal node rbac configuration template: - src: canal-config.yml.j2 - dest: "{{kube_config_dir}}/canal-config.yaml" - -- name: Canal | Write canal node configuration - template: - src: canal-node.yml.j2 - dest: "{{kube_config_dir}}/canal-node.yaml" - register: canal_node_manifest + src: "{{item.file}}.j2" + dest: "{{kube_config_dir}}/{{item.file}}" + with_items: + - {name: canal-config, file: canal-config.yml, type: cm} + - {name: canal-node, file: canal-node.yml, type: ds} + - {name: canal, file: canal-node-sa.yml, type: sa} + - {name: calico, file: canal-cr-calico.yml, type: clusterrole} + - {name: flannel, file: canal-cr-flannel.yml, type: clusterrole} + - {name: canal-calico, file: canal-cr-calico.yml, type: clusterrolebinding} + - {name: canal-flannel, file: canal-cr-flannel.yml, type: clusterrolebinding} + register: canal_manifests + when: + - rbac_enabled or item.type not in rbac_resources - name: Canal | Copy cni plugins from hyperkube command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -ac /opt/cni/bin/ /cnibindir/" diff --git a/roles/network_plugin/canal/templates/canal-cr-calico.yml.j2 b/roles/network_plugin/canal/templates/canal-cr-calico.yml.j2 new file mode 100644 index 000000000..e3b048c64 --- /dev/null +++ b/roles/network_plugin/canal/templates/canal-cr-calico.yml.j2 @@ -0,0 +1,80 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: calico + namespace: {{ system_namespace }} +rules: + - apiGroups: [""] + resources: + - namespaces + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - pods/status + verbs: + - update + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - update + - watch + - apiGroups: ["extensions"] + resources: + - thirdpartyresources + verbs: + - create + - get + - list + - watch + - apiGroups: ["extensions"] + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: ["projectcalico.org"] + resources: + - globalbgppeers + verbs: + - get + - list + - apiGroups: ["projectcalico.org"] + resources: + - globalconfigs + - globalbgpconfigs + verbs: + - create + - get + - list + - update + - watch + - apiGroups: ["projectcalico.org"] + resources: + - ippools + verbs: + - create + - get + - list + - update + - watch + - apiGroups: ["alpha.projectcalico.org"] + resources: + - systemnetworkpolicies + verbs: + - get + - list diff --git a/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2 b/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2 new file mode 100644 index 000000000..32b822219 --- /dev/null +++ b/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2 @@ -0,0 +1,25 @@ +--- +# Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: flannel +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: diff --git a/roles/network_plugin/canal/templates/canal-crb-calico.yml.j2 b/roles/network_plugin/canal/templates/canal-crb-calico.yml.j2 new file mode 100644 index 000000000..e1c1f5050 --- /dev/null +++ b/roles/network_plugin/canal/templates/canal-crb-calico.yml.j2 @@ -0,0 +1,14 @@ +--- +# Bind the calico ClusterRole to the canal ServiceAccount. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: canal-calico +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico +subjects: +- kind: ServiceAccount + name: canal + namespace: {{ system_namespace }} diff --git a/roles/network_plugin/canal/templates/canal-crb-flannel.yml.j2 b/roles/network_plugin/canal/templates/canal-crb-flannel.yml.j2 new file mode 100644 index 000000000..3b00017b1 --- /dev/null +++ b/roles/network_plugin/canal/templates/canal-crb-flannel.yml.j2 @@ -0,0 +1,14 @@ +--- +# Bind the flannel ClusterRole to the canal ServiceAccount. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: canal-flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel +subjects: +- kind: ServiceAccount + name: canal + namespace: {{ system_namespace }} diff --git a/roles/network_plugin/canal/templates/canal-node-sa.yml.j2 b/roles/network_plugin/canal/templates/canal-node-sa.yml.j2 new file mode 100644 index 000000000..d5b9a6e97 --- /dev/null +++ b/roles/network_plugin/canal/templates/canal-node-sa.yml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: canal + namespace: {{ system_namespace }} + labels: + kubernetes.io/cluster-service: "true" + diff --git a/roles/network_plugin/canal/templates/canal-node.yml.j2 b/roles/network_plugin/canal/templates/canal-node.yml.j2 index cd9312832..16dd64118 100644 --- a/roles/network_plugin/canal/templates/canal-node.yml.j2 +++ b/roles/network_plugin/canal/templates/canal-node.yml.j2 @@ -19,6 +19,7 @@ spec: k8s-app: canal-node spec: hostNetwork: true + serviceAccountName: canal tolerations: - effect: NoSchedule operator: Exists diff --git a/tests/cloud_playbooks/create-gce.yml b/tests/cloud_playbooks/create-gce.yml index 1a82c50d7..a6621a0f9 100644 --- a/tests/cloud_playbooks/create-gce.yml +++ b/tests/cloud_playbooks/create-gce.yml @@ -52,5 +52,5 @@ when: mode in ['scale', 'separate-scale', 'ha-scale'] - name: Wait for SSH to come up - wait_for: host={{item.public_ip}} port=22 delay=30 timeout=180 state=started + wait_for: host={{item.public_ip}} port=22 delay=2 timeout=180 state=started with_items: "{{gce.instance_data}}"