Add RBAC support for canal

Refactored how rbac_enabled is set
Added RBAC to ubuntu-canal-ha CI job

.gitlab-ci.yml

@@ -269,9 +269,10 @@ before_script:
   ##User-data to simply turn off coreos upgrades
   STARTUP_SCRIPT: 'systemctl disable locksmithd && systemctl stop locksmithd'
 
-.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
+.ubuntu_canal_ha_rbac_variables: &ubuntu_canal_ha_rbac_variables
 # stage: deploy-gce-part1
   KUBE_NETWORK_PLUGIN: canal
+  AUTHORIZATION_MODES: "{ 'authorization_modes':  [ 'RBAC' ] }"
   CLOUD_IMAGE: ubuntu-1604-xenial
   CLOUD_REGION: europe-west1-b
   CLUSTER_MODE: ha
@@ -445,24 +446,24 @@ ubuntu-weave-sep-triggers:
   only: ['triggers']
 
 # More builds for PRs/merges (manual) and triggers (auto)
-ubuntu-canal-ha:
+ubuntu-canal-ha-rbac:
   stage: deploy-gce-part1
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *ubuntu_canal_ha_variables
+    <<: *ubuntu_canal_ha_rbac_variables
   when: manual
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
-ubuntu-canal-ha-triggers:
+ubuntu-canal-ha-rbac-triggers:
   stage: deploy-gce-part1
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *ubuntu_canal_ha_variables
+    <<: *ubuntu_canal_ha_rbac_variables
   when: on_success
   only: ['triggers']
 

roles/kubernetes-apps/network_plugin/canal/tasks/main.yml

@@ -1,20 +1,11 @@
 ---
-- name: Create canal ConfigMap
+- name: Canal | Start Resources
-  run_once: true
   kube:
-    name: "canal-config"
+    name: "{{item.item.name}}"
-    kubectl: "{{bin_dir}}/kubectl"
-    filename: "{{kube_config_dir}}/canal-config.yaml"
-    resource: "configmap"
     namespace: "{{ system_namespace }}"
-
-- name: Start flannel and calico-node
-  run_once: true
-  kube:
-    name: "canal-node"
     kubectl: "{{bin_dir}}/kubectl"
-    filename: "{{kube_config_dir}}/canal-node.yaml"
+    resource: "{{item.item.type}}"
-    resource: "ds"
+    filename: "{{kube_config_dir}}/{{item.item.file}}"
-    namespace: "{{system_namespace}}"
+    state: "{{item.changed | ternary('latest','present') }}"
-    state: "{{ item | ternary('latest','present') }}"
+  with_items: "{{ canal_manifests.results }}"
-  with_items: "{{ canal_node_manifest.changed }}"
+  failed_when: canal_manifests|failed and "Error from server (AlreadyExists)" not in canal_manifests.msg

roles/network_plugin/canal/defaults/main.yml

@@ -31,3 +31,8 @@ calicoctl_memory_limit: 170M
 calicoctl_cpu_limit: 100m
 calicoctl_memory_requests: 32M
 calicoctl_cpu_requests: 25m
+
+rbac_resources:
+  - sa
+  - clusterrole
+  - clusterrolebinding

roles/network_plugin/canal/tasks/main.yml

@@ -32,16 +32,21 @@
   delegate_to: "{{groups['etcd'][0]}}"
   run_once: true
 
-- name: Canal | Write canal configmap
+- name: Canal | Create canal node rbac configuration
   template:
-    src: canal-config.yml.j2
+    src: "{{item.file}}.j2"
-    dest: "{{kube_config_dir}}/canal-config.yaml"
+    dest: "{{kube_config_dir}}/{{item.file}}"
-
+  with_items:
-- name: Canal | Write canal node configuration
+    - {name: canal-config, file: canal-config.yml, type: cm}
-  template:
+    - {name: canal-node, file: canal-node.yml, type: ds}
-    src: canal-node.yml.j2
+    - {name: canal, file: canal-node-sa.yml, type: sa}
-    dest: "{{kube_config_dir}}/canal-node.yaml"
+    - {name: calico, file: canal-cr-calico.yml, type: clusterrole}
-  register: canal_node_manifest
+    - {name: flannel, file: canal-cr-flannel.yml, type: clusterrole}
+    - {name: canal-calico, file: canal-cr-calico.yml, type: clusterrolebinding}
+    - {name: canal-flannel, file: canal-cr-flannel.yml, type: clusterrolebinding}
+  register: canal_manifests
+  when:
+    - rbac_enabled or item.type not in rbac_resources
 
 - name: Canal | Copy cni plugins from hyperkube
   command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -ac /opt/cni/bin/ /cnibindir/"

roles/network_plugin/canal/templates/canal-cr-calico.yml.j2

@@ -0,0 +1,80 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico
+  namespace: {{ system_namespace }}
+rules:
+  - apiGroups: [""]
+    resources:
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: [""]
+    resources:
+      - pods/status
+    verbs:
+      - update
+  - apiGroups: [""]
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: [""]
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups: ["extensions"]
+    resources:
+      - thirdpartyresources
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+  - apiGroups: ["extensions"]
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: ["projectcalico.org"]
+    resources:
+      - globalbgppeers
+    verbs:
+      - get
+      - list
+  - apiGroups: ["projectcalico.org"]
+    resources:
+      - globalconfigs
+      - globalbgpconfigs
+    verbs:
+      - create
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups: ["projectcalico.org"]
+    resources:
+      - ippools
+    verbs:
+      - create
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups: ["alpha.projectcalico.org"]
+    resources:
+      - systemnetworkpolicies
+    verbs:
+      - get
+      - list

roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2

@@ -0,0 +1,25 @@
+---
+# Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: flannel
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:

roles/network_plugin/canal/templates/canal-crb-calico.yml.j2

@@ -0,0 +1,14 @@
+---
+# Bind the calico ClusterRole to the canal ServiceAccount.
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: canal-calico
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico
+subjects:
+- kind: ServiceAccount
+  name: canal
+  namespace: {{ system_namespace }}

roles/network_plugin/canal/templates/canal-crb-flannel.yml.j2

@@ -0,0 +1,14 @@
+---
+# Bind the flannel ClusterRole to the canal ServiceAccount.
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: canal-flannel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flannel
+subjects:
+- kind: ServiceAccount
+  name: canal
+  namespace: {{ system_namespace }}

roles/network_plugin/canal/templates/canal-node-sa.yml.j2

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: canal
+  namespace: {{ system_namespace }}
+  labels:
+    kubernetes.io/cluster-service: "true"
+

roles/network_plugin/canal/templates/canal-node.yml.j2

@@ -19,6 +19,7 @@ spec:
         k8s-app: canal-node
     spec:
       hostNetwork: true
+      serviceAccountName: canal
       tolerations:
         - effect: NoSchedule
           operator: Exists

tests/cloud_playbooks/create-gce.yml

@@ -52,5 +52,5 @@
       when: mode in ['scale', 'separate-scale', 'ha-scale']
 
     - name: Wait for SSH to come up
-      wait_for: host={{item.public_ip}} port=22 delay=30 timeout=180 state=started
+      wait_for: host={{item.public_ip}} port=22 delay=2 timeout=180 state=started
       with_items: "{{gce.instance_data}}"