OCI Cloud Provider Update (#4186)

* OCI subnet AD 2 is not required for CCM >= 0.7.0

Reorganize OCI provider to generate configuration, rather than pull

Add pull secret option to OCI cloud provider

* Updated oci example to document new parameters

inventory/sample/group_vars/all/oci.yml

@@ -23,3 +23,6 @@
   #rate_limit_qps_write:
   #rate_limit_bucket_read:
   #rate_limit_bucket_write:
+# Other optional variables
+#oci_cloud_controller_pull_source: (default iad.ocir.io/oracle/cloud-provider-oci)
+#oci_cloud_controller_pull_secret: (name of pull secret to use if you define your own mirror above)

roles/kubernetes-apps/cloud_controller/oci/defaults/main.yml

@@ -2,4 +2,5 @@
 
 oci_security_list_management: All
 oci_use_instance_principals: false
-oci_cloud_controller_version: 0.6.0
+oci_cloud_controller_version: 0.7.0
+oci_cloud_controller_pull_source: iad.ocir.io/oracle/cloud-provider-oci

roles/kubernetes-apps/cloud_controller/oci/tasks/credentials-check.yml

@@ -48,7 +48,9 @@
 - name: "OCI Cloud Controller | Credentials Check | oci_subnet2_id"
   fail:
     msg: "oci_subnet2_id is missing. Two subnets are required for load balancer high availability"
-  when: oci_subnet2_id is not defined or oci_subnet2_id == ""
+  when:
+  - oci_cloud_controller_version | version_compare('0.7.0', '<')
+  - oci_subnet2_id is not defined or oci_subnet2_id == ""
 
 - name: "OCI Cloud Controller | Credentials Check | oci_security_list_management"
   fail:

roles/kubernetes-apps/cloud_controller/oci/tasks/main.yml

@@ -3,51 +3,35 @@
 - include: credentials-check.yml
   tags: oci
 
-- name: "OCI Cloud Controller | Generate Configuration"
+- name: "OCI Cloud Controller | Generate Cloud Provider Configuration"
   template:
     src: controller-manager-config.yml.j2
-    dest: /tmp/controller-manager-config.yml
+    dest: "{{ kube_config_dir }}/controller-manager-config.yml"
-  register: controller_manager_config
   when: inventory_hostname == groups['kube-master'][0]
   tags: oci
 
+- name: "OCI Cloud Controller | Slurp Configuration"
+  slurp:
+    src: "{{ kube_config_dir }}/controller-manager-config.yml"
+  register: controller_manager_config
+
 - name: "OCI Cloud Controller | Encode Configuration"
   set_fact:
-    controller_manager_config_base64: "{{ lookup('file', '/tmp/controller-manager-config.yml') | b64encode }}"
+    controller_manager_config_base64: "{{ controller_manager_config.content }}"
   when: inventory_hostname == groups['kube-master'][0]
   tags: oci
 
-- name: "OCI Cloud Controller | Apply Configuration To Secret"
+- name: "OCI Cloud Controller | Generate Manifests"
   template:
-    src: cloud-provider.yml.j2
+    src: oci-cloud-provider.yml.j2
-    dest: /tmp/cloud-provider.yml
+    dest: "{{ kube_config_dir }}/oci-cloud-provider.yml"
   when: inventory_hostname == groups['kube-master'][0]
   tags: oci
 
-- name: "OCI Cloud Controller | Apply Configuration"
+- name: "OCI Cloud Controller | Apply Manifests"
   kube:
     kubectl: "{{ bin_dir }}/kubectl"
-    filename: "/tmp/cloud-provider.yml"
+    filename: "{{ kube_config_dir }}/oci-cloud-provider.yml"
-    state: latest
-  when: inventory_hostname == groups['kube-master'][0]
-  tags: oci
-
-- name: "OCI Cloud Controller | Download Controller Manifest"
-  get_url:
-    url: "https://raw.githubusercontent.com/oracle/oci-cloud-controller-manager/{{oci_cloud_controller_version}}/manifests/oci-cloud-controller-manager.yaml"
-    dest: "/tmp/oci-cloud-controller-manager.yml"
-    force: yes
-  register: result
-  until: "'OK' in result.msg"
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  when: inventory_hostname == groups['kube-master'][0]
-  tags: oci
-
-- name: "OCI Cloud Controller | Apply Controller Manifest"
-  kube:
-    kubectl: "{{ bin_dir }}/kubectl"
-    filename: "/tmp/oci-cloud-controller-manager.yml"
     state: latest
   when: inventory_hostname == groups['kube-master'][0]
   tags: oci

roles/kubernetes-apps/cloud_controller/oci/templates/cloud-provider.yml.j2

@@ -1,8 +0,0 @@
-apiVersion: v1
-data:
-  cloud-provider.yaml: {{ controller_manager_config_base64 }}
-kind: Secret
-metadata:
-  name: oci-cloud-controller-manager
-  namespace: kube-system
-type: Opaque

roles/kubernetes-apps/cloud_controller/oci/templates/controller-manager-config.yml.j2

@@ -1,4 +1,4 @@
-auth:
+{% macro private_key() %}{{ oci_private_key }}{% endmacro %}
 
 {% if oci_use_instance_principals %}
   # (https://docs.us-phoenix-1.oraclecloud.com/Content/Identity/Tasks/callingservicesfrominstances.htm).
@@ -6,6 +6,15 @@ auth:
   # allow dynamic-group [your dynamic group name] to read instance-family in compartment [your compartment name]
   # allow dynamic-group [your dynamic group name] to use virtual-network-family in compartment [your compartment name]
   # allow dynamic-group [your dynamic group name] to manage load-balancers in compartment [your compartment name]
+useInstancePrincipals: true
+{% else %}
+useInstancePrincipals: false
+{% endif %}
+
+auth:
+
+{% if oci_use_instance_principals %}
+  # This key is put here too for backwards compatibility
   useInstancePrincipals: true
 {% else %}
   useInstancePrincipals: false
@@ -34,11 +43,11 @@ loadBalancer:
   # subnet1 configures one of two subnets to which load balancers will be added.
   # OCI load balancers require two subnets to ensure high availability.
   subnet1: {{ oci_subnet1_id }}
-
+{% if oci_subnet2_id is defined %}
   # subnet2 configures the second of two subnets to which load balancers will be
   # added. OCI load balancers require two subnets to ensure high availability.
   subnet2: {{ oci_subnet2_id }}
-
+{% endif %}
   # SecurityListManagementMode configures how security lists are managed by the CCM.
   #   "All" (default): Manage all required security list rules for load balancer services.
   #   "Frontend":      Manage only security list rules for ingress to the load

roles/kubernetes-apps/cloud_controller/oci/templates/oci-cloud-provider.yml.j2

@@ -0,0 +1,69 @@
+apiVersion: v1
+data:
+  cloud-provider.yaml: {{ controller_manager_config_base64 }}
+kind: Secret
+metadata:
+  name: oci-cloud-controller-manager
+  namespace: kube-system
+type: Opaque
+
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: oci-cloud-controller-manager
+  namespace: kube-system
+  labels:
+    k8s-app: oci-cloud-controller-manager
+spec:
+  selector:
+    matchLabels:
+      component: oci-cloud-controller-manager
+      tier: control-plane
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        component: oci-cloud-controller-manager
+        tier: control-plane
+    spec:
+{% if oci_cloud_controller_pull_secret is defined %}
+      imagePullSecrets:
+      - name: {{oci_cloud_controller_pull_secret}}
+{% endif %}
+      serviceAccountName: cloud-controller-manager
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      volumes:
+        - name: cfg
+          secret:
+            secretName: oci-cloud-controller-manager
+        - name: kubernetes
+          hostPath:
+            path: /etc/kubernetes
+      containers:
+        - name: oci-cloud-controller-manager
+          image: {{oci_cloud_controller_pull_source}}:{{oci_cloud_controller_version}}
+          command: ["/usr/local/bin/oci-cloud-controller-manager"]
+          args:
+            - --cloud-config=/etc/oci/cloud-provider.yaml
+            - --cloud-provider=oci
+            - --leader-elect-resource-lock=configmaps
+            - -v=2
+          volumeMounts:
+            - name: cfg
+              mountPath: /etc/oci
+              readOnly: true
+            - name: kubernetes
+              mountPath: /etc/kubernetes
+              readOnly: true
+

roles/kubernetes-apps/cluster_roles/defaults/main.yml

@@ -1,2 +1 @@
 ---
-oci_cloud_controller_version: 0.5.0

roles/kubernetes-apps/cluster_roles/files/oci-rbac.yml

@@ -0,0 +1,126 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: system:cloud-controller-manager
+  labels:
+    kubernetes.io/cluster-service: "true"
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - watch
+  - patch
+
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - update
+
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+
+# For leader election
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  resourceNames:
+  - "cloud-controller-manager"
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  resourceNames:
+  - "cloud-controller-manager"
+  verbs:
+  - get
+  - update
+
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+
+# For the PVL
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - list
+  - watch
+  - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: oci-cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system

roles/kubernetes-apps/cluster_roles/tasks/oci.yml

@@ -1,23 +1,18 @@
 ---
-- name: Get OCI ClusterRole, and ClusterRoleBinding
+- name: Copy OCI RBAC Manifest
-  get_url:
+  copy:
-    url: "https://raw.githubusercontent.com/oracle/oci-cloud-controller-manager/{{oci_cloud_controller_version}}/manifests/oci-cloud-controller-manager-rbac.yaml"
+    src: "oci-rbac.yml"
-    dest: "/tmp/oci-cloud-controller-manager-rbac.yaml"
+    dest: "{{ kube_config_dir }}/oci-rbac.yml"
-    force: yes
-  register: result
-  until: "'OK' in result.msg"
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
   when:
-    - cloud_provider is defined
+  - cloud_provider is defined
-    - cloud_provider == 'oci'
+  - cloud_provider == 'oci'
-    - inventory_hostname == groups['kube-master'][0]
+  - inventory_hostname == groups['kube-master'][0]
 
-- name: Apply OCI ClusterRole, and ClusterRoleBinding
+- name: Apply OCI RBAC
   kube:
     kubectl: "{{bin_dir}}/kubectl"
-    filename: "/tmp/oci-cloud-controller-manager-rbac.yaml"
+    filename: "{{ kube_config_dir }}/oci-rbac.yml"
   when:
-    - cloud_provider is defined
+  - cloud_provider is defined
-    - cloud_provider == 'oci'
+  - cloud_provider == 'oci'
-    - inventory_hostname == groups['kube-master'][0]
+  - inventory_hostname == groups['kube-master'][0]