From b0ee1f6cc6a59b59a570030b472e7effa3e7c35f Mon Sep 17 00:00:00 2001 From: Ali Sanhaji Date: Fri, 1 Nov 2019 08:59:24 +0100 Subject: [PATCH] Deploy Cinder CSI driver to provision volumes over OpenStack (#5184) * Deploy Cinder CSI driver to provision volumes over OpenStack * Deploy Cinder CSI StorageClass * Cinder CSI doc --- docs/cinder-csi.md | 99 +++++++++ inventory/sample/group_vars/all/openstack.yml | 5 + .../csi_driver/cinder/defaults/main.yml | 16 ++ .../cinder/tasks/cinder-credential-check.yml | 34 +++ .../csi_driver/cinder/tasks/main.yml | 60 +++++ .../cinder-csi-cloud-config-secret.yml.j2 | 10 + .../templates/cinder-csi-cloud-config.j2 | 26 +++ .../cinder-csi-controllerplugin-rbac.yml.j2 | 209 ++++++++++++++++++ .../cinder-csi-controllerplugin.yml.j2 | 109 +++++++++ .../cinder/templates/cinder-csi-driver.yml.j2 | 7 + .../cinder-csi-nodeplugin-rbac.yml.j2 | 30 +++ .../templates/cinder-csi-nodeplugin.yml.j2 | 116 ++++++++++ roles/kubernetes-apps/meta/main.yml | 7 + .../cinder-csi/defaults/main.yml | 6 + .../cinder-csi/tasks/main.yml | 19 ++ .../templates/cinder-csi-storage-class.yml.j2 | 14 ++ .../persistent_volumes/meta/main.yml | 7 + roles/kubespray-defaults/defaults/main.yaml | 1 + 18 files changed, 775 insertions(+) create mode 100644 docs/cinder-csi.md create mode 100644 roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml create mode 100644 roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-credential-check.yml create mode 100644 roles/kubernetes-apps/csi_driver/cinder/tasks/main.yml create mode 100644 roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config-secret.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config.j2 create mode 100644 roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-driver.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 create mode 100644 roles/kubernetes-apps/persistent_volumes/cinder-csi/defaults/main.yml create mode 100644 roles/kubernetes-apps/persistent_volumes/cinder-csi/tasks/main.yml create mode 100644 roles/kubernetes-apps/persistent_volumes/cinder-csi/templates/cinder-csi-storage-class.yml.j2 diff --git a/docs/cinder-csi.md b/docs/cinder-csi.md new file mode 100644 index 000000000..11988fde9 --- /dev/null +++ b/docs/cinder-csi.md @@ -0,0 +1,99 @@ +Cinder CSI Driver +=============== + +Cinder CSI driver allows you to provision volumes over an OpenStack deployment. The Kubernetes historic in-tree cloud provider is deprecated and will be removed in future versions. + +To enable Cinder CSI driver, uncomment the `cinder_csi_enabled` option in `group_vars/all/openstack.yml` and set it to `true`. + +To set the number of replicas for the Cinder CSI controller, you can change `cinder_csi_controller_replicas` option in `group_vars/all/openstack.yml`. + +You need to source the OpenStack credentials you use to deploy your machines that will host Kubernetes: `source path/to/your/openstack-rc` or `. path/to/your/openstack-rc`. + +Make sure the hostnames in your `inventory` file are identical to your instance names in OpenStack. Otherwise [cinder](https://docs.openstack.org/cinder/latest/) won't work as expected. + +If you want to deploy the cinder provisioner used with Cinder CSI Driver, you should set `persistent_volumes_enabled` in `group_vars/k8s-cluster/k8s-cluster.yml` to `true`. + +You can now run the kubespray playbook (cluster.yml) to deploy Kubernetes over OpenStack with Cinder CSI Driver enabled. + +## Usage example ## + +To check if Cinder CSI Driver works properly, see first that the cinder-csi pods are running: + +``` +$ kubectl -n kube-system get pods | grep cinder +csi-cinder-controllerplugin-7f8bf99785-cpb5v 5/5 Running 0 100m +csi-cinder-nodeplugin-rm5x2 2/2 Running 0 100m +``` + +Check the associated storage class (if you enabled persistent_volumes): + +``` +$ kubectl get storageclass +NAME PROVISIONER AGE +cinder-csi cinder.csi.openstack.org 100m +``` + +You can run a PVC and an Nginx Pod using this file `nginx.yaml`: + +``` +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: csi-pvc-cinderplugin +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: cinder-csi + +--- +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - image: nginx + imagePullPolicy: IfNotPresent + name: nginx + ports: + - containerPort: 80 + protocol: TCP + volumeMounts: + - mountPath: /var/lib/www/html + name: csi-data-cinderplugin + volumes: + - name: csi-data-cinderplugin + persistentVolumeClaim: + claimName: csi-pvc-cinderplugin + readOnly: false +``` + +Apply this conf to your cluster: ```kubectl apply -f nginx.yml``` + +You should see the PVC provisioned and bound: +``` +$ kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +csi-pvc-cinderplugin Bound pvc-f21ad0a1-5b7b-405e-a462-48da5cb76beb 1Gi RWO cinder-csi 8s +``` + +And the volume mounted to the Nginx Pod (wait until the Pod is Running): + +``` +kubectl exec -it nginx -- df -h | grep /var/lib/www/html +/dev/vdb 976M 2.6M 958M 1% /var/lib/www/html +``` + +## Compatibility with in-tree cloud provider ## +It is not necessary to enable OpenStack as a cloud provider for Cinder CSI Driver to work. +Though, you can run both the in-tree openstack cloud provider and the Cinder CSI Driver at the same time. The storage class provisioners associated to each one of them are differently named. + +## Cinder v2 support ## +For the moment, only Cinder v3 is supported by the CSI Driver. + +## More info ## +For further information about the Cinder CSI Driver, you can refer to this page: [Cloud Provider OpenStack](https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/using-cinder-csi-plugin.md). diff --git a/inventory/sample/group_vars/all/openstack.yml b/inventory/sample/group_vars/all/openstack.yml index 46db02c67..8c39396aa 100644 --- a/inventory/sample/group_vars/all/openstack.yml +++ b/inventory/sample/group_vars/all/openstack.yml @@ -14,3 +14,8 @@ # openstack_lbaas_monitor_delay: "1m" # openstack_lbaas_monitor_timeout: "30s" # openstack_lbaas_monitor_max_retries: "3" + +## To use Cinder CSI plugin to provision volumes set this value to true +## Make sure to source in the openstack credentials +# cinder_csi_enabled: true +# cinder_csi_controller_replicas: 1 diff --git a/roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml b/roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml new file mode 100644 index 000000000..c703eae60 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/cinder/defaults/main.yml @@ -0,0 +1,16 @@ +--- +# To access Cinder, the CSI controller will need credentials to access +# openstack apis. Per default this values will be +# read from the environment. +cinder_auth_url: "{{ lookup('env','OS_AUTH_URL') }}" +cinder_username: "{{ lookup('env','OS_USERNAME') }}" +cinder_password: "{{ lookup('env','OS_PASSWORD') }}" +cinder_region: "{{ lookup('env','OS_REGION_NAME') }}" +cinder_tenant_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID')|default(lookup('env','OS_PROJECT_NAME'),true),true) }}" +cinder_tenant_name: "{{ lookup('env','OS_TENANT_NAME') }}" +cinder_domain_name: "{{ lookup('env','OS_USER_DOMAIN_NAME') }}" +cinder_domain_id: "{{ lookup('env','OS_USER_DOMAIN_ID') }}" +cinder_cacert: "{{ lookup('env','OS_CACERT') }}" +# For now, only Cinder v3 is supported in Cinder CSI driver +cinder_blockstorage_version: "v3" +cinder_csi_controller_replicas: 1 \ No newline at end of file diff --git a/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-credential-check.yml b/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-credential-check.yml new file mode 100644 index 000000000..8e24c9a02 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/cinder/tasks/cinder-credential-check.yml @@ -0,0 +1,34 @@ +--- +- name: Cinder CSI Driver | check cinder_auth_url value + fail: + msg: "cinder_auth_url is missing" + when: cinder_auth_url is not defined or not cinder_auth_url + +- name: Cinder CSI Driver | check cinder_username value + fail: + msg: "cinder_username is missing" + when: cinder_username is not defined or not cinder_username + +- name: Cinder CSI Driver | check cinder_password value + fail: + msg: "cinder_password is missing" + when: cinder_password is not defined or not cinder_password + +- name: Cinder CSI Driver | check cinder_region value + fail: + msg: "cinder_region is missing" + when: cinder_region is not defined or not cinder_region + +- name: Cinder CSI Driver | check cinder_tenant_id value + fail: + msg: "one of cinder_tenant_id or cinder_trust_id must be specified" + when: + - cinder_tenant_id is not defined or not cinder_tenant_id + - cinder_trust_id is not defined + +- name: Cinder CSI Driver | check cinder_trust_id value + fail: + msg: "one of cinder_tenant_id or cinder_trust_id must be specified" + when: + - cinder_trust_id is not defined or not cinder_trust_id + - cinder_tenant_id is not defined diff --git a/roles/kubernetes-apps/csi_driver/cinder/tasks/main.yml b/roles/kubernetes-apps/csi_driver/cinder/tasks/main.yml new file mode 100644 index 000000000..aa5a1c6e8 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/cinder/tasks/main.yml @@ -0,0 +1,60 @@ +--- +- include_tasks: cinder-credential-check.yml + tags: cinder-csi-driver + +- name: Cinder CSI Driver | Write cacert file + copy: + src: "{{ cinder_cacert }}" + dest: "{{ kube_config_dir }}/cinder-cacert.pem" + group: "{{ kube_cert_group }}" + mode: 0640 + when: + - inventory_hostname in groups['k8s-cluster'] + - cinder_cacert is defined + - cinder_cacert | length > 0 + tags: cinder-csi-driver + +- name: Cinder CSI Driver | Write Cinder cloud-config + template: + src: "cinder-csi-cloud-config.j2" + dest: "{{ kube_config_dir }}/cinder_cloud_config" + group: "{{ kube_cert_group }}" + mode: 0640 + when: inventory_hostname == groups['kube-master'][0] + tags: cinder-csi-driver + +- name: Cinder CSI Driver | Get base64 cloud-config + slurp: + src: "{{ kube_config_dir }}/cinder_cloud_config" + register: cloud_config_secret + when: inventory_hostname == groups['kube-master'][0] + tags: cinder-csi-driver + +- name: Cinder CSI Driver | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - {name: cinder-csi-driver, file: cinder-csi-driver.yml} + - {name: cinder-csi-cloud-config-secret, file: cinder-csi-cloud-config-secret.yml} + - {name: cinder-csi-controllerplugin, file: cinder-csi-controllerplugin-rbac.yml} + - {name: cinder-csi-controllerplugin, file: cinder-csi-controllerplugin.yml} + - {name: cinder-csi-nodeplugin, file: cinder-csi-nodeplugin-rbac.yml} + - {name: cinder-csi-nodeplugin, file: cinder-csi-nodeplugin.yml} + register: cinder_csi_manifests + when: inventory_hostname == groups['kube-master'][0] + tags: cinder-csi-driver + +- name: Cinder CSI Driver | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ cinder_csi_manifests.results }}" + when: + - inventory_hostname == groups['kube-master'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + tags: cinder-csi-driver diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config-secret.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config-secret.yml.j2 new file mode 100644 index 000000000..cb3cba6ac --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config-secret.yml.j2 @@ -0,0 +1,10 @@ +# This YAML file contains secret objects, +# which are necessary to run csi cinder plugin. + +kind: Secret +apiVersion: v1 +metadata: + name: cloud-config + namespace: kube-system +data: + cloud.conf: {{ cloud_config_secret.content }} diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config.j2 new file mode 100644 index 000000000..c68a1085f --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-cloud-config.j2 @@ -0,0 +1,26 @@ +[Global] +auth-url="{{ cinder_auth_url }}" +username="{{ cinder_username }}" +password="{{ cinder_password }}" +region="{{ cinder_region }}" +{% if cinder_trust_id is defined and cinder_trust_id != "" %} +trust-id="{{ cinder_trust_id }}" +{% else %} +tenant-id="{{ cinder_tenant_id }}" +{% endif %} +{% if cinder_tenant_name is defined and cinder_tenant_name != "" %} +tenant-name="{{ cinder_tenant_name }}" +{% endif %} +{% if cinder_domain_name is defined and cinder_domain_name != "" %} +domain-name="{{ cinder_domain_name }}" +{% elif cinder_domain_id is defined and cinder_domain_id != "" %} +domain-id ="{{ cinder_domain_id }}" +{% endif %} +{% if cinder_cacert is defined and cinder_cacert != "" %} +ca-file="{{ kube_config_dir }}/cinder-cacert.pem" +{% endif %} + +[BlockStorage] +{% if cinder_blockstorage_version is defined %} +bs-version={{ cinder_blockstorage_version }} +{% endif %} diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 new file mode 100644 index 000000000..241c67af3 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin-rbac.yml.j2 @@ -0,0 +1,209 @@ +# This YAML file contains RBAC API objects, +# which are necessary to run csi controller plugin + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cinder-controller-sa + namespace: kube-system + +--- +# external attacher +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- +# external Provisioner +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- +# external snapshotter +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshotter-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- + +# External Resizer +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-resizer-role +rules: + # The following rule should be uncommented for plugins that require secrets + # for provisioning. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-resizer-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-resizer-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: kube-system + name: external-resizer-cfg +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-resizer-role-cfg + namespace: kube-system +subjects: + - kind: ServiceAccount + name: csi-cinder-controller-sa + namespace: kube-system +roleRef: + kind: Role + name: external-resizer-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 new file mode 100644 index 000000000..58c448165 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 @@ -0,0 +1,109 @@ +# This YAML file contains CSI Controller Plugin Sidecars +# external-attacher, external-provisioner, external-snapshotter + +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-cinder-controllerplugin + namespace: kube-system +spec: + replicas: {{ cinder_csi_controller_replicas }} + selector: + matchLabels: + app: csi-cinder-controllerplugin + template: + metadata: + labels: + app: csi-cinder-controllerplugin + spec: + serviceAccount: csi-cinder-controller-sa + containers: + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v1.2.1 + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.3.0 + args: + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-snapshotter + image: quay.io/k8scsi/csi-snapshotter:v1.2.0 + args: + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: Always + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:v0.2.0 + args: + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: cinder-csi-plugin + image: docker.io/k8scloudprovider/cinder-csi-plugin:latest + args : + - /bin/cinder-csi-plugin + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--cloud-config=$(CLOUD_CONFIG)" + - "--cluster=$(CLUSTER_NAME)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://csi/csi.sock + - name: CLOUD_CONFIG + value: /etc/config/cloud.conf + - name: CLUSTER_NAME + value: kubernetes + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: secret-cinderplugin + mountPath: /etc/config + readOnly: true +{% if cinder_cacert is defined %} + - name: cinder-cacert + mountPath: {{ kube_config_dir }}/cinder-cacert.pem + readOnly: true +{% endif %} + volumes: + - name: socket-dir + emptyDir: + - name: secret-cinderplugin + secret: + secretName: cloud-config +{% if cinder_cacert is defined %} + - name: cinder-cacert + hostPath: + path: {{ kube_config_dir }}/cinder-cacert.pem + type: FileOrCreate +{% endif %} diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-driver.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-driver.yml.j2 new file mode 100644 index 000000000..671e2ec13 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-driver.yml.j2 @@ -0,0 +1,7 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: cinder.csi.openstack.org +spec: + attachRequired: true + podInfoOnMount: false diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 new file mode 100644 index 000000000..912923fca --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin-rbac.yml.j2 @@ -0,0 +1,30 @@ +# This YAML defines all API objects to create RBAC roles for csi node plugin. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-cinder-node-sa + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin-binding +subjects: + - kind: ServiceAccount + name: csi-cinder-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-nodeplugin-role + apiGroup: rbac.authorization.k8s.io diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 new file mode 100644 index 000000000..e2f1afa7c --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 @@ -0,0 +1,116 @@ +# This YAML file contains driver-registrar & csi driver nodeplugin API objects, +# which are necessary to run csi nodeplugin for cinder. + +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-cinder-nodeplugin + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-cinder-nodeplugin + template: + metadata: + labels: + app: csi-cinder-nodeplugin + spec: + serviceAccount: csi-cinder-node-sa + hostNetwork: true + containers: + - name: node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.1.0 + args: + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/cinder.csi.openstack.org /registration/cinder.csi.openstack.org-reg.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/cinder.csi.openstack.org/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: cinder-csi-plugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: docker.io/k8scloudprovider/cinder-csi-plugin:latest + args : + - /bin/cinder-csi-plugin + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--cloud-config=$(CLOUD_CONFIG)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://csi/csi.sock + - name: CLOUD_CONFIG + value: /etc/config/cloud.conf + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: pods-cloud-data + mountPath: /var/lib/cloud/data + readOnly: true + - name: pods-probe-dir + mountPath: /dev + mountPropagation: "HostToContainer" + - name: secret-cinderplugin + mountPath: /etc/config + readOnly: true +{% if cinder_cacert is defined %} + - name: cinder-cacert + mountPath: {{ kube_config_dir }}/cinder-cacert.pem + readOnly: true +{% endif %} + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/cinder.csi.openstack.org + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: pods-cloud-data + hostPath: + path: /var/lib/cloud/data + type: Directory + - name: pods-probe-dir + hostPath: + path: /dev + type: Directory + - name: secret-cinderplugin + secret: + secretName: cloud-config +{% if cinder_cacert is defined %} + - name: cinder-cacert + hostPath: + path: {{ kube_config_dir }}/cinder-cacert.pem + type: FileOrCreate +{% endif %} diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index 13cf0af89..fd53ec03f 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -29,6 +29,13 @@ dependencies: - apps - metrics_server + - role: kubernetes-apps/csi_driver/cinder + when: + - cinder_csi_enabled + tags: + - apps + - cinder-csi-driver + - role: kubernetes-apps/persistent_volumes when: - persistent_volumes_enabled diff --git a/roles/kubernetes-apps/persistent_volumes/cinder-csi/defaults/main.yml b/roles/kubernetes-apps/persistent_volumes/cinder-csi/defaults/main.yml new file mode 100644 index 000000000..c00d8d6eb --- /dev/null +++ b/roles/kubernetes-apps/persistent_volumes/cinder-csi/defaults/main.yml @@ -0,0 +1,6 @@ +--- +storage_classes: + - name: cinder-csi + is_default: false + parameters: + availability: nova diff --git a/roles/kubernetes-apps/persistent_volumes/cinder-csi/tasks/main.yml b/roles/kubernetes-apps/persistent_volumes/cinder-csi/tasks/main.yml new file mode 100644 index 000000000..f94f8ca3e --- /dev/null +++ b/roles/kubernetes-apps/persistent_volumes/cinder-csi/tasks/main.yml @@ -0,0 +1,19 @@ +--- +- name: Kubernetes Persistent Volumes | Copy Cinder CSI Storage Class template + template: + src: "cinder-csi-storage-class.yml.j2" + dest: "{{ kube_config_dir }}/cinder-csi-storage-class.yml" + register: manifests + when: + - inventory_hostname == groups['kube-master'][0] + +- name: Kubernetes Persistent Volumes | Add Cinder CSI Storage Class + kube: + name: cinder-csi + kubectl: "{{ bin_dir }}/kubectl" + resource: StorageClass + filename: "{{ kube_config_dir }}/cinder-csi-storage-class.yml" + state: "latest" + when: + - inventory_hostname == groups['kube-master'][0] + - manifests.changed diff --git a/roles/kubernetes-apps/persistent_volumes/cinder-csi/templates/cinder-csi-storage-class.yml.j2 b/roles/kubernetes-apps/persistent_volumes/cinder-csi/templates/cinder-csi-storage-class.yml.j2 new file mode 100644 index 000000000..a13d94ff3 --- /dev/null +++ b/roles/kubernetes-apps/persistent_volumes/cinder-csi/templates/cinder-csi-storage-class.yml.j2 @@ -0,0 +1,14 @@ +{% for class in storage_classes %} +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: "{{ class.name }}" + annotations: + storageclass.kubernetes.io/is-default-class: "{{ class.is_default | default(false) }}" +provisioner: cinder.csi.openstack.org +parameters: +{% for key, value in (class.parameters | default({})).items() %} + "{{ key }}": "{{ value }}" +{% endfor %} +{% endfor %} diff --git a/roles/kubernetes-apps/persistent_volumes/meta/main.yml b/roles/kubernetes-apps/persistent_volumes/meta/main.yml index 8a4855bff..bc5f14fab 100644 --- a/roles/kubernetes-apps/persistent_volumes/meta/main.yml +++ b/roles/kubernetes-apps/persistent_volumes/meta/main.yml @@ -6,3 +6,10 @@ dependencies: - cloud_provider in [ 'openstack' ] tags: - persistent_volumes_openstack + + - role: kubernetes-apps/persistent_volumes/cinder-csi + when: + - cinder_csi_enabled + tags: + - persistent_volumes_cinder_csi + - cinder-csi-driver diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 27d2ed7e7..3072d7fbb 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -300,6 +300,7 @@ metrics_server_enabled: false enable_network_policy: true local_volume_provisioner_enabled: "{{ local_volumes_enabled | default('false') }}" local_volume_provisioner_directory_mode: 0700 +cinder_csi_enabled: false persistent_volumes_enabled: false cephfs_provisioner_enabled: false rbd_provisioner_enabled: false