From 6a001e4971289e927de2cdc7bc624302c83f5d09 Mon Sep 17 00:00:00 2001 From: Eugene Artemenko Date: Tue, 4 May 2021 10:05:11 +0300 Subject: [PATCH] Add suport of Vsphere CSI driver 2.X versions (#7480) --- docs/vsphere-csi.md | 2 + inventory/sample/group_vars/all/vsphere.yml | 4 + .../csi_driver/vsphere/defaults/main.yml | 4 + .../csi_driver/vsphere/tasks/main.yml | 7 +- .../vsphere-csi-controller-deployment.yml.j2 | 188 ++++++++++++++ .../vsphere-csi-controller-rbac.yml.j2 | 38 ++- .../vsphere/templates/vsphere-csi-node.yml.j2 | 230 ++++++++++-------- 7 files changed, 368 insertions(+), 105 deletions(-) create mode 100644 roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2 diff --git a/docs/vsphere-csi.md b/docs/vsphere-csi.md index 15f3edf8c..1a7aa58cf 100644 --- a/docs/vsphere-csi.md +++ b/docs/vsphere-csi.md @@ -17,6 +17,7 @@ You need to source the vSphere credentials you use to deploy your machines that | external_vsphere_password | TRUE | string | | | Password for vCenter | | external_vsphere_datacenter | TRUE | string | | | Datacenter name to use | | external_vsphere_kubernetes_cluster_id | TRUE | string | | "kubernetes-cluster-id" | Kubernetes cluster ID to use | +| external_vsphere_version | TRUE | string | | "6.7u3" | Vmware Vsphere version where located all VMs | | vsphere_cloud_controller_image_tag | TRUE | string | | "latest" | Kubernetes cluster ID to use | | vsphere_syncer_image_tag | TRUE | string | | "v1.0.2" | Syncer image tag to use | | vsphere_csi_attacher_image_tag | TRUE | string | | "v1.1.1" | CSI attacher image tag to use | @@ -26,6 +27,7 @@ You need to source the vSphere credentials you use to deploy your machines that | vsphere_csi_provisioner_image_tag | TRUE | string | | "v1.2.2" | CSI provisioner image tag to use | | vsphere_csi_node_driver_registrar_image_tag | TRUE | string | | "v1.1.0" | CSI node driver registrat image tag to use | | vsphere_csi_driver_image_tag | TRUE | string | | "v1.0.2" | CSI driver image tag to use | +vsphere_csi_resizer_tag | TRUE | string | | "v1.0.0" | CSI resizer image tag to use ## Usage example diff --git a/inventory/sample/group_vars/all/vsphere.yml b/inventory/sample/group_vars/all/vsphere.yml index 4ceae9fed..e6033d455 100644 --- a/inventory/sample/group_vars/all/vsphere.yml +++ b/inventory/sample/group_vars/all/vsphere.yml @@ -7,6 +7,9 @@ # external_vsphere_datacenter: "DATACENTER_name" # external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id" +## Vsphere version where located VMs +# external_vsphere_version: "6.7u3" + ## Tags for the external vSphere Cloud Provider images # external_vsphere_cloud_controller_image_tag: "latest" # vsphere_syncer_image_tag: "v1.0.2" @@ -14,6 +17,7 @@ # vsphere_csi_controller: "v1.0.2" # vsphere_csi_liveness_probe_image_tag: "v1.1.0" # vsphere_csi_provisioner_image_tag: "v1.2.2" +# vsphere_csi_resizer_tag: "v1.0.0" ## To use vSphere CSI plugin to provision volumes set this value to true # vsphere_csi_enabled: true diff --git a/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml b/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml index b5ce7f743..cedd2eac9 100644 --- a/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml +++ b/roles/kubernetes-apps/csi_driver/vsphere/defaults/main.yml @@ -2,6 +2,7 @@ external_vsphere_vcenter_port: "443" external_vsphere_insecure: "true" external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id" +external_vsphere_version: "6.7u3" vsphere_syncer_image_tag: "v1.0.2" vsphere_csi_attacher_image_tag: "v1.1.1" @@ -10,5 +11,8 @@ vsphere_csi_liveness_probe_image_tag: "v1.1.0" vsphere_csi_provisioner_image_tag: "v1.2.2" vsphere_csi_node_driver_registrar_image_tag: "v1.1.0" vsphere_csi_driver_image_tag: "v1.0.2" +vsphere_csi_resizer_tag: "v1.0.0" vsphere_csi_controller_replicas: 1 + +csi_endpoint: '{% if external_vsphere_version >= "7.0u1" %}/csi{% else %}/var/lib/csi/sockets/pluginproxy{% endif %}' diff --git a/roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml b/roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml index 26e8751ac..c4e3755ef 100644 --- a/roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml +++ b/roles/kubernetes-apps/csi_driver/vsphere/tasks/main.yml @@ -2,6 +2,11 @@ - include_tasks: vsphere-credentials-check.yml tags: vsphere-csi-driver +- name: vSphere CSI Driver | Choose how to deploy CSI driver based on controller version + set_fact: + controller_spec: "{% if vsphere_csi_controller is version('v2.0.0', '<') %}vsphere-csi-controller-ss.yml{% else %}vsphere-csi-controller-deployment.yml{% endif %}" + tags: vsphere-csi-driver + - name: vSphere CSI Driver | Generate CSI cloud-config template: src: "{{ item }}.j2" @@ -18,7 +23,7 @@ dest: "{{ kube_config_dir }}/{{ item }}" with_items: - vsphere-csi-controller-rbac.yml - - vsphere-csi-controller-ss.yml + - "{{ controller_spec }}" - vsphere-csi-node.yml register: vsphere_csi_manifests when: inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2 b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2 new file mode 100644 index 000000000..83972fb8b --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-deployment.yml.j2 @@ -0,0 +1,188 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: vsphere-csi-controller + namespace: kube-system +spec: + replicas: {{ vsphere_csi_controller_replicas }} + selector: + matchLabels: + app: vsphere-csi-controller + template: + metadata: + labels: + app: vsphere-csi-controller + role: vsphere-csi + spec: + serviceAccountName: vsphere-csi-controller + nodeSelector: +{% if kube_version is version('v1.20.0', '<') %} + node-role.kubernetes.io/master: "" +{% else %} + node-role.kubernetes.io/control-plane: "" +{% endif %} + tolerations: + - operator: "Exists" + key: node-role.kubernetes.io/master + effect: NoSchedule + - operator: "Exists" + key: node-role.kubernetes.io/control-plane + effect: NoSchedule + dnsPolicy: "Default" + containers: + - name: csi-attacher + image: {{ quay_image_repo }}/k8scsi/csi-attacher:{{ vsphere_csi_attacher_image_tag }} + args: + - "--v=4" + - "--timeout=300s" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir +{% if external_vsphere_version >= "7.0" %} + - name: csi-resizer + image: {{ gcr_image_repo }}/k8scsi/csi-resizer:{{ vsphere_csi_resizer_tag }} + args: + - "--v=4" + - "--timeout=300s" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir +{% endif %} + - name: vsphere-csi-controller + image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_controller }} +{% if external_vsphere_version >= "7.0u1" %} + args: + - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" + - "--fss-namespace=$(CSI_NAMESPACE)" +{% endif %} + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: CSI_ENDPOINT + value: unix://{{ csi_endpoint }}/csi.sock + - name: X_CSI_MODE + value: "controller" + - name: VSPHERE_CSI_CONFIG + value: "/etc/cloud/csi-vsphere.conf" + - name: LOGGER_LEVEL + value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION +{% if external_vsphere_version >= "7.0u1" %} + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{% endif %} + - name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT + value: 3m + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + - mountPath: {{ csi_endpoint }} + name: socket-dir + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 5 + failureThreshold: 3 + - name: liveness-probe + image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }} + args: + - "--v=4" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: {{ csi_endpoint }}/csi.sock + volumeMounts: + - name: socket-dir + mountPath: {{ csi_endpoint }} + - name: vsphere-syncer + image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/syncer:{{ vsphere_syncer_image_tag }} + args: + - "--leader-election" +{% if external_vsphere_version >= "7.0u1" %} + - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" + - "--fss-namespace=$(CSI_NAMESPACE)" +{% endif %} + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: FULL_SYNC_INTERVAL_MINUTES + value: "30" + - name: VSPHERE_CSI_CONFIG + value: "/etc/cloud/csi-vsphere.conf" + - name: LOGGER_LEVEL + value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION +{% if external_vsphere_version >= "7.0u1" %} + - name: INCLUSTER_CLIENT_QPS + value: "100" + - name: INCLUSTER_CLIENT_BURST + value: "100" + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{% endif %} + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + - name: csi-provisioner + image: {{ quay_image_repo }}/k8scsi/csi-provisioner:{{ vsphere_csi_provisioner_image_tag }} + args: + - "--v=4" + - "--timeout=300s" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--default-fstype=ext4" + # needed only for topology aware setup + #- "--feature-gates=Topology=true" + #- "--strict-topology" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - name: vsphere-config-volume + secret: + secretName: vsphere-config-secret + - name: socket-dir + emptyDir: {} +--- +apiVersion: v1 +data: + "csi-migration": "false" # csi-migration feature is only available for vSphere 7.0U1 +kind: ConfigMap +metadata: + name: internal-feature-states.csi.vsphere.vmware.com + namespace: kube-system +--- +apiVersion: storage.k8s.io/v1 # For k8s 1.17 or lower use storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: csi.vsphere.vmware.com +spec: + attachRequired: true + podInfoOnMount: false + diff --git a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2 b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2 index a5decf3b2..d0abaf56b 100644 --- a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-rbac.yml.j2 @@ -10,23 +10,47 @@ metadata: name: vsphere-csi-controller-role rules: - apiGroups: [""] - resources: ["nodes", "persistentvolumeclaims", "pods"] + resources: ["nodes", "persistentvolumeclaims", "pods", "configmaps"] verbs: ["get", "list", "watch"] +{% if external_vsphere_version >= "7.0" %} + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] +{% if external_vsphere_version >= "7.0u1" %} + verbs: ["patch"] +{% else %} + verbs: ["update", "patch"] +{% endif %} +{% endif %} - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "update", "delete"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] - apiGroups: [""] resources: ["events"] verbs: ["get", "list", "watch", "create", "update", "patch"] +{% if vsphere_csi_controller is version('v2.0.0', '>=') %} + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +{% endif %} - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] + resources: ["storageclasses","csinodes"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] + verbs: ["get", "list", "watch", "patch"] +{% if external_vsphere_version >= "7.0u1" %} + - apiGroups: ["cns.vmware.com"] + resources: ["cnsvspherevolumemigrations"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "create"] +{% endif %} +{% if vsphere_csi_controller is version('v2.0.0', '>=') %} + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] +{% endif %} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2 b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2 index b9e37697c..779f9e555 100644 --- a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-node.yml.j2 @@ -9,6 +9,8 @@ spec: app: vsphere-csi-node updateStrategy: type: "RollingUpdate" + rollingUpdate: + maxUnavailable: 1 template: metadata: labels: @@ -17,105 +19,139 @@ spec: spec: dnsPolicy: "Default" containers: - - name: node-driver-registrar - image: {{ quay_image_repo }}/k8scsi/csi-node-driver-registrar:{{ vsphere_csi_node_driver_registrar_image_tag }} - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "rm -rf /registration/csi.vsphere.vmware.com /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com-reg.sock"] - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com/csi.sock - securityContext: - privileged: true - volumeMounts: - - name: plugin-dir - mountPath: /csi - - name: registration-dir - mountPath: /registration - - name: vsphere-csi-node - image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_driver_image_tag }} - imagePullPolicy: {{ k8s_image_pull_policy }} - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: unix:///csi/csi.sock - - name: X_CSI_MODE - value: "node" - - name: X_CSI_SPEC_REQ_VALIDATION - value: "false" - # needed only for topology aware setups - #- name: VSPHERE_CSI_CONFIG - # value: "/etc/cloud/csi-vsphere.conf" # here csi-vsphere.conf is the name of the file used for creating secret using "--from-file" flag - args: - - "--v=4" - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - volumeMounts: - # needed only for topology aware setups - #- name: vsphere-config-volume - # mountPath: /etc/cloud - # readOnly: true - - name: plugin-dir - mountPath: /csi - - name: pods-mount-dir - mountPath: /var/lib/kubelet - # needed so that any mounts setup inside this container are - # propagated back to the host machine. - mountPropagation: "Bidirectional" - - name: device-dir - mountPath: /dev - ports: - - name: healthz - containerPort: 9808 - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - timeoutSeconds: 3 - periodSeconds: 5 - failureThreshold: 3 - - name: liveness-probe - image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }} - args: - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: /csi/csi.sock - volumeMounts: - - name: plugin-dir - mountPath: /csi - volumes: + - name: node-driver-registrar + image: {{ quay_image_repo }}/k8scsi/csi-node-driver-registrar:{{ vsphere_csi_node_driver_registrar_image_tag }} +{% if external_vsphere_version < "7.0u1" %} + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock"] +{% endif %} + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" +{% if external_vsphere_version >= "7.0u1" %} + - "--health-port=9809" +{% endif %} + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock + securityContext: + privileged: true + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration +{% if external_vsphere_version >= "7.0u1" %} + ports: + - containerPort: 9809 + name: healthz + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 5 + timeoutSeconds: 5 +{% endif %} + - name: vsphere-csi-node + image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_driver_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} +{% if external_vsphere_version >= "7.0u1" %} + args: + - "--fss-name=internal-feature-states.csi.vsphere.vmware.com" + - "--fss-namespace=$(CSI_NAMESPACE)" +{% endif %} + imagePullPolicy: "Always" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: X_CSI_MODE + value: "node" + - name: X_CSI_SPEC_REQ_VALIDATION + value: "false" + # needed only for topology aware setups + #- name: VSPHERE_CSI_CONFIG + # value: "/etc/cloud/csi-vsphere.conf" # here csi-vsphere.conf is the name of the file used for creating secret using "--from-file" flag + - name: X_CSI_DEBUG + value: "true" + - name: LOGGER_LEVEL + value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION +{% if external_vsphere_version >= "7.0u1" %} + - name: CSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{% endif %} + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + volumeMounts: # needed only for topology aware setups #- name: vsphere-config-volume - # secret: - # secretName: vsphere-config-secret - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry - type: DirectoryOrCreate + # mountPath: /etc/cloud + # readOnly: true - name: plugin-dir - hostPath: - path: /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com - type: DirectoryOrCreate + mountPath: /csi - name: pods-mount-dir - hostPath: - path: /var/lib/kubelet - type: Directory + mountPath: /var/lib/kubelet + # needed so that any mounts setup inside this container are + # propagated back to the host machine. + mountPropagation: "Bidirectional" - name: device-dir - hostPath: - path: /dev + mountPath: /dev + ports: + - containerPort: 9808 + name: healthz + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 5 + periodSeconds: 5 + failureThreshold: 3 + - name: liveness-probe + image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }} + args: +{% if external_vsphere_version >= "7.0u1" %} + - "--v=4" +{% endif %} + - "--csi-address=/csi/csi.sock" + volumeMounts: + - name: plugin-dir + mountPath: /csi + volumes: + # needed only for topology aware setups + #- name: vsphere-config-volume + # secret: + # secretName: vsphere-config-secret + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: device-dir + hostPath: + path: /dev + tolerations: + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists