Add suport of Vsphere CSI driver 2.X versions (#7480)

This commit is contained in:
Eugene Artemenko 2021-05-04 10:05:11 +03:00 committed by GitHub
parent 96e6a6ac3f
commit 6a001e4971
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 368 additions and 105 deletions

View file

@ -17,6 +17,7 @@ You need to source the vSphere credentials you use to deploy your machines that
| external_vsphere_password | TRUE | string | | | Password for vCenter | | external_vsphere_password | TRUE | string | | | Password for vCenter |
| external_vsphere_datacenter | TRUE | string | | | Datacenter name to use | | external_vsphere_datacenter | TRUE | string | | | Datacenter name to use |
| external_vsphere_kubernetes_cluster_id | TRUE | string | | "kubernetes-cluster-id" | Kubernetes cluster ID to use | | external_vsphere_kubernetes_cluster_id | TRUE | string | | "kubernetes-cluster-id" | Kubernetes cluster ID to use |
| external_vsphere_version | TRUE | string | | "6.7u3" | Vmware Vsphere version where located all VMs |
| vsphere_cloud_controller_image_tag | TRUE | string | | "latest" | Kubernetes cluster ID to use | | vsphere_cloud_controller_image_tag | TRUE | string | | "latest" | Kubernetes cluster ID to use |
| vsphere_syncer_image_tag | TRUE | string | | "v1.0.2" | Syncer image tag to use | | vsphere_syncer_image_tag | TRUE | string | | "v1.0.2" | Syncer image tag to use |
| vsphere_csi_attacher_image_tag | TRUE | string | | "v1.1.1" | CSI attacher image tag to use | | vsphere_csi_attacher_image_tag | TRUE | string | | "v1.1.1" | CSI attacher image tag to use |
@ -26,6 +27,7 @@ You need to source the vSphere credentials you use to deploy your machines that
| vsphere_csi_provisioner_image_tag | TRUE | string | | "v1.2.2" | CSI provisioner image tag to use | | vsphere_csi_provisioner_image_tag | TRUE | string | | "v1.2.2" | CSI provisioner image tag to use |
| vsphere_csi_node_driver_registrar_image_tag | TRUE | string | | "v1.1.0" | CSI node driver registrat image tag to use | | vsphere_csi_node_driver_registrar_image_tag | TRUE | string | | "v1.1.0" | CSI node driver registrat image tag to use |
| vsphere_csi_driver_image_tag | TRUE | string | | "v1.0.2" | CSI driver image tag to use | | vsphere_csi_driver_image_tag | TRUE | string | | "v1.0.2" | CSI driver image tag to use |
vsphere_csi_resizer_tag | TRUE | string | | "v1.0.0" | CSI resizer image tag to use
## Usage example ## Usage example

View file

@ -7,6 +7,9 @@
# external_vsphere_datacenter: "DATACENTER_name" # external_vsphere_datacenter: "DATACENTER_name"
# external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id" # external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id"
## Vsphere version where located VMs
# external_vsphere_version: "6.7u3"
## Tags for the external vSphere Cloud Provider images ## Tags for the external vSphere Cloud Provider images
# external_vsphere_cloud_controller_image_tag: "latest" # external_vsphere_cloud_controller_image_tag: "latest"
# vsphere_syncer_image_tag: "v1.0.2" # vsphere_syncer_image_tag: "v1.0.2"
@ -14,6 +17,7 @@
# vsphere_csi_controller: "v1.0.2" # vsphere_csi_controller: "v1.0.2"
# vsphere_csi_liveness_probe_image_tag: "v1.1.0" # vsphere_csi_liveness_probe_image_tag: "v1.1.0"
# vsphere_csi_provisioner_image_tag: "v1.2.2" # vsphere_csi_provisioner_image_tag: "v1.2.2"
# vsphere_csi_resizer_tag: "v1.0.0"
## To use vSphere CSI plugin to provision volumes set this value to true ## To use vSphere CSI plugin to provision volumes set this value to true
# vsphere_csi_enabled: true # vsphere_csi_enabled: true

View file

@ -2,6 +2,7 @@
external_vsphere_vcenter_port: "443" external_vsphere_vcenter_port: "443"
external_vsphere_insecure: "true" external_vsphere_insecure: "true"
external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id" external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id"
external_vsphere_version: "6.7u3"
vsphere_syncer_image_tag: "v1.0.2" vsphere_syncer_image_tag: "v1.0.2"
vsphere_csi_attacher_image_tag: "v1.1.1" vsphere_csi_attacher_image_tag: "v1.1.1"
@ -10,5 +11,8 @@ vsphere_csi_liveness_probe_image_tag: "v1.1.0"
vsphere_csi_provisioner_image_tag: "v1.2.2" vsphere_csi_provisioner_image_tag: "v1.2.2"
vsphere_csi_node_driver_registrar_image_tag: "v1.1.0" vsphere_csi_node_driver_registrar_image_tag: "v1.1.0"
vsphere_csi_driver_image_tag: "v1.0.2" vsphere_csi_driver_image_tag: "v1.0.2"
vsphere_csi_resizer_tag: "v1.0.0"
vsphere_csi_controller_replicas: 1 vsphere_csi_controller_replicas: 1
csi_endpoint: '{% if external_vsphere_version >= "7.0u1" %}/csi{% else %}/var/lib/csi/sockets/pluginproxy{% endif %}'

View file

@ -2,6 +2,11 @@
- include_tasks: vsphere-credentials-check.yml - include_tasks: vsphere-credentials-check.yml
tags: vsphere-csi-driver tags: vsphere-csi-driver
- name: vSphere CSI Driver | Choose how to deploy CSI driver based on controller version
set_fact:
controller_spec: "{% if vsphere_csi_controller is version('v2.0.0', '<') %}vsphere-csi-controller-ss.yml{% else %}vsphere-csi-controller-deployment.yml{% endif %}"
tags: vsphere-csi-driver
- name: vSphere CSI Driver | Generate CSI cloud-config - name: vSphere CSI Driver | Generate CSI cloud-config
template: template:
src: "{{ item }}.j2" src: "{{ item }}.j2"
@ -18,7 +23,7 @@
dest: "{{ kube_config_dir }}/{{ item }}" dest: "{{ kube_config_dir }}/{{ item }}"
with_items: with_items:
- vsphere-csi-controller-rbac.yml - vsphere-csi-controller-rbac.yml
- vsphere-csi-controller-ss.yml - "{{ controller_spec }}"
- vsphere-csi-node.yml - vsphere-csi-node.yml
register: vsphere_csi_manifests register: vsphere_csi_manifests
when: inventory_hostname == groups['kube_control_plane'][0] when: inventory_hostname == groups['kube_control_plane'][0]

View file

@ -0,0 +1,188 @@
kind: Deployment
apiVersion: apps/v1
metadata:
name: vsphere-csi-controller
namespace: kube-system
spec:
replicas: {{ vsphere_csi_controller_replicas }}
selector:
matchLabels:
app: vsphere-csi-controller
template:
metadata:
labels:
app: vsphere-csi-controller
role: vsphere-csi
spec:
serviceAccountName: vsphere-csi-controller
nodeSelector:
{% if kube_version is version('v1.20.0', '<') %}
node-role.kubernetes.io/master: ""
{% else %}
node-role.kubernetes.io/control-plane: ""
{% endif %}
tolerations:
- operator: "Exists"
key: node-role.kubernetes.io/master
effect: NoSchedule
- operator: "Exists"
key: node-role.kubernetes.io/control-plane
effect: NoSchedule
dnsPolicy: "Default"
containers:
- name: csi-attacher
image: {{ quay_image_repo }}/k8scsi/csi-attacher:{{ vsphere_csi_attacher_image_tag }}
args:
- "--v=4"
- "--timeout=300s"
- "--csi-address=$(ADDRESS)"
- "--leader-election"
env:
- name: ADDRESS
value: /csi/csi.sock
volumeMounts:
- mountPath: /csi
name: socket-dir
{% if external_vsphere_version >= "7.0" %}
- name: csi-resizer
image: {{ gcr_image_repo }}/k8scsi/csi-resizer:{{ vsphere_csi_resizer_tag }}
args:
- "--v=4"
- "--timeout=300s"
- "--csi-address=$(ADDRESS)"
- "--leader-election"
env:
- name: ADDRESS
value: /csi/csi.sock
volumeMounts:
- mountPath: /csi
name: socket-dir
{% endif %}
- name: vsphere-csi-controller
image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_controller }}
{% if external_vsphere_version >= "7.0u1" %}
args:
- "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
- "--fss-namespace=$(CSI_NAMESPACE)"
{% endif %}
imagePullPolicy: {{ k8s_image_pull_policy }}
env:
- name: CSI_ENDPOINT
value: unix://{{ csi_endpoint }}/csi.sock
- name: X_CSI_MODE
value: "controller"
- name: VSPHERE_CSI_CONFIG
value: "/etc/cloud/csi-vsphere.conf"
- name: LOGGER_LEVEL
value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION
{% if external_vsphere_version >= "7.0u1" %}
- name: INCLUSTER_CLIENT_QPS
value: "100"
- name: INCLUSTER_CLIENT_BURST
value: "100"
- name: CSI_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{% endif %}
- name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT
value: 3m
volumeMounts:
- mountPath: /etc/cloud
name: vsphere-config-volume
readOnly: true
- mountPath: {{ csi_endpoint }}
name: socket-dir
ports:
- name: healthz
containerPort: 9808
protocol: TCP
livenessProbe:
httpGet:
path: /healthz
port: healthz
initialDelaySeconds: 10
timeoutSeconds: 3
periodSeconds: 5
failureThreshold: 3
- name: liveness-probe
image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
args:
- "--v=4"
- "--csi-address=$(ADDRESS)"
env:
- name: ADDRESS
value: {{ csi_endpoint }}/csi.sock
volumeMounts:
- name: socket-dir
mountPath: {{ csi_endpoint }}
- name: vsphere-syncer
image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/syncer:{{ vsphere_syncer_image_tag }}
args:
- "--leader-election"
{% if external_vsphere_version >= "7.0u1" %}
- "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
- "--fss-namespace=$(CSI_NAMESPACE)"
{% endif %}
imagePullPolicy: {{ k8s_image_pull_policy }}
env:
- name: FULL_SYNC_INTERVAL_MINUTES
value: "30"
- name: VSPHERE_CSI_CONFIG
value: "/etc/cloud/csi-vsphere.conf"
- name: LOGGER_LEVEL
value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION
{% if external_vsphere_version >= "7.0u1" %}
- name: INCLUSTER_CLIENT_QPS
value: "100"
- name: INCLUSTER_CLIENT_BURST
value: "100"
- name: CSI_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{% endif %}
volumeMounts:
- mountPath: /etc/cloud
name: vsphere-config-volume
readOnly: true
- name: csi-provisioner
image: {{ quay_image_repo }}/k8scsi/csi-provisioner:{{ vsphere_csi_provisioner_image_tag }}
args:
- "--v=4"
- "--timeout=300s"
- "--csi-address=$(ADDRESS)"
- "--leader-election"
- "--default-fstype=ext4"
# needed only for topology aware setup
#- "--feature-gates=Topology=true"
#- "--strict-topology"
env:
- name: ADDRESS
value: /csi/csi.sock
volumeMounts:
- mountPath: /csi
name: socket-dir
volumes:
- name: vsphere-config-volume
secret:
secretName: vsphere-config-secret
- name: socket-dir
emptyDir: {}
---
apiVersion: v1
data:
"csi-migration": "false" # csi-migration feature is only available for vSphere 7.0U1
kind: ConfigMap
metadata:
name: internal-feature-states.csi.vsphere.vmware.com
namespace: kube-system
---
apiVersion: storage.k8s.io/v1 # For k8s 1.17 or lower use storage.k8s.io/v1beta1
kind: CSIDriver
metadata:
name: csi.vsphere.vmware.com
spec:
attachRequired: true
podInfoOnMount: false

View file

@ -10,23 +10,47 @@ metadata:
name: vsphere-csi-controller-role name: vsphere-csi-controller-role
rules: rules:
- apiGroups: [""] - apiGroups: [""]
resources: ["nodes", "persistentvolumeclaims", "pods"] resources: ["nodes", "persistentvolumeclaims", "pods", "configmaps"]
verbs: ["get", "list", "watch"] verbs: ["get", "list", "watch"]
{% if external_vsphere_version >= "7.0" %}
- apiGroups: [""]
resources: ["persistentvolumeclaims/status"]
{% if external_vsphere_version >= "7.0u1" %}
verbs: ["patch"]
{% else %}
verbs: ["update", "patch"]
{% endif %}
{% endif %}
- apiGroups: [""] - apiGroups: [""]
resources: ["persistentvolumes"] resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "update", "delete"] verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
- apiGroups: [""] - apiGroups: [""]
resources: ["events"] resources: ["events"]
verbs: ["get", "list", "watch", "create", "update", "patch"] verbs: ["get", "list", "watch", "create", "update", "patch"]
{% if vsphere_csi_controller is version('v2.0.0', '>=') %}
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
{% endif %}
- apiGroups: ["storage.k8s.io"] - apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"] resources: ["storageclasses","csinodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["csinodes"]
verbs: ["get", "list", "watch"] verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"] - apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"] resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"] verbs: ["get", "list", "watch", "patch"]
{% if external_vsphere_version >= "7.0u1" %}
- apiGroups: ["cns.vmware.com"]
resources: ["cnsvspherevolumemigrations"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["get", "create"]
{% endif %}
{% if vsphere_csi_controller is version('v2.0.0', '>=') %}
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments/status"]
verbs: ["patch"]
{% endif %}
--- ---
kind: ClusterRoleBinding kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1 apiVersion: rbac.authorization.k8s.io/v1

View file

@ -9,6 +9,8 @@ spec:
app: vsphere-csi-node app: vsphere-csi-node
updateStrategy: updateStrategy:
type: "RollingUpdate" type: "RollingUpdate"
rollingUpdate:
maxUnavailable: 1
template: template:
metadata: metadata:
labels: labels:
@ -19,19 +21,24 @@ spec:
containers: containers:
- name: node-driver-registrar - name: node-driver-registrar
image: {{ quay_image_repo }}/k8scsi/csi-node-driver-registrar:{{ vsphere_csi_node_driver_registrar_image_tag }} image: {{ quay_image_repo }}/k8scsi/csi-node-driver-registrar:{{ vsphere_csi_node_driver_registrar_image_tag }}
{% if external_vsphere_version < "7.0u1" %}
lifecycle: lifecycle:
preStop: preStop:
exec: exec:
command: ["/bin/sh", "-c", "rm -rf /registration/csi.vsphere.vmware.com /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com-reg.sock"] command: ["/bin/sh", "-c", "rm -rf /registration/csi.vsphere.vmware.com-reg.sock /csi/csi.sock"]
{% endif %}
args: args:
- "--v=5" - "--v=5"
- "--csi-address=$(ADDRESS)" - "--csi-address=$(ADDRESS)"
- "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
{% if external_vsphere_version >= "7.0u1" %}
- "--health-port=9809"
{% endif %}
env: env:
- name: ADDRESS - name: ADDRESS
value: /csi/csi.sock value: /csi/csi.sock
- name: DRIVER_REG_SOCK_PATH - name: DRIVER_REG_SOCK_PATH
value: /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com/csi.sock value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock
securityContext: securityContext:
privileged: true privileged: true
volumeMounts: volumeMounts:
@ -39,9 +46,26 @@ spec:
mountPath: /csi mountPath: /csi
- name: registration-dir - name: registration-dir
mountPath: /registration mountPath: /registration
{% if external_vsphere_version >= "7.0u1" %}
ports:
- containerPort: 9809
name: healthz
livenessProbe:
httpGet:
path: /healthz
port: healthz
initialDelaySeconds: 5
timeoutSeconds: 5
{% endif %}
- name: vsphere-csi-node - name: vsphere-csi-node
image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_driver_image_tag }} image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_driver_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }} imagePullPolicy: {{ k8s_image_pull_policy }}
{% if external_vsphere_version >= "7.0u1" %}
args:
- "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
- "--fss-namespace=$(CSI_NAMESPACE)"
{% endif %}
imagePullPolicy: "Always"
env: env:
- name: NODE_NAME - name: NODE_NAME
valueFrom: valueFrom:
@ -56,8 +80,16 @@ spec:
# needed only for topology aware setups # needed only for topology aware setups
#- name: VSPHERE_CSI_CONFIG #- name: VSPHERE_CSI_CONFIG
# value: "/etc/cloud/csi-vsphere.conf" # here csi-vsphere.conf is the name of the file used for creating secret using "--from-file" flag # value: "/etc/cloud/csi-vsphere.conf" # here csi-vsphere.conf is the name of the file used for creating secret using "--from-file" flag
args: - name: X_CSI_DEBUG
- "--v=4" value: "true"
- name: LOGGER_LEVEL
value: "PRODUCTION" # Options: DEVELOPMENT, PRODUCTION
{% if external_vsphere_version >= "7.0u1" %}
- name: CSI_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{% endif %}
securityContext: securityContext:
privileged: true privileged: true
capabilities: capabilities:
@ -78,24 +110,23 @@ spec:
- name: device-dir - name: device-dir
mountPath: /dev mountPath: /dev
ports: ports:
- name: healthz - containerPort: 9808
containerPort: 9808 name: healthz
protocol: TCP
livenessProbe: livenessProbe:
httpGet: httpGet:
path: /healthz path: /healthz
port: healthz port: healthz
initialDelaySeconds: 10 initialDelaySeconds: 10
timeoutSeconds: 3 timeoutSeconds: 5
periodSeconds: 5 periodSeconds: 5
failureThreshold: 3 failureThreshold: 3
- name: liveness-probe - name: liveness-probe
image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }} image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
args: args:
- "--csi-address=$(ADDRESS)" {% if external_vsphere_version >= "7.0u1" %}
env: - "--v=4"
- name: ADDRESS {% endif %}
value: /csi/csi.sock - "--csi-address=/csi/csi.sock"
volumeMounts: volumeMounts:
- name: plugin-dir - name: plugin-dir
mountPath: /csi mountPath: /csi
@ -107,10 +138,10 @@ spec:
- name: registration-dir - name: registration-dir
hostPath: hostPath:
path: /var/lib/kubelet/plugins_registry path: /var/lib/kubelet/plugins_registry
type: DirectoryOrCreate type: Directory
- name: plugin-dir - name: plugin-dir
hostPath: hostPath:
path: /var/lib/kubelet/plugins_registry/csi.vsphere.vmware.com path: /var/lib/kubelet/plugins/csi.vsphere.vmware.com
type: DirectoryOrCreate type: DirectoryOrCreate
- name: pods-mount-dir - name: pods-mount-dir
hostPath: hostPath:
@ -119,3 +150,8 @@ spec:
- name: device-dir - name: device-dir
hostPath: hostPath:
path: /dev path: /dev
tolerations:
- effect: NoExecute
operator: Exists
- effect: NoSchedule
operator: Exists