vSphere-CSI: update to 2.4.0 (#8295)

This commit is contained in:
Cristian Calin 2021-12-10 21:07:23 +02:00 committed by GitHub
parent b396801e28
commit 0e969c0b72
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 155 additions and 241 deletions

View file

@ -14,18 +14,18 @@
## gcr.io/cloud-provider-vsphere/cpi/release/manager
# external_vsphere_cloud_controller_image_tag: "latest"
## gcr.io/cloud-provider-vsphere/csi/release/syncer
# vsphere_syncer_image_tag: "v2.2.1"
## quay.io/k8scsi/csi-attacher
# vsphere_csi_attacher_image_tag: "v3.1.0"
# vsphere_syncer_image_tag: "v2.4.0"
## k8s.gcr.io/sig-storage/csi-attacher
# vsphere_csi_attacher_image_tag: "v3.3.0"
## gcr.io/cloud-provider-vsphere/csi/release/driver
# vsphere_csi_controller: "v2.2.1"
## quay.io/k8scsi/livenessprobe
# vsphere_csi_liveness_probe_image_tag: "v2.2.0"
## quay.io/k8scsi/csi-provisioner
# vsphere_csi_provisioner_image_tag: "v2.1.0"
## quay.io/k8scsi/csi-resizer
# vsphere_csi_controller: "v2.4.0"
## k8s.gcr.io/sig-storage/livenessprobe
# vsphere_csi_liveness_probe_image_tag: "v2.4.0"
## k8s.gcr.io/sig-storage/csi-provisioner
# vsphere_csi_provisioner_image_tag: "v3.0.0"
## k8s.gcr.io/sig-storage/csi-resizer
## makes sense only for vSphere version >=7.0
# vsphere_csi_resizer_tag: "v1.1.0"
# vsphere_csi_resizer_tag: "v1.3.0"
## To use vSphere CSI plugin to provision volumes set this value to true
# vsphere_csi_enabled: true

View file

@ -4,14 +4,14 @@ external_vsphere_insecure: "true"
external_vsphere_kubernetes_cluster_id: "kubernetes-cluster-id"
external_vsphere_version: "6.7u3"
vsphere_syncer_image_tag: "v1.0.2"
vsphere_csi_attacher_image_tag: "v1.1.1"
vsphere_csi_controller: "v1.0.2"
vsphere_csi_liveness_probe_image_tag: "v1.1.0"
vsphere_csi_provisioner_image_tag: "v1.2.2"
vsphere_csi_node_driver_registrar_image_tag: "v1.1.0"
vsphere_csi_driver_image_tag: "v1.0.2"
vsphere_csi_resizer_tag: "v1.0.0"
vsphere_syncer_image_tag: "v2.4.0"
vsphere_csi_attacher_image_tag: "v3.3.0"
vsphere_csi_controller: "v2.4.0"
vsphere_csi_liveness_probe_image_tag: "v2.4.0"
vsphere_csi_provisioner_image_tag: "v3.0.0"
vsphere_csi_node_driver_registrar_image_tag: "v2.3.0"
vsphere_csi_driver_image_tag: "v2.4.0"
vsphere_csi_resizer_tag: "v1.3.0"
vsphere_csi_controller_replicas: 1

View file

@ -2,11 +2,6 @@
- include_tasks: vsphere-credentials-check.yml
tags: vsphere-csi-driver
- name: vSphere CSI Driver | Choose how to deploy CSI driver based on controller version
set_fact:
controller_spec: "{% if vsphere_csi_controller is version('v2.0.0', '<') %}vsphere-csi-controller-ss.yml{% else %}vsphere-csi-controller-deployment.yml{% endif %}"
tags: vsphere-csi-driver
- name: vSphere CSI Driver | Generate CSI cloud-config
template:
src: "{{ item }}.j2"
@ -22,9 +17,12 @@
src: "{{ item }}.j2"
dest: "{{ kube_config_dir }}/{{ item }}"
with_items:
- vsphere-csi-driver.yml
- vsphere-csi-controller-rbac.yml
- vsphere-csi-node-rbac.yaml
- "{{ controller_spec }}"
- vsphere-csi-node-rbac.yml
- vsphere-csi-controller-config.yml
- vsphere-csi-controller-deployment.yml
- vsphere-csi-controller-service.yml
- vsphere-csi-node.yml
register: vsphere_csi_manifests
when: inventory_hostname == groups['kube_control_plane'][0]

View file

@ -0,0 +1,15 @@
apiVersion: v1
data:
"csi-migration": "false"
"csi-auth-check": "true"
"online-volume-extend": "true"
"trigger-csi-fullsync": "false"
"async-query-volume": "true"
"improved-csi-idempotency": "true"
"improved-volume-topology": "true"
"block-volume-snapshot": "false"
"csi-windows-support": "false"
kind: ConfigMap
metadata:
name: internal-feature-states.csi.vsphere.vmware.com
namespace: kube-system

View file

@ -5,6 +5,11 @@ metadata:
namespace: kube-system
spec:
replicas: {{ vsphere_csi_controller_replicas }}
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
maxSurge: 0
selector:
matchLabels:
app: vsphere-csi-controller
@ -14,6 +19,16 @@ spec:
app: vsphere-csi-controller
role: vsphere-csi
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- vsphere-csi-controller
topologyKey: "kubernetes.io/hostname"
serviceAccountName: vsphere-csi-controller
nodeSelector:
node-role.kubernetes.io/control-plane: ""
@ -39,12 +54,14 @@ spec:
dnsPolicy: "Default"
containers:
- name: csi-attacher
image: {{ quay_image_repo }}/k8scsi/csi-attacher:{{ vsphere_csi_attacher_image_tag }}
image: {{ kube_image_repo }}/sig-storage/csi-attacher:{{ vsphere_csi_attacher_image_tag }}
args:
- "--v=4"
- "--timeout=300s"
- "--csi-address=$(ADDRESS)"
- "--leader-election"
- "--kube-api-qps=100"
- "--kube-api-burst=100"
env:
- name: ADDRESS
value: /csi/csi.sock
@ -53,17 +70,15 @@ spec:
name: socket-dir
{% if external_vsphere_version >= "7.0" %}
- name: csi-resizer
image: {{ quay_image_repo }}/k8scsi/csi-resizer:{{ vsphere_csi_resizer_tag }}
image: {{ kube_image_repo }}/sig-storage/csi-resizer:{{ vsphere_csi_resizer_tag }}
args:
- "--v=4"
- "--timeout=300s"
- "--csi-address=$(ADDRESS)"
- "--leader-election"
{% if vsphere_csi_controller is version('v2.2.0', '>=') %}
- "--handle-volume-inuse-error=false"
- "--kube-api-qps=100"
- "--kube-api-burst=100"
{% endif %}
env:
- name: ADDRESS
value: /csi/csi.sock
@ -77,6 +92,7 @@ spec:
args:
- "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
- "--fss-namespace=$(CSI_NAMESPACE)"
- "--use-gocsi=false"
{% endif %}
imagePullPolicy: {{ k8s_image_pull_policy }}
env:
@ -84,6 +100,10 @@ spec:
value: unix://{{ csi_endpoint }}/csi.sock
- name: X_CSI_MODE
value: "controller"
- name: X_CSI_SPEC_DISABLE_LEN_CHECK
value: "true"
- name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT
value: 3m
- name: VSPHERE_CSI_CONFIG
value: "/etc/cloud/csi-vsphere.conf"
- name: LOGGER_LEVEL
@ -98,8 +118,6 @@ spec:
fieldRef:
fieldPath: metadata.namespace
{% endif %}
- name: X_CSI_SERIAL_VOL_ACCESS_TIMEOUT
value: 3m
volumeMounts:
- mountPath: /etc/cloud
name: vsphere-config-volume
@ -122,7 +140,7 @@ spec:
periodSeconds: 5
failureThreshold: 3
- name: liveness-probe
image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
image: {{ kube_image_repo }}/sig-storage/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
args:
- "--v=4"
- "--csi-address=$(ADDRESS)"
@ -167,7 +185,7 @@ spec:
name: vsphere-config-volume
readOnly: true
- name: csi-provisioner
image: {{ quay_image_repo }}/k8scsi/csi-provisioner:{{ vsphere_csi_provisioner_image_tag }}
image: {{ kube_image_repo }}/sig-storage/csi-provisioner:{{ vsphere_csi_provisioner_image_tag }}
args:
- "--v=4"
- "--timeout=300s"
@ -193,42 +211,3 @@ spec:
secretName: vsphere-config-secret
- name: socket-dir
emptyDir: {}
---
apiVersion: v1
data:
"csi-migration": "false" # csi-migration feature is only available for vSphere 7.0U1
"csi-auth-check": "true"
"online-volume-extend": "true"
kind: ConfigMap
metadata:
name: internal-feature-states.csi.vsphere.vmware.com
namespace: kube-system
---
apiVersion: storage.k8s.io/v1 # For k8s 1.17 or lower use storage.k8s.io/v1beta1
kind: CSIDriver
metadata:
name: csi.vsphere.vmware.com
spec:
attachRequired: true
podInfoOnMount: false
---
apiVersion: v1
kind: Service
metadata:
name: vsphere-csi-controller
namespace: kube-system
labels:
app: vsphere-csi-controller
spec:
ports:
- name: ctlr
port: 2112
targetPort: 2112
protocol: TCP
- name: syncer
port: 2113
targetPort: 2113
protocol: TCP
selector:
app: vsphere-csi-controller

View file

@ -39,18 +39,37 @@ rules:
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "patch"]
{% if external_vsphere_version >= "7.0u1" %}
- apiGroups: ["cns.vmware.com"]
resources: ["triggercsifullsyncs"]
verbs: ["create", "get", "update", "watch", "list"]
- apiGroups: ["cns.vmware.com"]
resources: ["cnsvspherevolumemigrations"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["get", "create"]
verbs: ["get", "create", "update"]
- apiGroups: ["cns.vmware.com"]
resources: ["cnsvolumeoperationrequests"]
verbs: ["create", "get", "list", "update", "delete"]
- apiGroups: [ "cns.vmware.com" ]
resources: [ "csinodetopologies" ]
verbs: ["get", "update", "watch", "list"]
{% endif %}
{% if vsphere_csi_controller is version('v2.0.0', '>=') %}
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments/status"]
verbs: ["patch"]
{% endif %}
- apiGroups: [ "snapshot.storage.k8s.io" ]
resources: [ "volumesnapshots" ]
verbs: [ "get", "list" ]
- apiGroups: [ "snapshot.storage.k8s.io" ]
resources: [ "volumesnapshotclasses" ]
verbs: [ "watch", "get", "list" ]
- apiGroups: [ "snapshot.storage.k8s.io" ]
resources: [ "volumesnapshotcontents" ]
verbs: [ "create", "get", "list", "watch", "update", "delete" ]
- apiGroups: [ "snapshot.storage.k8s.io" ]
resources: [ "volumesnapshotcontents/status" ]
verbs: [ "update", "patch" ]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1

View file

@ -0,0 +1,19 @@
apiVersion: v1
kind: Service
metadata:
name: vsphere-csi-controller
namespace: kube-system
labels:
app: vsphere-csi-controller
spec:
ports:
- name: ctlr
port: 2112
targetPort: 2112
protocol: TCP
- name: syncer
port: 2113
targetPort: 2113
protocol: TCP
selector:
app: vsphere-csi-controller

View file

@ -1,131 +0,0 @@
kind: StatefulSet
apiVersion: apps/v1
metadata:
name: vsphere-csi-controller
namespace: kube-system
spec:
serviceName: vsphere-csi-controller
replicas: {{ vsphere_csi_controller_replicas }}
updateStrategy:
type: "RollingUpdate"
selector:
matchLabels:
app: vsphere-csi-controller
template:
metadata:
labels:
app: vsphere-csi-controller
role: vsphere-csi
spec:
serviceAccountName: vsphere-csi-controller
nodeSelector:
node-role.kubernetes.io/control-plane: ""
tolerations:
- operator: "Exists"
key: node-role.kubernetes.io/master
effect: NoSchedule
- operator: "Exists"
key: node-role.kubernetes.io/control-plane
effect: NoSchedule
dnsPolicy: "Default"
containers:
- name: csi-attacher
image: {{ quay_image_repo }}/k8scsi/csi-attacher:{{ vsphere_csi_attacher_image_tag }}
args:
- "--v=4"
- "--timeout=300s"
- "--csi-address=$(ADDRESS)"
env:
- name: ADDRESS
value: /csi/csi.sock
volumeMounts:
- mountPath: /csi
name: socket-dir
- name: vsphere-csi-controller
image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_controller }}
imagePullPolicy: {{ k8s_image_pull_policy }}
lifecycle:
preStop:
exec:
command: ["/bin/sh", "-c", "rm -rf /var/lib/csi/sockets/pluginproxy/csi.vsphere.vmware.com"]
args:
- "--v=4"
env:
- name: CSI_ENDPOINT
value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
- name: X_CSI_MODE
value: "controller"
- name: VSPHERE_CSI_CONFIG
value: "/etc/cloud/csi-vsphere.conf"
volumeMounts:
- mountPath: /etc/cloud
name: vsphere-config-volume
readOnly: true
- mountPath: /var/lib/csi/sockets/pluginproxy/
name: socket-dir
ports:
- name: healthz
containerPort: 9808
protocol: TCP
livenessProbe:
httpGet:
path: /healthz
port: healthz
initialDelaySeconds: 10
timeoutSeconds: 3
periodSeconds: 5
failureThreshold: 3
- name: liveness-probe
image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
args:
- "--csi-address=$(ADDRESS)"
env:
- name: ADDRESS
value: /var/lib/csi/sockets/pluginproxy/csi.sock
volumeMounts:
- mountPath: /var/lib/csi/sockets/pluginproxy/
name: socket-dir
- name: vsphere-syncer
image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/syncer:{{ vsphere_syncer_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }}
args:
- "--v=2"
env:
- name: FULL_SYNC_INTERVAL_MINUTES
value: "30"
- name: VSPHERE_CSI_CONFIG
value: "/etc/cloud/csi-vsphere.conf"
volumeMounts:
- mountPath: /etc/cloud
name: vsphere-config-volume
readOnly: true
- name: csi-provisioner
image: {{ quay_image_repo }}/k8scsi/csi-provisioner:{{ vsphere_csi_provisioner_image_tag }}
args:
- "--v=4"
- "--timeout=300s"
- "--csi-address=$(ADDRESS)"
- "--feature-gates=Topology=true"
- "--strict-topology"
env:
- name: ADDRESS
value: /csi/csi.sock
volumeMounts:
- mountPath: /csi
name: socket-dir
volumes:
- name: vsphere-config-volume
secret:
secretName: vsphere-config-secret
- name: socket-dir
hostPath:
path: /var/lib/csi/sockets/pluginproxy/csi.vsphere.vmware.com
type: DirectoryOrCreate
---
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
name: csi.vsphere.vmware.com
spec:
attachRequired: true
podInfoOnMount: false

View file

@ -0,0 +1,7 @@
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
name: csi.vsphere.vmware.com
spec:
attachRequired: true
podInfoOnMount: false

View file

@ -5,6 +5,31 @@ metadata:
name: vsphere-csi-node
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vsphere-csi-node-cluster-role
rules:
- apiGroups: ["cns.vmware.com"]
resources: ["csinodetopologies"]
verbs: ["create", "watch"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: vsphere-csi-node-cluster-role-binding
subjects:
- kind: ServiceAccount
name: vsphere-csi-node
namespace: kube-system
roleRef:
kind: ClusterRole
name: vsphere-csi-node-cluster-role
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:

View file

@ -17,10 +17,14 @@ spec:
app: vsphere-csi-node
role: vsphere-csi
spec:
dnsPolicy: "Default"
nodeSelector:
kubernetes.io/os: linux
serviceAccountName: vsphere-csi-node
hostNetwork: true
dnsPolicy: "ClusterFirstWithHostNet"
containers:
- name: node-driver-registrar
image: {{ quay_image_repo }}/k8scsi/csi-node-driver-registrar:{{ vsphere_csi_node_driver_registrar_image_tag }}
image: {{ kube_image_repo }}/sig-storage/csi-node-driver-registrar:{{ vsphere_csi_node_driver_registrar_image_tag }}
{% if external_vsphere_version < "7.0u1" %}
lifecycle:
preStop:
@ -31,34 +35,23 @@ spec:
- "--v=5"
- "--csi-address=$(ADDRESS)"
- "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
{% if external_vsphere_version >= "7.0u1" %}
- "--health-port=9809"
{% endif %}
env:
- name: ADDRESS
value: /csi/csi.sock
- name: DRIVER_REG_SOCK_PATH
value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock
{% if vsphere_csi_controller is version('v2.2.0', '<') %}
securityContext:
privileged: true
{% endif %}
volumeMounts:
- name: plugin-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
{% if external_vsphere_version >= "7.0u1" %}
ports:
- containerPort: 9809
name: healthz
livenessProbe:
httpGet:
path: /healthz
port: healthz
initialDelaySeconds: 5
timeoutSeconds: 5
{% endif %}
exec:
command:
- /csi-node-driver-registrar
- --kubelet-registration-path=/var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock
- --mode=kubelet-registration-probe
initialDelaySeconds: 3
- name: vsphere-csi-node
image: {{ gcr_image_repo }}/cloud-provider-vsphere/csi/release/driver:{{ vsphere_csi_driver_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }}
@ -66,6 +59,7 @@ spec:
args:
- "--fss-name=internal-feature-states.csi.vsphere.vmware.com"
- "--fss-namespace=$(CSI_NAMESPACE)"
- "--use-gocsi=false"
{% endif %}
imagePullPolicy: "Always"
env:
@ -75,13 +69,12 @@ spec:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: MAX_VOLUMES_PER_NODE
value: "59" # Maximum number of volumes that controller can publish to the node. If value is not set or zero Kubernetes decide how many volumes can be published by the controller to the node.
- name: X_CSI_MODE
value: "node"
- name: X_CSI_SPEC_REQ_VALIDATION
value: "false"
# needed only for topology aware setups
#- name: VSPHERE_CSI_CONFIG
# value: "/etc/cloud/csi-vsphere.conf" # here csi-vsphere.conf is the name of the file used for creating secret using "--from-file" flag
- name: X_CSI_DEBUG
value: "true"
- name: LOGGER_LEVEL
@ -92,16 +85,14 @@ spec:
fieldRef:
fieldPath: metadata.namespace
{% endif %}
- name: NODEGETINFO_WATCH_TIMEOUT_MINUTES
value: "1"
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
volumeMounts:
# needed only for topology aware setups
#- name: vsphere-config-volume
# mountPath: /etc/cloud
# readOnly: true
- name: plugin-dir
mountPath: /csi
- name: pods-mount-dir
@ -111,12 +102,10 @@ spec:
mountPropagation: "Bidirectional"
- name: device-dir
mountPath: /dev
{% if vsphere_csi_controller is version('v2.2.0', '>=') %}
- name: blocks-dir
mountPath: /sys/block
- name: sys-devices-dir
mountPath: /sys/devices
{% endif %}
ports:
- containerPort: 9808
name: healthz
@ -129,7 +118,7 @@ spec:
periodSeconds: 5
failureThreshold: 3
- name: liveness-probe
image: {{ quay_image_repo }}/k8scsi/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
image: {{ kube_image_repo }}/sig-storage/livenessprobe:{{ vsphere_csi_liveness_probe_image_tag }}
args:
{% if external_vsphere_version >= "7.0u1" %}
- "--v=4"
@ -139,10 +128,6 @@ spec:
- name: plugin-dir
mountPath: /csi
volumes:
# needed only for topology aware setups
#- name: vsphere-config-volume
# secret:
# secretName: vsphere-config-secret
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins_registry
@ -158,7 +143,6 @@ spec:
- name: device-dir
hostPath:
path: /dev
{% if vsphere_csi_controller is version('v2.2.0', '>=') %}
- name: blocks-dir
hostPath:
path: /sys/block
@ -167,7 +151,6 @@ spec:
hostPath:
path: /sys/devices
type: Directory
{% endif %}
tolerations:
- effect: NoExecute
operator: Exists