feat: upgrade azuredisk csi to v1.10.0 (#8432)

Signed-off-by: Cyril Corbon <corboncyril@gmail.com>
2022-01-24 09:41:56 +01:00 · 2022-01-24 09:41:56 +01:00 · 9fce9ca42a
commit 9fce9ca42a
parent f1adb734e3
7 changed files with 138 additions and 137 deletions
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@ -673,15 +673,14 @@ aws_ebs_csi_plugin_image_repo: "{{ docker_image_repo }}/amazon/aws-ebs-csi-drive
 aws_ebs_csi_plugin_image_tag: "v0.5.0"

 azure_csi_image_repo: "mcr.microsoft.com/oss/kubernetes-csi"
-azure_csi_provisioner_image_tag: "v1.5.0"
-azure_csi_attacher_image_tag: "v1.2.0"
-azure_csi_cluster_registrar_image_tag: "v1.0.1"
-azure_csi_node_registrar_image_tag: "v1.1.0"
-azure_csi_snapshotter_image_tag: "v2.0.0"
-azure_csi_resizer_image_tag: "v0.3.0"
-azure_csi_livenessprobe_image_tag: "v1.1.0"
+azure_csi_provisioner_image_tag: "v2.2.2"
+azure_csi_attacher_image_tag: "v3.3.0"
+azure_csi_resizer_image_tag: "v1.3.0"
+azure_csi_livenessprobe_image_tag: "v2.5.0"
+azure_csi_node_registrar_image_tag: "v2.4.0"
+azure_csi_snapshotter_image_tag: "v3.0.3"
 azure_csi_plugin_image_repo: "mcr.microsoft.com/k8s/csi"
-azure_csi_plugin_image_tag: "v0.7.0"
+azure_csi_plugin_image_tag: "v1.10.0"

 gcp_pd_csi_image_repo: "gke.gcr.io"
 gcp_pd_csi_driver_image_tag: "v0.7.0-gke.0"
--- a/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml
@ -1,4 +1,6 @@
 ---
 azure_csi_use_instance_metadata: true
-azure_csi_controller_replicas: 1
+azure_csi_controller_replicas: 2
 azure_csi_plugin_image_tag: latest
+azure_csi_controller_affinity: {}
+azure_csi_node_affinity: {}
--- a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2
@ -71,7 +71,10 @@ rules:
    verbs: ["get", "list", "watch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["volumeattachments"]
-    verbs: ["get", "list", "watch", "update"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: ["coordination.k8s.io"]
    resources: ["leases"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
@ -128,15 +131,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
  name: azuredisk-external-snapshotter-role
 rules:
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "create", "update", "patch"]
@ -150,14 +144,11 @@ rules:
    resources: ["volumesnapshotcontents"]
    verbs: ["create", "get", "list", "watch", "update", "delete"]
  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots"]
-    verbs: ["get", "list", "watch", "update"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
  - apiGroups: ["apiextensions.k8s.io"]
    resources: ["customresourcedefinitions"]
    verbs: ["create", "list", "watch", "delete"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshotcontents/status"]
-    verbs: ["update"]
  - apiGroups: ["coordination.k8s.io"]
    resources: ["leases"]
    verbs: ["get", "watch", "list", "delete", "update", "create"]
@ -197,6 +188,9 @@ rules:
  - apiGroups: ["coordination.k8s.io"]
    resources: ["leases"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@ -210,3 +204,27 @@ roleRef:
  kind: ClusterRole
  name: azuredisk-external-resizer-role
  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-azuredisk-controller-secret-role
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-azuredisk-controller-secret-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-azuredisk-controller-secret-role
+  apiGroup: rbac.authorization.k8s.io
--- a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2
@ -24,19 +24,23 @@ spec:
          effect: "NoSchedule"
        - key: "node-role.kubernetes.io/control-plane"
          effect: "NoSchedule"
+{% if azure_csi_controller_affinity %}
+      affinity:
+        {{ azure_csi_controller_affinity | to_nice_yaml | indent(width=8) }}
+{% endif %}
      containers:
        - name: csi-provisioner
          image: {{ azure_csi_image_repo }}/csi-provisioner:{{ azure_csi_provisioner_image_tag }}
          imagePullPolicy: {{ k8s_image_pull_policy }}
          args:
-            - "--provisioner=disk.csi.azure.com"
            - "--feature-gates=Topology=true"
            - "--csi-address=$(ADDRESS)"
-            - "--connection-timeout=15s"
-            - "--v=5"
-            - "--timeout=120s"
-            - "--enable-leader-election"
-            - "--leader-election-type=leases"
+            - "--v=2"
+            - "--timeout=15s"
+            - "--leader-election"
+            - "--worker-threads=40"
+            - "--extra-create-metadata=true"
+            - "--strict-topology=true"
          env:
            - name: ADDRESS
              value: /csi/csi.sock
@ -45,8 +49,7 @@ spec:
              name: socket-dir
          resources:
            limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 500Mi
            requests:
              cpu: 10m
              memory: 20Mi
@ -54,40 +57,20 @@ spec:
          image: {{ azure_csi_image_repo }}/csi-attacher:{{ azure_csi_attacher_image_tag }}
          imagePullPolicy: {{ k8s_image_pull_policy }}
          args:
-            - "-v=5"
+            - "-v=2"
            - "-csi-address=$(ADDRESS)"
-            - "-timeout=120s"
+            - "-timeout=600s"
            - "-leader-election"
-            - "-leader-election-type=leases"
+            - "-worker-threads=500"
          env:
            - name: ADDRESS
              value: /csi/csi.sock
          volumeMounts:
-            - mountPath: /csi
-              name: socket-dir
+          - mountPath: /csi
+            name: socket-dir
          resources:
            limits:
-              cpu: 200m
-              memory: 200Mi
-            requests:
-              cpu: 10m
-              memory: 20Mi
-        - name: cluster-driver-registrar
-          image: {{ azure_csi_image_repo }}/csi-cluster-driver-registrar:{{ azure_csi_cluster_registrar_image_tag }}
-          args:
-            - --csi-address=$(ADDRESS)
-            - --driver-requires-attachment=true
-            - --v=5
-          env:
-            - name: ADDRESS
-              value: /csi/csi.sock
-          volumeMounts:
-            - name: socket-dir
-              mountPath: /csi
-          resources:
-            limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 500Mi
            requests:
              cpu: 10m
              memory: 20Mi
@ -96,7 +79,7 @@ spec:
          args:
            - "-csi-address=$(ADDRESS)"
            - "-leader-election"
-            - "--v=5"
+            - "-v=2"
          env:
            - name: ADDRESS
              value: /csi/csi.sock
@ -105,8 +88,7 @@ spec:
              mountPath: /csi
          resources:
            limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 100Mi
            requests:
              cpu: 10m
              memory: 20Mi
@ -114,8 +96,10 @@ spec:
          image: {{ azure_csi_image_repo }}/csi-resizer:{{ azure_csi_resizer_image_tag }}
          args:
            - "-csi-address=$(ADDRESS)"
-            - "-v=5"
+            - "-v=2"
            - "-leader-election"
+            - '-handle-volume-inuse-error=false'
+            - "-timeout=60s"
          env:
            - name: ADDRESS
              value: /csi/csi.sock
@ -124,8 +108,7 @@ spec:
              mountPath: /csi
          resources:
            limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 500Mi
            requests:
              cpu: 10m
              memory: 20Mi
@ -133,16 +116,15 @@ spec:
          image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }}
          args:
            - --csi-address=/csi/csi.sock
-            - --connection-timeout=3s
+            - --probe-timeout=3s
            - --health-port=29602
-            - --v=5
+            - --v=2
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
          resources:
            limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 100Mi
            requests:
              cpu: 10m
              memory: 20Mi
@ -152,7 +134,11 @@ spec:
          args:
            - "--v=5"
            - "--endpoint=$(CSI_ENDPOINT)"
-            - "--nodeid=$(KUBE_NODE_NAME)"
+            - "--metrics-address=0.0.0.0:29604"
+            - "--disable-avset-nodes=true"
+            - "--drivername=disk.csi.azure.com"
+            - "--cloud-config-secret-name=cloud-config"
+            - "--cloud-config-secret-namespace=kube-system"
          ports:
            - containerPort: 29602
              name: healthz
@ -184,8 +170,7 @@ spec:
              name: msi
          resources:
            limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 500Mi
            requests:
              cpu: 10m
              memory: 20Mi
@ -195,6 +180,3 @@ spec:
        - name: azure-cred
          secret:
            secretName: cloud-config
-        - name: msi
-          hostPath:
-            path: /var/lib/waagent/ManagedIdentity-Settings
--- a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node-rbac.yml.j2
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node-rbac.yml.j2
@ -0,0 +1,30 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-azuredisk-node-sa
+  namespace: kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-azuredisk-node-secret-role
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-azuredisk-node-secret-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-node-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-azuredisk-node-secret-role
+  apiGroup: rbac.authorization.k8s.io
--- a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2
@ -5,6 +5,10 @@ metadata:
  name: csi-azuredisk-node
  namespace: kube-system
 spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
  selector:
    matchLabels:
      app: csi-azuredisk-node
@ -14,9 +18,17 @@ spec:
        app: csi-azuredisk-node
    spec:
      hostNetwork: true
+      dnsPolicy: Default
+      serviceAccountName: csi-azuredisk-node-sa
      nodeSelector:
        kubernetes.io/os: linux
+{% if azure_csi_node_affinity %}
+      affinity:
+        {{ azure_csi_node_affinity | to_nice_yaml | indent(width=8) }}
+{% endif %}
      priorityClassName: system-node-critical
+      tolerations:
+        - operator: Exists
      containers:
        - name: liveness-probe
          volumeMounts:
@ -26,13 +38,12 @@ spec:
          imagePullPolicy: {{ k8s_image_pull_policy }}
          args:
            - --csi-address=/csi/csi.sock
-            - --connection-timeout=3s
+            - --probe-timeout=3s
            - --health-port=29603
-            - --v=5
+            - --v=2
          resources:
            limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 100Mi
            requests:
              cpu: 10m
              memory: 20Mi
@ -41,11 +52,15 @@ spec:
          args:
            - --csi-address=$(ADDRESS)
            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
-            - --v=5
-          lifecycle:
-            preStop:
-              exec:
-                command: ["/bin/sh", "-c", "rm -rf /registration/disk.csi.azure.com-reg.sock /csi/csi.sock"]
+            - --v=2
+          livenessProbe:
+            exec:
+              command:
+                - /csi-node-driver-registrar
+                - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+                - --mode=kubelet-registration-probe
+            initialDelaySeconds: 30
+            timeoutSeconds: 15
          env:
            - name: ADDRESS
              value: /csi/csi.sock
@ -58,8 +73,7 @@ spec:
              mountPath: /registration
          resources:
            limits:
-              cpu: 200m
-              memory: 200Mi
+              memory: 100Mi
            requests:
              cpu: 10m
              memory: 20Mi
@ -71,6 +85,11 @@ spec:
            - "--endpoint=$(CSI_ENDPOINT)"
            - "--nodeid=$(KUBE_NODE_NAME)"
            - "--metrics-address=0.0.0.0:29605"
+            - "--enable-perf-optimization=true"
+            - "--drivername=disk.csi.azure.com"
+            - "--volume-attach-limit=-1"
+            - "--cloud-config-secret-name=cloud-config"
+            - "--cloud-config-secret-namespace=kube-system"
          ports:
            - containerPort: 29603
              name: healthz
@ -106,10 +125,6 @@ spec:
              name: mountpoint-dir
            - mountPath: /etc/kubernetes/
              name: azure-cred
-              readOnly: true
-            - mountPath: /var/lib/waagent/ManagedIdentity-Settings
-              readOnly: true
-              name: msi
            - mountPath: /dev
              name: device-dir
            - mountPath: /sys/bus/scsi/devices
@ -118,7 +133,6 @@ spec:
              name: scsi-host-dir
          resources:
            limits:
-              cpu: 200m
              memory: 200Mi
            requests:
              cpu: 10m
@ -136,12 +150,10 @@ spec:
            path: /var/lib/kubelet/plugins_registry/
            type: DirectoryOrCreate
          name: registration-dir
-        - name: azure-cred
-          secret:
-            secretName: cloud-config
        - hostPath:
-            path: /var/lib/waagent/ManagedIdentity-Settings
-          name: msi
+            path: /etc/kubernetes/
+            type: DirectoryOrCreate
+          name: azure-cred
        - hostPath:
            path: /dev
            type: Directory
--- a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2
@ -1,42 +0,0 @@
---
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  creationTimestamp: null
-  name: csinodeinfos.csi.storage.k8s.io
-spec:
-  group: csi.storage.k8s.io
-  scope: Cluster
-  names:
-    kind: CSINodeInfo
-    plural: csinodeinfos
-  versions:
-  - name: v1alpha1
-    served: true
-    storage: true
-    schema:
-      openAPIV3Schema:
-        type: object
-        properties:
-          csiDrivers:
-            description: List of CSI drivers running on the node and their properties.
-            items:
-              properties:
-                driver:
-                  description: The CSI driver that this object refers to.
-                  type: string
-                nodeID:
-                  description: The node from the driver point of view.
-                  type: string
-                topologyKeys:
-                  description: List of keys supported by the driver.
-                  items:
-                    type: string
-                  type: array
-            type: array
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []