From 937adec51529ab644f35640da74781d144e242e5 Mon Sep 17 00:00:00 2001 From: Ali Sanhaji Date: Wed, 1 Apr 2020 09:53:27 +0200 Subject: [PATCH] Azure Disk CSI deployment (#5833) * Azure Disk CSI deployment * Mention Azure CSI support * Fix: remove unnecessary file * Typo in documentation * Add newline to end of file --- docs/azure-csi.md | 119 ++++++++++ inventory/sample/group_vars/all/azure.yml | 20 ++ .../group_vars/k8s-cluster/k8s-cluster.yml | 3 +- roles/download/defaults/main.yml | 11 + .../csi_driver/azuredisk/defaults/main.yml | 4 + .../tasks/azure-credential-check.yml | 54 +++++ .../csi_driver/azuredisk/tasks/main.yml | 48 ++++ ...azure-csi-azuredisk-controller-rbac.yml.j2 | 212 ++++++++++++++++++ .../azure-csi-azuredisk-controller.yml.j2 | 200 +++++++++++++++++ .../azure-csi-azuredisk-driver.yml.j2 | 10 + .../templates/azure-csi-azuredisk-node.yml.j2 | 156 +++++++++++++ .../azure-csi-cloud-config-secret.yml.j2 | 7 + .../templates/azure-csi-cloud-config.j2 | 14 ++ .../templates/azure-csi-node-info-crd.yml.j2 | 38 ++++ roles/kubernetes-apps/meta/main.yml | 8 + .../azuredisk-csi/defaults/main.yml | 3 + .../azuredisk-csi/tasks/main.yml | 19 ++ .../templates/azure-csi-storage-class.yml.j2 | 11 + .../persistent_volumes/meta/main.yml | 7 + roles/kubespray-defaults/defaults/main.yaml | 1 + 20 files changed, 944 insertions(+), 1 deletion(-) create mode 100644 docs/azure-csi.md create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2 create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2 create mode 100644 roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml create mode 100644 roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml create mode 100644 roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2 diff --git a/docs/azure-csi.md b/docs/azure-csi.md new file mode 100644 index 000000000..e121c8874 --- /dev/null +++ b/docs/azure-csi.md @@ -0,0 +1,119 @@ +# Azure Disk CSI Driver + +The Azure Disk CSI driver allows you to provision volumes for pods with a Kubernetes deployment over Azure Cloud. The CSI driver replaces to volume provioning done by the in-tree azure cloud provider which is deprecated. + +This documentation is an updated version of the in-tree Azure cloud provider documentation (azure.md). + +To deploy Azure Disk CSI driver, uncomment the `azure_csi_enabled` option in `group_vars/all/azure.yml` and set it to `true`. + +## Azure Disk CSI Storage Class + +If you want to deploy the Azure Disk storage class to provision volumes dynamically, you should set `persistent_volumes_enabled` in `group_vars/k8s-cluster/k8s-cluster.yml` to `true`. + +## Parameters + +Before creating the instances you must first set the `azure_csi_` variables in the `group_vars/all.yml` file. + +All of the values can be retrieved using the azure cli tool which can be downloaded here: + +After installation you have to run `az login` to get access to your account. + +### azure\_csi\_tenant\_id + azure\_csi\_subscription\_id + +Run `az account show` to retrieve your subscription id and tenant id: +`azure_csi_tenant_id` -> tenantId field +`azure_csi_subscription_id` -> id field + +### azure\_csi\_location + +The region your instances are located in, it can be something like `francecentral` or `norwayeast`. A full list of region names can be retrieved via `az account list-locations` + +### azure\_csi\_resource\_group + +The name of the resource group your instances are in, a list of your resource groups can be retrieved via `az group list` + +Or you can do `az vm list | grep resourceGroup` and get the resource group corresponding to the VMs of your cluster. + +The resource group name is not case sensitive. + +### azure\_csi\_vnet\_name + +The name of the virtual network your instances are in, can be retrieved via `az network vnet list` + +### azure\_csi\_vnet\_resource\_group + +The name of the resource group your vnet is in, can be retrieved via `az network vnet list | grep resourceGroup` and get the resource group corresponding to the vnet of your cluster. + +### azure\_csi\_subnet\_name + +The name of the subnet your instances are in, can be retrieved via `az network vnet subnet list --resource-group RESOURCE_GROUP --vnet-name VNET_NAME` + +### azure\_csi\_security\_group\_name + +The name of the network security group your instances are in, can be retrieved via `az network nsg list` + +### azure\_csi\_aad\_client\_id + azure\_csi\_aad\_client\_secret + +These will have to be generated first: + +- Create an Azure AD Application with: +`az ad app create --display-name kubespray --identifier-uris http://kubespray --homepage http://kubespray.com --password CLIENT_SECRET` + +Display name, identifier-uri, homepage and the password can be chosen + +Note the AppId in the output. + +- Create Service principal for the application with: +`az ad sp create --id AppId` + +This is the AppId from the last command + +- Create the role assignment with: +`az role assignment create --role "Owner" --assignee http://kubespray --subscription SUBSCRIPTION_ID` + +azure\_csi\_aad\_client\_id must be set to the AppId, azure\_csi\_aad\_client\_secret is your chosen secret. + +### azure\_csi\_use\_instance\_metadata + +Use instance metadata service where possible. Boolean value. + +## Test the Azure Disk CSI driver + +To test the dynamic provisioning using Azure CSI driver, make sure to have the storage class deployed (through persistent volumes), and apply the following manifest: + +```yml +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-azuredisk +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: disk.csi.azure.com +--- +kind: Pod +apiVersion: v1 +metadata: + name: nginx-azuredisk +spec: + nodeSelector: + beta.kubernetes.io/os: linux + containers: + - image: nginx + name: nginx-azuredisk + command: + - "/bin/sh" + - "-c" + - while true; do echo $(date) >> /mnt/azuredisk/outfile; sleep 1; done + volumeMounts: + - name: azuredisk + mountPath: "/mnt/azuredisk" + volumes: + - name: azuredisk + persistentVolumeClaim: + claimName: pvc-azuredisk +``` diff --git a/inventory/sample/group_vars/all/azure.yml b/inventory/sample/group_vars/all/azure.yml index 8b0313fde..02ea0f91a 100644 --- a/inventory/sample/group_vars/all/azure.yml +++ b/inventory/sample/group_vars/all/azure.yml @@ -14,3 +14,23 @@ # azure_route_table_name: # supported values are 'standard' or 'vmss' # azure_vmtype: standard + +## Azure Disk CSI credentials and parameters +## see docs/azure-csi.md for details on how to get these values + +# azure_csi_tenant_id: +# azure_csi_subscription_id: +# azure_csi_aad_client_id: +# azure_csi_aad_client_secret: +# azure_csi_location: +# azure_csi_resource_group: +# azure_csi_vnet_name: +# azure_csi_vnet_resource_group: +# azure_csi_subnet_name: +# azure_csi_security_group_name: +# azure_csi_use_instance_metadata: + +## To enable Azure Disk CSI, uncomment below +# azure_csi_enabled: true +# azure_csi_controller_replicas: 1 +# azure_csi_plugin_image_tag: latest diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index d30857d50..233c050e8 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -255,7 +255,8 @@ podsecuritypolicy_enabled: false ## See https://github.com/kubernetes-sigs/kubespray/issues/2141 ## Set this variable to true to get rid of this issue volume_cross_zone_attachment: false -# Add Persistent Volumes Storage Class for corresponding cloud provider (supported: in-tree OpenStack, Cinder CSI, AWS EBS CSI, GCP Persistent Disk CSI) +## Add Persistent Volumes Storage Class for corresponding cloud provider (supported: in-tree OpenStack, Cinder CSI, +## AWS EBS CSI, Azure Disk CSI, GCP Persistent Disk CSI) persistent_volumes_enabled: false ## Container Engine Acceleration diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index b402ca079..7bbb92d76 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -531,6 +531,17 @@ cinder_csi_plugin_image_tag: "latest" aws_ebs_csi_plugin_image_repo: "{{ docker_image_repo }}/amazon/aws-ebs-csi-driver" aws_ebs_csi_plugin_image_tag: "latest" +azure_csi_image_repo: "mcr.microsoft.com/oss/kubernetes-csi" +azure_csi_provisioner_image_tag: "v1.5.0" +azure_csi_attacher_image_tag: "v1.2.0" +azure_csi_cluster_registrar_image_tag: "v1.0.1" +azure_csi_node_registrar_image_tag: "v1.1.0" +azure_csi_snapshotter_image_tag: "v2.0.0" +azure_csi_resizer_image_tag: "v0.3.0" +azure_csi_livenessprobe_image_tag: "v1.1.0" +azure_csi_plugin_image_repo: "mcr.microsoft.com/k8s/csi" +azure_csi_plugin_image_tag: "latest" + gcp_pd_csi_image_repo: "gke.gcr.io" gcp_pd_csi_driver_image_tag: "v0.7.0-gke.0" gcp_pd_csi_provisioner_image_tag: "v1.5.0-gke.0" diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml b/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml new file mode 100644 index 000000000..c1eec6401 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml @@ -0,0 +1,4 @@ +--- +azure_csi_use_instance_metadata: true +azure_csi_controller_replicas: 1 +azure_csi_plugin_image_tag: latest diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml b/roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml new file mode 100644 index 000000000..0a858ee75 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml @@ -0,0 +1,54 @@ +--- +- name: Azure CSI Driver | check azure_csi_tenant_id value + fail: + msg: "azure_csi_tenant_id is missing" + when: azure_csi_tenant_id is not defined or not azure_csi_tenant_id + +- name: Azure CSI Driver | check azure_csi_subscription_id value + fail: + msg: "azure_csi_subscription_id is missing" + when: azure_csi_subscription_id is not defined or not azure_csi_subscription_id + +- name: Azure CSI Driver | check azure_csi_aad_client_id value + fail: + msg: "azure_csi_aad_client_id is missing" + when: azure_csi_aad_client_id is not defined or not azure_csi_aad_client_id + +- name: Azure CSI Driver | check azure_csi_aad_client_secret value + fail: + msg: "azure_csi_aad_client_secret is missing" + when: azure_csi_aad_client_secret is not defined or not azure_csi_aad_client_secret + +- name: Azure CSI Driver | check azure_csi_resource_group value + fail: + msg: "azure_csi_resource_group is missing" + when: azure_csi_resource_group is not defined or not azure_csi_resource_group + +- name: Azure CSI Driver | check azure_csi_location value + fail: + msg: "azure_csi_location is missing" + when: azure_csi_location is not defined or not azure_csi_location + +- name: Azure CSI Driver | check azure_csi_subnet_name value + fail: + msg: "azure_csi_subnet_name is missing" + when: azure_csi_subnet_name is not defined or not azure_csi_subnet_name + +- name: Azure CSI Driver | check azure_csi_security_group_name value + fail: + msg: "azure_csi_security_group_name is missing" + when: azure_csi_security_group_name is not defined or not azure_csi_security_group_name + +- name: Azure CSI Driver | check azure_csi_vnet_name value + fail: + msg: "azure_csi_vnet_name is missing" + when: azure_csi_vnet_name is not defined or not azure_csi_vnet_name + +- name: Azure CSI Driver | check azure_csi_vnet_resource_group value + fail: + msg: "azure_csi_vnet_resource_group is missing" + when: azure_csi_vnet_resource_group is not defined or not azure_csi_vnet_resource_group + +- name: "Azure CSI Driver | check azure_csi_use_instance_metadata is a bool" + assert: + that: azure_csi_use_instance_metadata | type_debug == 'bool' diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml b/roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml new file mode 100644 index 000000000..e33ca292f --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml @@ -0,0 +1,48 @@ +--- +- include_tasks: azure-credential-check.yml + tags: azure-csi-driver + +- name: Azure CSI Driver | Write Azure CSI cloud-config + template: + src: "azure-csi-cloud-config.j2" + dest: "{{ kube_config_dir }}/azure_csi_cloud_config" + group: "{{ kube_cert_group }}" + mode: 0640 + when: inventory_hostname == groups['kube-master'][0] + tags: azure-csi-driver + +- name: Azure CSI Driver | Get base64 cloud-config + slurp: + src: "{{ kube_config_dir }}/azure_csi_cloud_config" + register: cloud_config_secret + when: inventory_hostname == groups['kube-master'][0] + tags: azure-csi-driver + +- name: Azure CSI Driver | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - {name: azure-csi-azuredisk-driver, file: azure-csi-azuredisk-driver.yml} + - {name: azure-csi-cloud-config-secret, file: azure-csi-cloud-config-secret.yml} + - {name: azure-csi-azuredisk-controller, file: azure-csi-azuredisk-controller-rbac.yml} + - {name: azure-csi-azuredisk-controller, file: azure-csi-azuredisk-controller.yml} + - {name: azure-csi-azuredisk-node, file: azure-csi-azuredisk-node.yml} + - {name: azure-csi-node-info-crd.yml.j2, file: azure-csi-node-info-crd.yml} + register: azure_csi_manifests + when: inventory_hostname == groups['kube-master'][0] + tags: azure-csi-driver + +- name: Azure CSI Driver | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ azure_csi_manifests.results }}" + when: + - inventory_hostname == groups['kube-master'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + tags: azure-csi-driver diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 new file mode 100644 index 000000000..ad974d38c --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 @@ -0,0 +1,212 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-azuredisk-controller-sa + namespace: kube-system +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-cluster-driver-registrar-role +rules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csidrivers"] + verbs: ["create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-driver-registrar-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-cluster-driver-registrar-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-snapshotter-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-resizer-role +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-resizer-role + apiGroup: rbac.authorization.k8s.io diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 new file mode 100644 index 000000000..b9cd4dc4f --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 @@ -0,0 +1,200 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-azuredisk-controller + namespace: kube-system +spec: + replicas: {{ azure_csi_controller_replicas }} + selector: + matchLabels: + app: csi-azuredisk-controller + template: + metadata: + labels: + app: csi-azuredisk-controller + spec: + hostNetwork: true + serviceAccountName: csi-azuredisk-controller-sa + nodeSelector: + beta.kubernetes.io/os: linux + priorityClassName: system-cluster-critical + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "true" + effect: "NoSchedule" + containers: + - name: csi-provisioner + image: {{ azure_csi_image_repo }}/csi-provisioner:{{ azure_csi_provisioner_image_tag }} + args: + - "--provisioner=disk.csi.azure.com" + - "--feature-gates=Topology=true" + - "--csi-address=$(ADDRESS)" + - "--connection-timeout=15s" + - "--v=5" + - "--timeout=120s" + - "--enable-leader-election" + - "--leader-election-type=leases" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-attacher + image: {{ azure_csi_image_repo }}/csi-attacher:{{ azure_csi_attacher_image_tag }} + args: + - "-v=5" + - "-csi-address=$(ADDRESS)" + - "-timeout=120s" + - "-leader-election" + - "-leader-election-type=leases" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: cluster-driver-registrar + image: {{ azure_csi_image_repo }}/csi-cluster-driver-registrar:{{ azure_csi_cluster_registrar_image_tag }} + args: + - --csi-address=$(ADDRESS) + - --driver-requires-attachment=true + - --v=5 + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-snapshotter + image: {{ azure_csi_image_repo }}/csi-snapshotter:{{ azure_csi_snapshotter_image_tag }} + args: + - "-csi-address=$(ADDRESS)" + - "-leader-election" + - "--v=5" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-resizer + image: {{ azure_csi_image_repo }}/csi-resizer:{{ azure_csi_resizer_image_tag }} + args: + - "-csi-address=$(ADDRESS)" + - "-v=5" + - "-leader-election" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: liveness-probe + image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }} + args: + - --csi-address=/csi/csi.sock + - --connection-timeout=3s + - --health-port=29602 + - --v=5 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: azuredisk + image: {{ azure_csi_plugin_image_repo }}/azuredisk-csi:{{ azure_csi_plugin_image_tag }} + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + ports: + - containerPort: 29602 + name: healthz + protocol: TCP + - containerPort: 29604 + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + value: "/etc/kubernetes/azure.json" + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + readOnly: true + - mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + name: msi + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: socket-dir + emptyDir: {} + - name: azure-cred + secret: + secretName: cloud-config + - name: msi + hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2 new file mode 100644 index 000000000..4c24cc244 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: disk.csi.azure.com +spec: + attachRequired: true + podInfoOnMount: true + volumeLifecycleModes: # added in Kubernetes 1.16 + - Persistent diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 new file mode 100644 index 000000000..08957781a --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 @@ -0,0 +1,156 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-azuredisk-node + namespace: kube-system +spec: + selector: + matchLabels: + app: csi-azuredisk-node + template: + metadata: + labels: + app: csi-azuredisk-node + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/os: linux + priorityClassName: system-node-critical + containers: + - name: liveness-probe + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /csi + name: socket-dir + image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }} + args: + - --csi-address=/csi/csi.sock + - --connection-timeout=3s + - --health-port=29603 + - --v=5 + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: node-driver-registrar + image: {{ azure_csi_image_repo }}/csi-node-driver-registrar:{{ azure_csi_node_registrar_image_tag }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/disk.csi.azure.com-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/disk.csi.azure.com/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: azuredisk + image: {{ azure_csi_plugin_image_repo }}/azuredisk-csi:{{ azure_csi_plugin_image_tag }} + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--metrics-address=0.0.0.0:29605" + ports: + - containerPort: 29603 + name: healthz + protocol: TCP + - containerPort: 29605 + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + value: "/etc/kubernetes/azure.json" + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/ + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + readOnly: true + - mountPath: /var/lib/waagent/ManagedIdentity-Settings + readOnly: true + name: msi + - mountPath: /dev + name: device-dir + - mountPath: /sys/bus/scsi/devices + name: sys-devices-dir + - mountPath: /sys/class/scsi_host/ + name: scsi-host-dir + resources: + limits: + cpu: 200m + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/disk.csi.azure.com + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/ + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - name: azure-cred + secret: + secretName: cloud-config + - hostPath: + path: /var/lib/waagent/ManagedIdentity-Settings + name: msi + - hostPath: + path: /dev + type: Directory + name: device-dir + - hostPath: + path: /sys/bus/scsi/devices + type: Directory + name: sys-devices-dir + - hostPath: + path: /sys/class/scsi_host/ + type: Directory + name: scsi-host-dir diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2 new file mode 100644 index 000000000..f259cec9f --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2 @@ -0,0 +1,7 @@ +kind: Secret +apiVersion: v1 +metadata: + name: cloud-config + namespace: kube-system +data: + azure.json: {{ cloud_config_secret.content }} diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2 new file mode 100644 index 000000000..d3932f50d --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2 @@ -0,0 +1,14 @@ +{ + "cloud":"AzurePublicCloud", + "tenantId": "{{ azure_csi_tenant_id }}", + "subscriptionId": "{{ azure_csi_subscription_id }}", + "aadClientId": "{{ azure_csi_aad_client_id }}", + "aadClientSecret": "{{ azure_csi_aad_client_secret }}", + "location": "{{ azure_csi_location }}", + "resourceGroup": "{{ azure_csi_resource_group }}", + "vnetName": "{{ azure_csi_vnet_name }}", + "vnetResourceGroup": "{{ azure_csi_vnet_resource_group }}", + "subnetName": "{{ azure_csi_subnet_name }}", + "securityGroupName": "{{ azure_csi_security_group_name }}", + "useInstanceMetadata": {{ azure_csi_use_instance_metadata }}, +} diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2 new file mode 100644 index 000000000..4ac7ff056 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2 @@ -0,0 +1,38 @@ +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: csinodeinfos.csi.storage.k8s.io +spec: + group: csi.storage.k8s.io + names: + kind: CSINodeInfo + plural: csinodeinfos + scope: Cluster + validation: + openAPIV3Schema: + properties: + csiDrivers: + description: List of CSI drivers running on the node and their properties. + items: + properties: + driver: + description: The CSI driver that this object refers to. + type: string + nodeID: + description: The node from the driver point of view. + type: string + topologyKeys: + description: List of keys supported by the driver. + items: + type: string + type: array + type: array + version: v1alpha1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index 8e5758f7e..34fd366a3 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -45,6 +45,14 @@ dependencies: - aws-ebs-csi-driver - csi-driver + - role: kubernetes-apps/csi_driver/azuredisk + when: + - azure_csi_enabled + tags: + - apps + - azure-csi-driver + - csi-driver + - role: kubernetes-apps/csi_driver/gcp_pd when: - gcp_pd_csi_enabled diff --git a/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml b/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml new file mode 100644 index 000000000..fc92e17b5 --- /dev/null +++ b/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml @@ -0,0 +1,3 @@ +--- +## Available values: Standard_LRS, Premium_LRS, StandardSSD_LRS, UltraSSD_LRS +storage_account_type: StandardSSD_LRS diff --git a/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml b/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml new file mode 100644 index 000000000..8a35dbf61 --- /dev/null +++ b/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml @@ -0,0 +1,19 @@ +--- +- name: Kubernetes Persistent Volumes | Copy Azure CSI Storage Class template + template: + src: "azure-csi-storage-class.yml.j2" + dest: "{{ kube_config_dir }}/azure-csi-storage-class.yml" + register: manifests + when: + - inventory_hostname == groups['kube-master'][0] + +- name: Kubernetes Persistent Volumes | Add Azure CSI Storage Class + kube: + name: cinder-csi + kubectl: "{{ bin_dir }}/kubectl" + resource: StorageClass + filename: "{{ kube_config_dir }}/azure-csi-storage-class.yml" + state: "latest" + when: + - inventory_hostname == groups['kube-master'][0] + - manifests.changed diff --git a/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2 b/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2 new file mode 100644 index 000000000..80f02b3db --- /dev/null +++ b/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2 @@ -0,0 +1,11 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: disk.csi.azure.com +provisioner: disk.csi.azure.com +parameters: + skuname: {{ storage_account_type }} +reclaimPolicy: Delete +volumeBindingMode: Immediate +allowVolumeExpansion: true diff --git a/roles/kubernetes-apps/persistent_volumes/meta/main.yml b/roles/kubernetes-apps/persistent_volumes/meta/main.yml index 42096aa7b..c0522df9c 100644 --- a/roles/kubernetes-apps/persistent_volumes/meta/main.yml +++ b/roles/kubernetes-apps/persistent_volumes/meta/main.yml @@ -21,6 +21,13 @@ dependencies: - persistent_volumes_aws_ebs_csi - aws-ebs-csi-driver + - role: kubernetes-apps/persistent_volumes/azuredisk-csi + when: + - azure_csi_enabled + tags: + - persistent_volumes_azure_csi + - azure-csi-driver + - role: kubernetes-apps/persistent_volumes/gcp-pd-csi when: - gcp_pd_csi_enabled diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index e2ada4e9c..6c05d7152 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -305,6 +305,7 @@ local_volume_provisioner_enabled: "{{ local_volumes_enabled | default('false') } local_volume_provisioner_directory_mode: 0700 cinder_csi_enabled: false aws_ebs_csi_enabled: false +azure_csi_enabled: false gcp_pd_csi_enabled: false persistent_volumes_enabled: false cephfs_provisioner_enabled: false