AWS EBS CSI implementation (#5549)

* AWS EBS CSI implementation

* Fixing image repos

* Add OWNERS file

* Fix expressions

* Add csi-driver tag

* Add AWS EBS prefix to variables

* Add AWS EBS CSI Driver documentation

docs/aws-ebs-csi.md

@@ -0,0 +1,87 @@
+# AWS EBS CSI Driver
+
+AWS EBS CSI driver allows you to provision EBS volumes for pods in EC2 instances. The old in-tree AWS cloud provider is deprecated and will be removed in future versions of Kubernetes. So transitioning to the CSI driver is advised.
+
+To enable AWS EBS CSI driver, uncomment the `aws_ebs_csi_enabled` option in `group_vars/all/aws.yml` and set it to `true`.
+
+To set the number of replicas for the AWS CSI controller, you can change `aws_ebs_csi_controller_replicas` option in `group_vars/all/aws.yml`.
+
+Make sure to add a role, for your EC2 instances hosting Kubernetes, that allows it to do the actions necessary to request a volume and attach it: [AWS CSI Policy](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/example-iam-policy.json)
+
+If you want to deploy the AWS EBS storage class used with the CSI Driver, you should set `persistent_volumes_enabled` in `group_vars/k8s-cluster/k8s-cluster.yml` to `true`.
+
+You can now run the kubespray playbook (cluster.yml) to deploy Kubernetes over AWS EC2 with EBS CSI Driver enabled.
+
+## Usage example
+
+To check if AWS EBS CSI Driver is deployed properly, check that the ebs-csi pods are running:
+
+```ShellSession
+$ kubectl -n kube-system get pods | grep ebs
+ebs-csi-controller-85d86bccc5-8gtq5                                  4/4     Running   4          40s
+ebs-csi-node-n4b99                                                   3/3     Running   3          40s
+```
+
+Check the associated storage class (if you enabled persistent_volumes):
+
+```ShellSession
+$ kubectl get storageclass
+NAME         PROVISIONER                AGE
+ebs-sc   ebs.csi.aws.com   45s
+```
+
+You can run a PVC and an example Pod using this file `ebs-pod.yml`:
+
+```yml
+--
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ebs-claim
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: ebs-sc
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: app
+spec:
+  containers:
+  - name: app
+    image: centos
+    command: ["/bin/sh"]
+    args: ["-c", "while true; do echo $(date -u) >> /data/out.txt; sleep 5; done"]
+    volumeMounts:
+    - name: persistent-storage
+      mountPath: /data
+  volumes:
+  - name: persistent-storage
+    persistentVolumeClaim:
+      claimName: ebs-claim
+```
+
+Apply this conf to your cluster: ```kubectl apply -f ebs-pod.yml```
+
+You should see the PVC provisioned and bound:
+
+```ShellSession
+$ kubectl get pvc
+NAME                   STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
+ebs-claim   Bound    pvc-0034cb9e-1ddd-4b3f-bb9e-0b5edbf5194c   1Gi        RWO            ebs-sc         50s
+```
+
+And the volume mounted to the example Pod (wait until the Pod is Running):
+
+```ShellSession
+$ kubectl exec -it app -- df -h | grep data
+/dev/nvme1n1   1014M   34M  981M   4% /data
+```
+
+## More info
+
+For further information about the AWS EBS CSI Driver, you can refer to this page: [AWS EBS Driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/).

inventory/sample/group_vars/all/aws.yml

@@ -0,0 +1,8 @@
+## To use AWS EBS CSI Driver to provision volumes, uncomment the first value
+## and configure the parameters below
+# aws_ebs_csi_enabled: true
+# aws_ebs_csi_enable_volume_scheduling: true
+# aws_ebs_csi_enable_volume_snapshot: false
+# aws_ebs_csi_enable_volume_resizing: false
+# aws_ebs_csi_controller_replicas: 1
+# aws_ebs_csi_plugin_image_tag: latest

inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml

@@ -255,7 +255,7 @@ podsecuritypolicy_enabled: false
 ## See https://github.com/kubernetes-sigs/kubespray/issues/2141
 ## Set this variable to true to get rid of this issue
 volume_cross_zone_attachment: false
-# Add Persistent Volumes Storage Class for corresponding cloud provider ( OpenStack is only supported now )
+# Add Persistent Volumes Storage Class for corresponding cloud provider (supported: in-tree OpenStack, Cinder CSI, AWS EBS CSI)
 persistent_volumes_enabled: false
 
 ## Container Engine Acceleration

roles/download/defaults/main.yml

@@ -494,18 +494,24 @@ addon_resizer_version: "1.8.8"
 addon_resizer_image_repo: "{{ kube_image_repo }}/addon-resizer"
 addon_resizer_image_tag: "{{ addon_resizer_version }}"
 
-cinder_csi_attacher_image_repo: "{{ quay_image_repo }}/k8scsi/csi-attacher"
-cinder_csi_attacher_image_tag: "v1.2.1"
-cinder_csi_provisioner_image_repo: "{{ quay_image_repo }}/k8scsi/csi-provisioner"
-cinder_csi_provisioner_image_tag: "v1.3.0"
-cinder_csi_snapshotter_image_repo: "{{ quay_image_repo }}/k8scsi/csi-snapshotter"
-cinder_csi_snapshotter_image_tag: "v1.2.0"
-cinder_csi_resizer_image_repo: "{{ quay_image_repo }}/k8scsi/csi-resizer"
-cinder_csi_resizer_image_tag: "v0.2.0"
+csi_attacher_image_repo: "{{ quay_image_repo }}/k8scsi/csi-attacher"
+csi_attacher_image_tag: "v1.2.1"
+csi_provisioner_image_repo: "{{ quay_image_repo }}/k8scsi/csi-provisioner"
+csi_provisioner_image_tag: "v1.3.0"
+csi_snapshotter_image_repo: "{{ quay_image_repo }}/k8scsi/csi-snapshotter"
+csi_snapshotter_image_tag: "v1.2.0"
+csi_resizer_image_repo: "{{ quay_image_repo }}/k8scsi/csi-resizer"
+csi_resizer_image_tag: "v0.2.0"
+csi_node_driver_registrar_image_repo: "{{ quay_image_repo }}/k8scsi/csi-node-driver-registrar"
+csi_node_driver_registrar_image_tag: "v1.1.0"
+csi_livenessprobe_image_repo: "{{ quay_image_repo }}/k8scsi/livenessprobe"
+csi_livenessprobe_image_tag: "v1.1.0"
+
 cinder_csi_plugin_image_repo: "{{ docker_image_repo }}/k8scloudprovider/cinder-csi-plugin"
 cinder_csi_plugin_image_tag: "latest"
-cinder_csi_node_driver_registrar_image_repo: "{{ quay_image_repo }}/k8scsi/csi-node-driver-registrar"
-cinder_csi_node_driver_registrar_image_tag: "v1.1.0"
+
+aws_ebs_csi_plugin_image_repo: "{{ docker_image_repo }}/amazon/aws-ebs-csi-driver"
+aws_ebs_csi_plugin_image_tag: "latest"
 
 dashboard_image_repo: "{{ gcr_image_repo }}/google_containers/kubernetes-dashboard-{{ image_arch }}"
 dashboard_image_tag: "v1.10.1"
@@ -1011,39 +1017,48 @@ downloads:
     groups:
       - kube-node
 
-  cinder_csi_attacher:
-    enabled: "{{ cinder_csi_enabled }}"
+  csi_attacher:
+    enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}"
     container: true
-    repo: "{{ cinder_csi_attacher_image_repo }}"
-    tag: "{{ cinder_csi_attacher_image_tag }}"
-    sha256: "{{ cinder_csi_attacher_digest_checksum|default(None) }}"
+    repo: "{{ csi_attacher_image_repo }}"
+    tag: "{{ csi_attacher_image_tag }}"
+    sha256: "{{ csi_attacher_digest_checksum|default(None) }}"
     groups:
       - kube-node
 
-  cinder_csi_provisioner:
-    enabled: "{{ cinder_csi_enabled }}"
+  csi_provisioner:
+    enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}"
     container: true
-    repo: "{{ cinder_csi_provisioner_image_repo }}"
-    tag: "{{ cinder_csi_provisioner_image_tag }}"
-    sha256: "{{ cinder_csi_provisioner_digest_checksum|default(None) }}"
+    repo: "{{ csi_provisioner_image_repo }}"
+    tag: "{{ csi_provisioner_image_tag }}"
+    sha256: "{{ csi_provisioner_digest_checksum|default(None) }}"
     groups:
       - kube-node
 
-  cinder_csi_snapshotter:
-    enabled: "{{ cinder_csi_enabled }}"
+  csi_snapshotter:
+    enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}"
     container: true
-    repo: "{{ cinder_csi_snapshotter_image_repo }}"
-    tag: "{{ cinder_csi_snapshotter_image_tag }}"
-    sha256: "{{ cinder_csi_snapshotter_digest_checksum|default(None) }}"
+    repo: "{{ csi_snapshotter_image_repo }}"
+    tag: "{{ csi_snapshotter_image_tag }}"
+    sha256: "{{ csi_snapshotter_digest_checksum|default(None) }}"
     groups:
       - kube-node
 
-  cinder_csi_resizer:
-    enabled: "{{ cinder_csi_enabled }}"
+  csi_resizer:
+    enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}"
     container: true
-    repo: "{{ cinder_csi_resizer_image_repo }}"
-    tag: "{{ cinder_csi_resizer_image_tag }}"
-    sha256: "{{ cinder_csi_resizer_digest_checksum|default(None) }}"
+    repo: "{{ csi_resizer_image_repo }}"
+    tag: "{{ csi_resizer_image_tag }}"
+    sha256: "{{ csi_resizer_digest_checksum|default(None) }}"
+    groups:
+      - kube-node
+
+  csi_node_driver_registrar:
+    enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}"
+    container: true
+    repo: "{{ csi_node_driver_registrar_image_repo }}"
+    tag: "{{ csi_node_driver_registrar_image_tag }}"
+    sha256: "{{ csi_node_driver_registrar_digest_checksum|default(None) }}"
     groups:
       - kube-node
 
@@ -1056,12 +1071,12 @@ downloads:
     groups:
       - kube-node
 
-  cinder_csi_node_driver_registrar:
-    enabled: "{{ cinder_csi_enabled }}"
+  aws_ebs_csi_plugin:
+    enabled: "{{ aws_ebs_csi_enabled }}"
     container: true
-    repo: "{{ cinder_csi_node_driver_registrar_image_repo }}"
-    tag: "{{ cinder_csi_node_driver_registrar_image_tag }}"
-    sha256: "{{ cinder_csi_node_driver_registrar_digest_checksum|default(None) }}"
+    repo: "{{ aws_ebs_csi_plugin_image_repo }}"
+    tag: "{{ aws_ebs_csi_plugin_image_tag }}"
+    sha256: "{{ aws_ebs_csi_plugin_digest_checksum|default(None) }}"
     groups:
       - kube-node
 

roles/kubernetes-apps/csi_driver/aws_ebs/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+aws_ebs_csi_enable_volume_scheduling: true
+aws_ebs_csi_enable_volume_snapshot: false
+aws_ebs_csi_enable_volume_resizing: false
+aws_ebs_csi_controller_replicas: 1
+aws_ebs_csi_plugin_image_tag: latest

roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: AWS CSI Driver | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  with_items:
+    - {name: aws-ebs-csi-driver, file: aws-ebs-csi-driver.yml}
+    - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice-rbac.yml}
+    - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice.yml}
+    - {name: aws-ebs-csi-nodeservice, file: aws-ebs-csi-nodeservice.yml}
+  register: aws_csi_manifests
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: aws-ebs-csi-driver
+
+- name: AWS CSI Driver | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ aws_csi_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+  tags: aws-ebs-csi-driver

roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice-rbac.yml.j2

@@ -0,0 +1,179 @@
+# Controller Service
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ebs-csi-controller-sa
+  namespace: kube-system
+
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-provisioner-binding
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-provisioner-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-attacher-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["csi.storage.k8s.io"]
+    resources: ["csinodeinfos"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update"]
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-attacher-binding
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-attacher-role
+  apiGroup: rbac.authorization.k8s.io
+
+{% if aws_ebs_csi_enable_volume_snapshot %}
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-snapshotter-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources: ["customresourcedefinitions"]
+    verbs: ["create", "list", "watch", "delete"]
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-snapshotter-binding
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-snapshotter-role
+  apiGroup: rbac.authorization.k8s.io
+
+{% endif %}
+
+{% if aws_ebs_csi_enable_volume_resizing %}
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-external-resizer-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: ebs-csi-resizer-binding
+subjects:
+  - kind: ServiceAccount
+    name: ebs-csi-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: ebs-external-resizer-role
+  apiGroup: rbac.authorization.k8s.io
+
+{% endif %}

roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2

@@ -0,0 +1,127 @@
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: ebs-csi-controller
+  namespace: kube-system
+spec:
+  replicas: {{ aws_ebs_csi_controller_replicas }}
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+      app.kubernetes.io/name: aws-ebs-csi-driver
+  template:
+    metadata:
+      labels:
+        app: ebs-csi-controller
+        app.kubernetes.io/name: aws-ebs-csi-driver
+    spec:
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      serviceAccount: ebs-csi-controller-sa
+      priorityClassName: system-cluster-critical
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+      containers:
+        - name: ebs-plugin
+          image: {{ aws_ebs_csi_plugin_image_repo }}:{{ aws_ebs_csi_plugin_image_tag }}
+          args:
+            - --endpoint=$(CSI_ENDPOINT)
+            - --logtostderr
+            - --v=5
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: aws-secret
+                  key: key_id
+                  optional: true
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: aws-secret
+                  key: access_key
+                  optional: true
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
+        - name: csi-provisioner
+          image: {{ csi_provisioner_image_repo }}:{{ csi_provisioner_image_tag }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v=5
+{% if aws_ebs_csi_enable_volume_scheduling %}
+            - --feature-gates=Topology=true
+{% endif %}
+            - --enable-leader-election
+            - --leader-election-type=leases
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-attacher
+          image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v=5
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+{% if aws_ebs_csi_enable_volume_snapshot %}
+        - name: csi-snapshotter
+          image: {{ csi_snapshotter_image_repo }}:{{ csi_snapshotter_image_tag }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --connection-timeout=15s
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+{% endif %}
+{% if aws_ebs_csi_enable_volume_resizing %}
+        - name: csi-resizer
+          image: {{ csi_resizer_image_repo }}:{{ csi_resizer_image_tag }}
+          imagePullPolicy: Always
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v=5
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+{% endif %}
+        - name: liveness-probe
+          image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }}
+          args:
+            - --csi-address=/csi/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+

roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-driver.yml.j2

@@ -0,0 +1,8 @@
+---
+apiVersion: storage.k8s.io/v1beta1
+kind: CSIDriver
+metadata:
+  name: ebs.csi.aws.com
+spec:
+  attachRequired: true
+  podInfoOnMount: false

roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-nodeservice.yml.j2

@@ -0,0 +1,101 @@
+---
+# Node Service
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: ebs-csi-node
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: ebs-csi-node
+      app.kubernetes.io/name: aws-ebs-csi-driver
+  template:
+    metadata:
+      labels:
+        app: ebs-csi-node
+        app.kubernetes.io/name: aws-ebs-csi-driver
+    spec:
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+      containers:
+        - name: ebs-plugin
+          securityContext:
+            privileged: true
+          image: {{ aws_ebs_csi_plugin_image_repo }}:{{ aws_ebs_csi_plugin_image_tag }}
+          args:
+            - --endpoint=$(CSI_ENDPOINT)
+            - --logtostderr
+            - --v=5
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:/csi/csi.sock
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: plugin-dir
+              mountPath: /csi
+            - name: device-dir
+              mountPath: /dev
+          ports:
+            - name: healthz
+              containerPort: 9808
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 10
+            failureThreshold: 5
+        - name: node-driver-registrar
+          image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+            - --v=5
+          lifecycle:
+            preStop:
+              exec:
+                command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"]
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: liveness-probe
+          image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }}
+          args:
+            - --csi-address=/csi/csi.sock
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+      volumes:
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/ebs.csi.aws.com/
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: device-dir
+          hostPath:
+            path: /dev
+            type: Directory

roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2

@@ -20,7 +20,7 @@ spec:
       serviceAccount: csi-cinder-controller-sa
       containers:
         - name: csi-attacher
-          image: {{ cinder_csi_attacher_image_repo }}:{{ cinder_csi_attacher_image_tag }}
+          image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }}
           args:
             - "--v=5"
             - "--csi-address=$(ADDRESS)"
@@ -37,7 +37,7 @@ spec:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
         - name: csi-provisioner
-          image: {{ cinder_csi_provisioner_image_repo }}:{{ cinder_csi_provisioner_image_tag }}
+          image: {{ csi_provisioner_image_repo }}:{{ csi_provisioner_image_tag }}
           args:
             - "--csi-address=$(ADDRESS)"
 {% if cinder_topology is defined and cinder_topology %}
@@ -56,7 +56,7 @@ spec:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
         - name: csi-snapshotter
-          image: {{ cinder_csi_snapshotter_image_repo }}:{{ cinder_csi_snapshotter_image_tag }}
+          image: {{ csi_snapshotter_image_repo }}:{{ csi_snapshotter_image_tag }}
           args:
             - "--csi-address=$(ADDRESS)"
 {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}
@@ -71,7 +71,7 @@ spec:
             - mountPath: /var/lib/csi/sockets/pluginproxy/
               name: socket-dir
         - name: csi-resizer
-          image: {{ cinder_csi_resizer_image_repo }}:{{ cinder_csi_resizer_image_tag }}
+          image: {{ csi_resizer_image_repo }}:{{ csi_resizer_image_tag }}
           args:
             - "--csi-address=$(ADDRESS)"
 {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %}

roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2

@@ -19,7 +19,7 @@ spec:
       hostNetwork: true
       containers:
         - name: node-driver-registrar
-          image: {{ cinder_csi_node_driver_registrar_image_repo }}:{{ cinder_csi_node_driver_registrar_image_tag }}
+          image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }}
           args:
             - "--csi-address=$(ADDRESS)"
             - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"

roles/kubernetes-apps/meta/main.yml

@@ -37,6 +37,14 @@ dependencies:
       - cinder-csi-driver
       - csi-driver
 
+  - role: kubernetes-apps/csi_driver/aws_ebs
+    when:
+      - aws_ebs_csi_enabled
+    tags:
+      - apps
+      - aws-ebs-csi-driver
+      - csi-driver
+
   - role: kubernetes-apps/persistent_volumes
     when:
       - persistent_volumes_enabled

roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/OWNERS

@@ -0,0 +1,5 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+  - alijahnas
+reviewers:

roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/defaults/main.yml

@@ -0,0 +1,8 @@
+---
+# To restrict which AZ the volume should be provisioned in
+# set this value to true and set the list of relevant AZs
+# For it to work, the flag aws_ebs_csi_enable_volume_scheduling
+# in AWS EBS Driver must be true
+restrict_az_provisioning: false
+aws_ebs_availability_zones:
+  - eu-west-3c

roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/tasks/main.yml

@@ -0,0 +1,19 @@
+---
+- name: Kubernetes Persistent Volumes | Copy AWS EBS CSI Storage Class template
+  template:
+    src: "aws-ebs-csi-storage-class.yml.j2"
+    dest: "{{ kube_config_dir }}/aws-ebs-csi-storage-class.yml"
+  register: manifests
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: Kubernetes Persistent Volumes | Add AWS EBS CSI Storage Class
+  kube:
+    name: aws-ebs-csi
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: StorageClass
+    filename: "{{ kube_config_dir }}/aws-ebs-csi-storage-class.yml"
+    state: "latest"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - manifests.changed

roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/templates/aws-ebs-csi-storage-class.yml.j2

@@ -0,0 +1,18 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: ebs-sc
+provisioner: ebs.csi.aws.com
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  csi.storage.k8s.io/fstype: xfs
+  type: gp2
+{% if restrict_az_provisioning %}
+allowedTopologies:
+- matchLabelExpressions:
+  - key: topology.ebs.csi.aws.com/zone
+    values:
+{% for value in aws_ebs_availability_zones %}
+      - {{ value }}
+{% endfor %}
+{% endif %}

roles/kubernetes-apps/persistent_volumes/meta/main.yml

@@ -13,3 +13,10 @@ dependencies:
     tags:
       - persistent_volumes_cinder_csi
       - cinder-csi-driver
+
+  - role: kubernetes-apps/persistent_volumes/aws-ebs-csi
+    when:
+      - aws_ebs_csi_enabled
+    tags:
+      - persistent_volumes_aws_ebs_csi
+      - aws-ebs-csi-driver

roles/kubespray-defaults/defaults/main.yaml

@@ -304,6 +304,7 @@ enable_network_policy: true
 local_volume_provisioner_enabled: "{{ local_volumes_enabled | default('false') }}"
 local_volume_provisioner_directory_mode: 0700
 cinder_csi_enabled: false
+aws_ebs_csi_enabled: false
 persistent_volumes_enabled: false
 cephfs_provisioner_enabled: false
 rbd_provisioner_enabled: false