From a8a05a21a4f7d79f786b76f4e993658ed7861929 Mon Sep 17 00:00:00 2001 From: Ali Sanhaji Date: Wed, 25 Mar 2020 21:10:25 +0100 Subject: [PATCH] AWS EBS CSI implementation (#5549) * AWS EBS CSI implementation * Fixing image repos * Add OWNERS file * Fix expressions * Add csi-driver tag * Add AWS EBS prefix to variables * Add AWS EBS CSI Driver documentation --- docs/aws-ebs-csi.md | 87 +++++++++ inventory/sample/group_vars/all/aws.yml | 8 + .../group_vars/k8s-cluster/k8s-cluster.yml | 2 +- roles/download/defaults/main.yml | 85 +++++---- .../csi_driver/aws_ebs/defaults/main.yml | 6 + .../csi_driver/aws_ebs/tasks/main.yml | 27 +++ .../aws-ebs-csi-controllerservice-rbac.yml.j2 | 179 ++++++++++++++++++ .../aws-ebs-csi-controllerservice.yml.j2 | 127 +++++++++++++ .../templates/aws-ebs-csi-driver.yml.j2 | 8 + .../templates/aws-ebs-csi-nodeservice.yml.j2 | 101 ++++++++++ .../cinder-csi-controllerplugin.yml.j2 | 8 +- .../templates/cinder-csi-nodeplugin.yml.j2 | 2 +- roles/kubernetes-apps/meta/main.yml | 8 + .../persistent_volumes/aws-ebs-csi/OWNERS | 5 + .../aws-ebs-csi/defaults/main.yml | 8 + .../aws-ebs-csi/tasks/main.yml | 19 ++ .../aws-ebs-csi-storage-class.yml.j2 | 18 ++ .../persistent_volumes/meta/main.yml | 7 + roles/kubespray-defaults/defaults/main.yaml | 1 + 19 files changed, 665 insertions(+), 41 deletions(-) create mode 100644 docs/aws-ebs-csi.md create mode 100644 inventory/sample/group_vars/all/aws.yml create mode 100644 roles/kubernetes-apps/csi_driver/aws_ebs/defaults/main.yml create mode 100644 roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml create mode 100644 roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice-rbac.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-driver.yml.j2 create mode 100644 roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-nodeservice.yml.j2 create mode 100644 roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/OWNERS create mode 100644 roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/defaults/main.yml create mode 100644 roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/tasks/main.yml create mode 100644 roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/templates/aws-ebs-csi-storage-class.yml.j2 diff --git a/docs/aws-ebs-csi.md b/docs/aws-ebs-csi.md new file mode 100644 index 000000000..4d8c96311 --- /dev/null +++ b/docs/aws-ebs-csi.md @@ -0,0 +1,87 @@ +# AWS EBS CSI Driver + +AWS EBS CSI driver allows you to provision EBS volumes for pods in EC2 instances. The old in-tree AWS cloud provider is deprecated and will be removed in future versions of Kubernetes. So transitioning to the CSI driver is advised. + +To enable AWS EBS CSI driver, uncomment the `aws_ebs_csi_enabled` option in `group_vars/all/aws.yml` and set it to `true`. + +To set the number of replicas for the AWS CSI controller, you can change `aws_ebs_csi_controller_replicas` option in `group_vars/all/aws.yml`. + +Make sure to add a role, for your EC2 instances hosting Kubernetes, that allows it to do the actions necessary to request a volume and attach it: [AWS CSI Policy](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/example-iam-policy.json) + +If you want to deploy the AWS EBS storage class used with the CSI Driver, you should set `persistent_volumes_enabled` in `group_vars/k8s-cluster/k8s-cluster.yml` to `true`. + +You can now run the kubespray playbook (cluster.yml) to deploy Kubernetes over AWS EC2 with EBS CSI Driver enabled. + +## Usage example + +To check if AWS EBS CSI Driver is deployed properly, check that the ebs-csi pods are running: + +```ShellSession +$ kubectl -n kube-system get pods | grep ebs +ebs-csi-controller-85d86bccc5-8gtq5 4/4 Running 4 40s +ebs-csi-node-n4b99 3/3 Running 3 40s +``` + +Check the associated storage class (if you enabled persistent_volumes): + +```ShellSession +$ kubectl get storageclass +NAME PROVISIONER AGE +ebs-sc ebs.csi.aws.com 45s +``` + +You can run a PVC and an example Pod using this file `ebs-pod.yml`: + +```yml +-- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ebs-claim +spec: + accessModes: + - ReadWriteOnce + storageClassName: ebs-sc + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Pod +metadata: + name: app +spec: + containers: + - name: app + image: centos + command: ["/bin/sh"] + args: ["-c", "while true; do echo $(date -u) >> /data/out.txt; sleep 5; done"] + volumeMounts: + - name: persistent-storage + mountPath: /data + volumes: + - name: persistent-storage + persistentVolumeClaim: + claimName: ebs-claim +``` + +Apply this conf to your cluster: ```kubectl apply -f ebs-pod.yml``` + +You should see the PVC provisioned and bound: + +```ShellSession +$ kubectl get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +ebs-claim Bound pvc-0034cb9e-1ddd-4b3f-bb9e-0b5edbf5194c 1Gi RWO ebs-sc 50s +``` + +And the volume mounted to the example Pod (wait until the Pod is Running): + +```ShellSession +$ kubectl exec -it app -- df -h | grep data +/dev/nvme1n1 1014M 34M 981M 4% /data +``` + +## More info + +For further information about the AWS EBS CSI Driver, you can refer to this page: [AWS EBS Driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/). diff --git a/inventory/sample/group_vars/all/aws.yml b/inventory/sample/group_vars/all/aws.yml new file mode 100644 index 000000000..ffaac3592 --- /dev/null +++ b/inventory/sample/group_vars/all/aws.yml @@ -0,0 +1,8 @@ +## To use AWS EBS CSI Driver to provision volumes, uncomment the first value +## and configure the parameters below +# aws_ebs_csi_enabled: true +# aws_ebs_csi_enable_volume_scheduling: true +# aws_ebs_csi_enable_volume_snapshot: false +# aws_ebs_csi_enable_volume_resizing: false +# aws_ebs_csi_controller_replicas: 1 +# aws_ebs_csi_plugin_image_tag: latest diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index 7c64707c5..38040a7ee 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -255,7 +255,7 @@ podsecuritypolicy_enabled: false ## See https://github.com/kubernetes-sigs/kubespray/issues/2141 ## Set this variable to true to get rid of this issue volume_cross_zone_attachment: false -# Add Persistent Volumes Storage Class for corresponding cloud provider ( OpenStack is only supported now ) +# Add Persistent Volumes Storage Class for corresponding cloud provider (supported: in-tree OpenStack, Cinder CSI, AWS EBS CSI) persistent_volumes_enabled: false ## Container Engine Acceleration diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 854ff2ea8..554b7352c 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -494,18 +494,24 @@ addon_resizer_version: "1.8.8" addon_resizer_image_repo: "{{ kube_image_repo }}/addon-resizer" addon_resizer_image_tag: "{{ addon_resizer_version }}" -cinder_csi_attacher_image_repo: "{{ quay_image_repo }}/k8scsi/csi-attacher" -cinder_csi_attacher_image_tag: "v1.2.1" -cinder_csi_provisioner_image_repo: "{{ quay_image_repo }}/k8scsi/csi-provisioner" -cinder_csi_provisioner_image_tag: "v1.3.0" -cinder_csi_snapshotter_image_repo: "{{ quay_image_repo }}/k8scsi/csi-snapshotter" -cinder_csi_snapshotter_image_tag: "v1.2.0" -cinder_csi_resizer_image_repo: "{{ quay_image_repo }}/k8scsi/csi-resizer" -cinder_csi_resizer_image_tag: "v0.2.0" +csi_attacher_image_repo: "{{ quay_image_repo }}/k8scsi/csi-attacher" +csi_attacher_image_tag: "v1.2.1" +csi_provisioner_image_repo: "{{ quay_image_repo }}/k8scsi/csi-provisioner" +csi_provisioner_image_tag: "v1.3.0" +csi_snapshotter_image_repo: "{{ quay_image_repo }}/k8scsi/csi-snapshotter" +csi_snapshotter_image_tag: "v1.2.0" +csi_resizer_image_repo: "{{ quay_image_repo }}/k8scsi/csi-resizer" +csi_resizer_image_tag: "v0.2.0" +csi_node_driver_registrar_image_repo: "{{ quay_image_repo }}/k8scsi/csi-node-driver-registrar" +csi_node_driver_registrar_image_tag: "v1.1.0" +csi_livenessprobe_image_repo: "{{ quay_image_repo }}/k8scsi/livenessprobe" +csi_livenessprobe_image_tag: "v1.1.0" + cinder_csi_plugin_image_repo: "{{ docker_image_repo }}/k8scloudprovider/cinder-csi-plugin" cinder_csi_plugin_image_tag: "latest" -cinder_csi_node_driver_registrar_image_repo: "{{ quay_image_repo }}/k8scsi/csi-node-driver-registrar" -cinder_csi_node_driver_registrar_image_tag: "v1.1.0" + +aws_ebs_csi_plugin_image_repo: "{{ docker_image_repo }}/amazon/aws-ebs-csi-driver" +aws_ebs_csi_plugin_image_tag: "latest" dashboard_image_repo: "{{ gcr_image_repo }}/google_containers/kubernetes-dashboard-{{ image_arch }}" dashboard_image_tag: "v1.10.1" @@ -1011,39 +1017,48 @@ downloads: groups: - kube-node - cinder_csi_attacher: - enabled: "{{ cinder_csi_enabled }}" + csi_attacher: + enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" container: true - repo: "{{ cinder_csi_attacher_image_repo }}" - tag: "{{ cinder_csi_attacher_image_tag }}" - sha256: "{{ cinder_csi_attacher_digest_checksum|default(None) }}" + repo: "{{ csi_attacher_image_repo }}" + tag: "{{ csi_attacher_image_tag }}" + sha256: "{{ csi_attacher_digest_checksum|default(None) }}" groups: - kube-node - cinder_csi_provisioner: - enabled: "{{ cinder_csi_enabled }}" + csi_provisioner: + enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" container: true - repo: "{{ cinder_csi_provisioner_image_repo }}" - tag: "{{ cinder_csi_provisioner_image_tag }}" - sha256: "{{ cinder_csi_provisioner_digest_checksum|default(None) }}" + repo: "{{ csi_provisioner_image_repo }}" + tag: "{{ csi_provisioner_image_tag }}" + sha256: "{{ csi_provisioner_digest_checksum|default(None) }}" groups: - kube-node - cinder_csi_snapshotter: - enabled: "{{ cinder_csi_enabled }}" + csi_snapshotter: + enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" container: true - repo: "{{ cinder_csi_snapshotter_image_repo }}" - tag: "{{ cinder_csi_snapshotter_image_tag }}" - sha256: "{{ cinder_csi_snapshotter_digest_checksum|default(None) }}" + repo: "{{ csi_snapshotter_image_repo }}" + tag: "{{ csi_snapshotter_image_tag }}" + sha256: "{{ csi_snapshotter_digest_checksum|default(None) }}" groups: - kube-node - cinder_csi_resizer: - enabled: "{{ cinder_csi_enabled }}" + csi_resizer: + enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" container: true - repo: "{{ cinder_csi_resizer_image_repo }}" - tag: "{{ cinder_csi_resizer_image_tag }}" - sha256: "{{ cinder_csi_resizer_digest_checksum|default(None) }}" + repo: "{{ csi_resizer_image_repo }}" + tag: "{{ csi_resizer_image_tag }}" + sha256: "{{ csi_resizer_digest_checksum|default(None) }}" + groups: + - kube-node + + csi_node_driver_registrar: + enabled: "{{ cinder_csi_enabled or aws_ebs_csi_enabled }}" + container: true + repo: "{{ csi_node_driver_registrar_image_repo }}" + tag: "{{ csi_node_driver_registrar_image_tag }}" + sha256: "{{ csi_node_driver_registrar_digest_checksum|default(None) }}" groups: - kube-node @@ -1056,12 +1071,12 @@ downloads: groups: - kube-node - cinder_csi_node_driver_registrar: - enabled: "{{ cinder_csi_enabled }}" + aws_ebs_csi_plugin: + enabled: "{{ aws_ebs_csi_enabled }}" container: true - repo: "{{ cinder_csi_node_driver_registrar_image_repo }}" - tag: "{{ cinder_csi_node_driver_registrar_image_tag }}" - sha256: "{{ cinder_csi_node_driver_registrar_digest_checksum|default(None) }}" + repo: "{{ aws_ebs_csi_plugin_image_repo }}" + tag: "{{ aws_ebs_csi_plugin_image_tag }}" + sha256: "{{ aws_ebs_csi_plugin_digest_checksum|default(None) }}" groups: - kube-node diff --git a/roles/kubernetes-apps/csi_driver/aws_ebs/defaults/main.yml b/roles/kubernetes-apps/csi_driver/aws_ebs/defaults/main.yml new file mode 100644 index 000000000..92751964d --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/aws_ebs/defaults/main.yml @@ -0,0 +1,6 @@ +--- +aws_ebs_csi_enable_volume_scheduling: true +aws_ebs_csi_enable_volume_snapshot: false +aws_ebs_csi_enable_volume_resizing: false +aws_ebs_csi_controller_replicas: 1 +aws_ebs_csi_plugin_image_tag: latest diff --git a/roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml b/roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml new file mode 100644 index 000000000..04bb9fd4d --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/aws_ebs/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- name: AWS CSI Driver | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - {name: aws-ebs-csi-driver, file: aws-ebs-csi-driver.yml} + - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice-rbac.yml} + - {name: aws-ebs-csi-controllerservice, file: aws-ebs-csi-controllerservice.yml} + - {name: aws-ebs-csi-nodeservice, file: aws-ebs-csi-nodeservice.yml} + register: aws_csi_manifests + when: inventory_hostname == groups['kube-master'][0] + tags: aws-ebs-csi-driver + +- name: AWS CSI Driver | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ aws_csi_manifests.results }}" + when: + - inventory_hostname == groups['kube-master'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + tags: aws-ebs-csi-driver diff --git a/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice-rbac.yml.j2 b/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice-rbac.yml.j2 new file mode 100644 index 000000000..e6fe07af8 --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice-rbac.yml.j2 @@ -0,0 +1,179 @@ +# Controller Service +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ebs-csi-controller-sa + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-attacher-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +{% if aws_ebs_csi_enable_volume_snapshot %} +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-snapshotter-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +{% endif %} + +{% if aws_ebs_csi_enable_volume_resizing %} +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-external-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: ebs-csi-resizer-binding +subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: ebs-external-resizer-role + apiGroup: rbac.authorization.k8s.io + +{% endif %} diff --git a/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2 b/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2 new file mode 100644 index 000000000..e807c202b --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-controllerservice.yml.j2 @@ -0,0 +1,127 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ebs-csi-controller + namespace: kube-system +spec: + replicas: {{ aws_ebs_csi_controller_replicas }} + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + nodeSelector: + beta.kubernetes.io/os: linux + serviceAccount: ebs-csi-controller-sa + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: ebs-plugin + image: {{ aws_ebs_csi_plugin_image_repo }}:{{ aws_ebs_csi_plugin_image_tag }} + args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-secret + key: key_id + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-secret + key: access_key + optional: true + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + - name: csi-provisioner + image: {{ csi_provisioner_image_repo }}:{{ csi_provisioner_image_tag }} + args: + - --csi-address=$(ADDRESS) + - --v=5 +{% if aws_ebs_csi_enable_volume_scheduling %} + - --feature-gates=Topology=true +{% endif %} + - --enable-leader-election + - --leader-election-type=leases + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: csi-attacher + image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }} + args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ +{% if aws_ebs_csi_enable_volume_snapshot %} + - name: csi-snapshotter + image: {{ csi_snapshotter_image_repo }}:{{ csi_snapshotter_image_tag }} + args: + - --csi-address=$(ADDRESS) + - --connection-timeout=15s + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ +{% endif %} +{% if aws_ebs_csi_enable_volume_resizing %} + - name: csi-resizer + image: {{ csi_resizer_image_repo }}:{{ csi_resizer_image_tag }} + imagePullPolicy: Always + args: + - --csi-address=$(ADDRESS) + - --v=5 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /var/lib/csi/sockets/pluginproxy/ +{% endif %} + - name: liveness-probe + image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + volumes: + - name: socket-dir + emptyDir: {} + diff --git a/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-driver.yml.j2 b/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-driver.yml.j2 new file mode 100644 index 000000000..aa9aca71f --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-driver.yml.j2 @@ -0,0 +1,8 @@ +--- +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: ebs.csi.aws.com +spec: + attachRequired: true + podInfoOnMount: false diff --git a/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-nodeservice.yml.j2 b/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-nodeservice.yml.j2 new file mode 100644 index 000000000..e1a7867cf --- /dev/null +++ b/roles/kubernetes-apps/csi_driver/aws_ebs/templates/aws-ebs-csi-nodeservice.yml.j2 @@ -0,0 +1,101 @@ +--- +# Node Service +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ebs-csi-node + namespace: kube-system +spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + nodeSelector: + beta.kubernetes.io/os: linux + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: ebs-plugin + securityContext: + privileged: true + image: {{ aws_ebs_csi_plugin_image_repo }}:{{ aws_ebs_csi_plugin_image_tag }} + args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=5 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet + mountPropagation: "Bidirectional" + - name: plugin-dir + mountPath: /csi + - name: device-dir + mountPath: /dev + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + - name: node-driver-registrar + image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/ebs.csi.aws.com-reg.sock /csi/csi.sock"] + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: liveness-probe + image: {{ csi_livenessprobe_image_repo }}:{{ csi_livenessprobe_image_tag }} + args: + - --csi-address=/csi/csi.sock + volumeMounts: + - name: plugin-dir + mountPath: /csi + volumes: + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: device-dir + hostPath: + path: /dev + type: Directory diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 index afc066edd..ade276450 100644 --- a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-controllerplugin.yml.j2 @@ -20,7 +20,7 @@ spec: serviceAccount: csi-cinder-controller-sa containers: - name: csi-attacher - image: {{ cinder_csi_attacher_image_repo }}:{{ cinder_csi_attacher_image_tag }} + image: {{ csi_attacher_image_repo }}:{{ csi_attacher_image_tag }} args: - "--v=5" - "--csi-address=$(ADDRESS)" @@ -37,7 +37,7 @@ spec: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-provisioner - image: {{ cinder_csi_provisioner_image_repo }}:{{ cinder_csi_provisioner_image_tag }} + image: {{ csi_provisioner_image_repo }}:{{ csi_provisioner_image_tag }} args: - "--csi-address=$(ADDRESS)" {% if cinder_topology is defined and cinder_topology %} @@ -56,7 +56,7 @@ spec: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: csi-snapshotter - image: {{ cinder_csi_snapshotter_image_repo }}:{{ cinder_csi_snapshotter_image_tag }} + image: {{ csi_snapshotter_image_repo }}:{{ csi_snapshotter_image_tag }} args: - "--csi-address=$(ADDRESS)" {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} @@ -71,7 +71,7 @@ spec: - mountPath: /var/lib/csi/sockets/pluginproxy/ name: socket-dir - name: csi-resizer - image: {{ cinder_csi_resizer_image_repo }}:{{ cinder_csi_resizer_image_tag }} + image: {{ csi_resizer_image_repo }}:{{ csi_resizer_image_tag }} args: - "--csi-address=$(ADDRESS)" {% if cinder_csi_controller_replicas is defined and cinder_csi_controller_replicas > 1 %} diff --git a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 index b4d92744a..b9cefa0c7 100644 --- a/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 +++ b/roles/kubernetes-apps/csi_driver/cinder/templates/cinder-csi-nodeplugin.yml.j2 @@ -19,7 +19,7 @@ spec: hostNetwork: true containers: - name: node-driver-registrar - image: {{ cinder_csi_node_driver_registrar_image_repo }}:{{ cinder_csi_node_driver_registrar_image_tag }} + image: {{ csi_node_driver_registrar_image_repo }}:{{ csi_node_driver_registrar_image_tag }} args: - "--csi-address=$(ADDRESS)" - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index bbfceb710..1a1ed0247 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -37,6 +37,14 @@ dependencies: - cinder-csi-driver - csi-driver + - role: kubernetes-apps/csi_driver/aws_ebs + when: + - aws_ebs_csi_enabled + tags: + - apps + - aws-ebs-csi-driver + - csi-driver + - role: kubernetes-apps/persistent_volumes when: - persistent_volumes_enabled diff --git a/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/OWNERS b/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/OWNERS new file mode 100644 index 000000000..6e44ceb4e --- /dev/null +++ b/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/OWNERS @@ -0,0 +1,5 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: + - alijahnas +reviewers: diff --git a/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/defaults/main.yml b/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/defaults/main.yml new file mode 100644 index 000000000..896d2d3cb --- /dev/null +++ b/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/defaults/main.yml @@ -0,0 +1,8 @@ +--- +# To restrict which AZ the volume should be provisioned in +# set this value to true and set the list of relevant AZs +# For it to work, the flag aws_ebs_csi_enable_volume_scheduling +# in AWS EBS Driver must be true +restrict_az_provisioning: false +aws_ebs_availability_zones: + - eu-west-3c diff --git a/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/tasks/main.yml b/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/tasks/main.yml new file mode 100644 index 000000000..006c35d1e --- /dev/null +++ b/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/tasks/main.yml @@ -0,0 +1,19 @@ +--- +- name: Kubernetes Persistent Volumes | Copy AWS EBS CSI Storage Class template + template: + src: "aws-ebs-csi-storage-class.yml.j2" + dest: "{{ kube_config_dir }}/aws-ebs-csi-storage-class.yml" + register: manifests + when: + - inventory_hostname == groups['kube-master'][0] + +- name: Kubernetes Persistent Volumes | Add AWS EBS CSI Storage Class + kube: + name: aws-ebs-csi + kubectl: "{{ bin_dir }}/kubectl" + resource: StorageClass + filename: "{{ kube_config_dir }}/aws-ebs-csi-storage-class.yml" + state: "latest" + when: + - inventory_hostname == groups['kube-master'][0] + - manifests.changed diff --git a/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/templates/aws-ebs-csi-storage-class.yml.j2 b/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/templates/aws-ebs-csi-storage-class.yml.j2 new file mode 100644 index 000000000..1632646f8 --- /dev/null +++ b/roles/kubernetes-apps/persistent_volumes/aws-ebs-csi/templates/aws-ebs-csi-storage-class.yml.j2 @@ -0,0 +1,18 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: ebs-sc +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer +parameters: + csi.storage.k8s.io/fstype: xfs + type: gp2 +{% if restrict_az_provisioning %} +allowedTopologies: +- matchLabelExpressions: + - key: topology.ebs.csi.aws.com/zone + values: +{% for value in aws_ebs_availability_zones %} + - {{ value }} +{% endfor %} +{% endif %} diff --git a/roles/kubernetes-apps/persistent_volumes/meta/main.yml b/roles/kubernetes-apps/persistent_volumes/meta/main.yml index bc5f14fab..ae33fd143 100644 --- a/roles/kubernetes-apps/persistent_volumes/meta/main.yml +++ b/roles/kubernetes-apps/persistent_volumes/meta/main.yml @@ -13,3 +13,10 @@ dependencies: tags: - persistent_volumes_cinder_csi - cinder-csi-driver + + - role: kubernetes-apps/persistent_volumes/aws-ebs-csi + when: + - aws_ebs_csi_enabled + tags: + - persistent_volumes_aws_ebs_csi + - aws-ebs-csi-driver diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index bcaa97d7a..cf5c9c5dc 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -304,6 +304,7 @@ enable_network_policy: true local_volume_provisioner_enabled: "{{ local_volumes_enabled | default('false') }}" local_volume_provisioner_directory_mode: 0700 cinder_csi_enabled: false +aws_ebs_csi_enabled: false persistent_volumes_enabled: false cephfs_provisioner_enabled: false rbd_provisioner_enabled: false