From 6602760a481ab54f6b35a8daa004bad7acf0dc4e Mon Sep 17 00:00:00 2001 From: Chad Swenson Date: Fri, 5 Oct 2018 07:52:25 -0500 Subject: [PATCH] Support multiple local volume provisioner StorageClasses (#3450) - Local Volume StorageClass configuration is now manged by `local_volume_provisioner_storage_classes`, a list of maps that specifies local storage classes with `name` `host_dir` and `mount_dir` keys per entry - Tasks and templates updated to loop through local volume storage classes - Previous defaults for path/class names were not changed - Fixed an issue where a `kubernetes/preinstall` was creating directories inconsistently with the `kubernetes-apps/external_provisioner/local_volume_provisioner` task --- .../sample/group_vars/k8s-cluster/addons.yml | 16 +++++++++++++--- .../local_volume_provisioner/README.md | 2 +- .../local_volume_provisioner/defaults/main.yml | 7 ++++--- .../local_volume_provisioner/tasks/main.yml | 8 +++++--- .../templates/local-volume-provisioner-cm.yml.j2 | 8 +++++--- .../templates/local-volume-provisioner-ds.yml.j2 | 12 ++++++++---- .../local-volume-provisioner-psp.yml.j2 | 4 +++- .../templates/local-volume-provisioner-sc.yml.j2 | 4 +++- .../node/templates/kubelet-container.j2 | 6 ++++-- .../node/templates/kubelet.rkt.service.j2 | 16 ++++++++++------ .../preinstall/tasks/0050-create_directories.yml | 10 +++++----- roles/kubespray-defaults/defaults/main.yaml | 8 +++++--- 12 files changed, 66 insertions(+), 35 deletions(-) diff --git a/inventory/sample/group_vars/k8s-cluster/addons.yml b/inventory/sample/group_vars/k8s-cluster/addons.yml index 35096f17f..2e70ba3ce 100644 --- a/inventory/sample/group_vars/k8s-cluster/addons.yml +++ b/inventory/sample/group_vars/k8s-cluster/addons.yml @@ -14,9 +14,19 @@ registry_enabled: false # Local volume provisioner deployment local_volume_provisioner_enabled: false # local_volume_provisioner_namespace: kube-system -# local_volume_provisioner_base_dir: /mnt/disks -# local_volume_provisioner_mount_dir: /mnt/disks -# local_volume_provisioner_storage_class: local-storage +# local_volume_provisioner_storage_classes: +# - name: "{{ local_volume_provisioner_storage_class | default('local-storage') }}" +# host_dir: "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}" +# mount_dir: "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}" +# - name: "local-ssd" +# host_dir: "/mnt/local-storage/ssd" +# mount_dir: "/mnt/local-storage/ssd" +# - name: "local-hdd" +# host_dir: "/mnt/local-storage/hdd" +# mount_dir: "/mnt/local-storage/hdd" +# - name: "local-shared" +# host_dir: "/mnt/local-storage/shared" +# mount_dir: "/mnt/local-storage/shared" # CephFS provisioner deployment cephfs_provisioner_enabled: false diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md index 900694795..cbc468f13 100644 --- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md @@ -3,7 +3,7 @@ Local Storage Provisioner The local storage provisioner is NOT a dynamic storage provisioner as you would expect from a cloud provider. Instead, it simply creates PersistentVolumes for -all manually created volumes located in the directory `local_volume_provisioner_base_dir`. +all manually created volumes located in the directories specified in the `local_volume_provisioner_storage_classes.host_dir` entries. The default path is /mnt/disks and the rest of this doc will use that path as an example. diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml index 4b18546d3..2a703cbf2 100644 --- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml @@ -1,5 +1,6 @@ --- local_volume_provisioner_namespace: "kube-system" -local_volume_provisioner_base_dir: /mnt/disks -local_volume_provisioner_mount_dir: /mnt/disks -local_volume_provisioner_storage_class: local-storage +local_volume_provisioner_storage_classes: + - name: "{{ local_volume_provisioner_storage_class | default('local-storage') }}" + host_dir: "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}" + mount_dir: "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}" \ No newline at end of file diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml index 070f4c00c..ba6304166 100644 --- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml @@ -2,13 +2,15 @@ - name: Local Volume Provisioner | Ensure base dir is created on all hosts file: - path: "{{ local_volume_provisioner_base_dir }}" + path: "{{ item[1].host_dir }}" state: directory owner: root group: root mode: 0700 - delegate_to: "{{ item }}" - with_items: "{{ groups['k8s-cluster'] }}" + delegate_to: "{{ item[0] }}" + with_nested: + - "{{ groups['k8s-cluster'] }}" + - "{{ local_volume_provisioner_storage_classes }}" failed_when: false - name: Local Volume Provisioner | Create addon dir diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 index 8ad76ab2d..8c5078eb6 100644 --- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 @@ -6,6 +6,8 @@ metadata: namespace: {{ local_volume_provisioner_namespace }} data: storageClassMap: | - {{ local_volume_provisioner_storage_class }}: - hostDir: {{ local_volume_provisioner_base_dir }} - mountDir: {{ local_volume_provisioner_mount_dir }} +{% for class in local_volume_provisioner_storage_classes %} + {{ class.name }}: + hostDir: {{ class.host_dir }} + mountDir: {{ class.mount_dir }} +{% endfor %} \ No newline at end of file diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 index 487e4f9f3..713b72287 100644 --- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 @@ -44,13 +44,17 @@ spec: - name: local-volume-provisioner mountPath: /etc/provisioner/config readOnly: true - - name: local-volume-provisioner-hostpath-mnt-disks - mountPath: {{ local_volume_provisioner_mount_dir }} +{% for class in local_volume_provisioner_storage_classes %} + - name: {{ class.name }} + mountPath: {{ class.mount_dir }} mountPropagation: "HostToContainer" +{% endfor %} volumes: - name: local-volume-provisioner configMap: name: local-volume-provisioner - - name: local-volume-provisioner-hostpath-mnt-disks +{% for class in local_volume_provisioner_storage_classes %} + - name: {{ class.name }} hostPath: - path: {{ local_volume_provisioner_base_dir }} + path: {{ class.host_dir }} +{% endfor %} diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-psp.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-psp.yml.j2 index 9daf694fa..0358d8dda 100644 --- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-psp.yml.j2 +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-psp.yml.j2 @@ -25,8 +25,10 @@ spec: - 'downwardAPI' - 'hostPath' allowedHostPaths: - - pathPrefix: "{{ local_volume_provisioner_base_dir }}" +{% for class in local_volume_provisioner_storage_classes %} + - pathPrefix: "{{ class.host_dir }}" readOnly: false +{% endfor %} hostNetwork: false hostIPC: false hostPID: false diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 index bf1f00262..2a5ad13d4 100644 --- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 +++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 @@ -1,7 +1,9 @@ +{% for class in local_volume_provisioner_storage_classes %} --- apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: {{ local_volume_provisioner_storage_class }} + name: {{ class.name }} provisioner: kubernetes.io/no-provisioner volumeBindingMode: WaitForFirstConsumer +{% endfor %} \ No newline at end of file diff --git a/roles/kubernetes/node/templates/kubelet-container.j2 b/roles/kubernetes/node/templates/kubelet-container.j2 index 58e54560d..c8ffbfce3 100644 --- a/roles/kubernetes/node/templates/kubelet-container.j2 +++ b/roles/kubernetes/node/templates/kubelet-container.j2 @@ -31,8 +31,10 @@ -v {{ kubelet_flexvolumes_plugins_dir }}:{{ kubelet_flexvolumes_plugins_dir }}:rw \ {% endif -%} {% if local_volume_provisioner_enabled -%} - -v {{ local_volume_provisioner_base_dir }}:{{ local_volume_provisioner_base_dir }}:rw \ - -v {{ local_volume_provisioner_mount_dir }}:{{ local_volume_provisioner_mount_dir }}:rw \ + {% for class in local_volume_provisioner_storage_classes -%} + -v {{ class.host_dir }}:{{ class.host_dir }}:rw \ + -v {{ class.mount_dir }}:{{ class.mount_dir }}:rw \ + {% endfor -%} {% endif %} -v {{kube_config_dir}}:{{kube_config_dir}}:ro \ -v /etc/os-release:/etc/os-release:ro \ diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 index 7f28f87b6..ee1eaa1b1 100644 --- a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 @@ -47,11 +47,13 @@ ExecStart=/usr/bin/rkt run \ --volume flexvolumes,kind=host,source={{ kubelet_flexvolumes_plugins_dir }},readOnly=false \ {% endif -%} {% if local_volume_provisioner_enabled %} - --volume local-volume-provisioner-base-dir,kind=host,source={{ local_volume_provisioner_base_dir }},readOnly=false \ +{% for class in local_volume_provisioner_storage_classes %} + --volume local-volume-provisioner-base-dir,kind=host,source={{ class.host_dir }},readOnly=false \ {# Not pretty, but needed to avoid double mount #} -{% if local_volume_provisioner_base_dir not in local_volume_provisioner_mount_dir and local_volume_provisioner_mount_dir not in local_volume_provisioner_base_dir %} - --volume local-volume-provisioner-mount-dir,kind=host,source={{ local_volume_provisioner_mount_dir }},readOnly=false \ +{% if class.host_dir not in class.mount_dir and class.mount_dir not in class.host_dir %} + --volume local-volume-provisioner-mount-dir,kind=host,source={{ class.mount_dir }},readOnly=false \ {% endif %} +{% endfor %} {% endif %} {% if kubelet_load_modules == true %} --mount volume=lib-modules,target=/lib/modules \ @@ -81,11 +83,13 @@ ExecStart=/usr/bin/rkt run \ --mount volume=flexvolumes,target={{ kubelet_flexvolumes_plugins_dir }} \ {% endif -%} {% if local_volume_provisioner_enabled %} - --mount volume=local-volume-provisioner-base-dir,target={{ local_volume_provisioner_base_dir }} \ +{% for class in local_volume_provisioner_storage_classes %} + --mount volume=local-volume-provisioner-base-dir,target={{ class.host_dir }} \ {# Not pretty, but needed to avoid double mount #} -{% if local_volume_provisioner_base_dir not in local_volume_provisioner_mount_dir and local_volume_provisioner_mount_dir not in local_volume_provisioner_base_dir %} - --mount volume=local-volume-provisioner-mount-dir,target={{ local_volume_provisioner_mount_dir }} \ +{% if class.host_dir not in class.mount_dir and class.mount_dir not in class.host_dir %} + --mount volume=local-volume-provisioner-mount-dir,target={{ class.mount_dir }} \ {% endif %} +{% endfor %} {% endif %} --stage1-from-dir=stage1-fly.aci \ {% if kube_hyperkube_image_repo == "docker" %} diff --git a/roles/kubernetes/preinstall/tasks/0050-create_directories.yml b/roles/kubernetes/preinstall/tasks/0050-create_directories.yml index 11f8e00d4..307116033 100644 --- a/roles/kubernetes/preinstall/tasks/0050-create_directories.yml +++ b/roles/kubernetes/preinstall/tasks/0050-create_directories.yml @@ -46,12 +46,12 @@ - name: Create local volume provisioner directories file: - path: "{{ item }}" + path: "{{ item.host_dir }}" state: directory - owner: kube - with_items: - - "{{ local_volume_provisioner_base_dir }}" - - "{{ local_volume_provisioner_mount_dir }}" + owner: root + group: root + mode: 0700 + with_items: "{{ local_volume_provisioner_storage_classes }}" when: - inventory_hostname in groups['k8s-cluster'] - local_volume_provisioner_enabled diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index ef691f10d..8ddd956c7 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -301,9 +301,11 @@ vault_config_dir: "{{ vault_base_dir }}/config" vault_roles_dir: "{{ vault_base_dir }}/roles" vault_secrets_dir: "{{ vault_base_dir }}/secrets" -# Local volume provisioner dirs -local_volume_provisioner_base_dir: /mnt/disks -local_volume_provisioner_mount_dir: /mnt/disks +# Local volume provisioner storage classes +local_volume_provisioner_storage_classes: + - name: "{{ local_volume_provisioner_storage_class | default('local-storage') }}" + host_dir: "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}" + mount_dir: "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}" # weave's network password for encryption # if null then no network encryption