More dynamic local-storage-provisioner approach (#3472)

* Makes local volume provisioner more dynamic

* Correct variable name in local storage provisioner defaults

* Updates external-provisioner readme

* Updates variable naming to be more clear, more documentation, fixes sample inventory

* Variable refactor, untangled some jinja2 loops

* Corrects variable name

* No variable substitution in dict keys, replaced with anchor

* Fixes default storage_classes dict, inline docs

* Fixes spelling in inline docs

* Addresses comments in review

* Updates all the defaults

* Fix failing CI task

* Fixes external provisioner daemonset

inventory/sample/group_vars/k8s-cluster/addons.yml

@@ -21,18 +21,17 @@ metrics_server_enabled: false
 local_volume_provisioner_enabled: false
 # local_volume_provisioner_namespace: kube-system
 # local_volume_provisioner_storage_classes:
-#   - name: "{{ local_volume_provisioner_storage_class | default('local-storage') }}"
+#   - local-storage:
-#     host_dir: "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}"
+#       host_dir: /mnt/disks
-#     mount_dir: "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}"
+#       mount_dir: /mnt/disks
-#   - name: "local-ssd"
+#   - fast-disks:
-#     host_dir: "/mnt/local-storage/ssd"
+#       host_dir: /mnt/fast-disks
-#     mount_dir: "/mnt/local-storage/ssd"
+#       mount_dir: /mnt/fast-disks
-#   - name: "local-hdd"
+#       block_cleaner_command:
-#     host_dir: "/mnt/local-storage/hdd"
+#          - "/scripts/shred.sh"
-#     mount_dir: "/mnt/local-storage/hdd"
+#          - "2"
-#   - name: "local-shared"
+#       volume_mode: Filesystem
-#     host_dir: "/mnt/local-storage/shared"
+#       fs_type: ext4
-#     mount_dir: "/mnt/local-storage/shared"
 
 # CephFS provisioner deployment
 cephfs_provisioner_enabled: false

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md

@@ -1,11 +1,51 @@
 Local Storage Provisioner
 =========================
 
-The local storage provisioner is NOT a dynamic storage provisioner as you would
+The [local storage provisioner](https://github.com/kubernetes-incubator/external-storage/tree/master/local-volume)
+is NOT a dynamic storage provisioner as you would
 expect from a cloud provider. Instead, it simply creates PersistentVolumes for
-all manually created volumes located in the directories specified in the `local_volume_provisioner_storage_classes.host_dir` entries.
+all mounts under the host_dir of the specified storage class.
-The default path is /mnt/disks and the rest of this doc will use that path as
+These storage classes are specified in the `local_volume_provisioner_storage_classes` list.
-an example.
+An example this list:
+
+```yaml
+local_volume_provisioner_storage_classes:
+  - local-storage:
+      host_dir: /mnt/disks
+      mount_dir: /mnt/disks
+  - fast-disks:
+      host_dir: /mnt/fast-disks
+      mount_dir: /mnt/fast-disks
+      block_cleaner_command:
+         - "/scripts/shred.sh"
+         - "2"
+      volume_mode: Filesystem
+      fs_type: ext4
+```
+
+For each dictionary in `local_volume_provisioner_storage_classes` a storageClass with the
+same name is created. The keys of this dictionary are converted to camelCase and added
+as attributes to the storageClass.
+The result of the above example is:
+
+```yaml
+data:
+  storageClassMap: |
+    local-storage:
+       hostDir: /mnt/disks
+       mountDir: /mnt/disks
+    fast-disks:
+       hostDir: /mnt/fast-disks
+       mountDir:  /mnt/fast-disks
+       blockCleanerCommand:
+         - "/scripts/shred.sh"
+         - "2"
+       volumeMode: Filesystem
+       fsType: ext4
+```
+
+The default StorageClass is local-storage on /mnt/disks,
+the rest of this doc will use that path as an example.
 
 Examples to create local storage volumes
 ----------------------------------------

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml

@@ -1,6 +1,11 @@
 ---
 local_volume_provisioner_namespace: "kube-system"
-local_volume_provisioner_storage_classes:
+# Levarages Ansibles string to Python datatype casting. Otherwise the dict_key isn't substituted
-  - name: "{{ local_volume_provisioner_storage_class | default('local-storage') }}"
+# see https://github.com/ansible/ansible/issues/17324
-    host_dir: "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}"
+local_volume_provisioner_storage_classes: |
-    mount_dir: "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}"
+  {
+    "{{ local_volume_provisioner_storage_class | default('local-storage') }}": {
+     "host_dir": "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}",
+      "mount_dir": "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}"
+    }
+  }

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml

@@ -1,8 +1,7 @@
 ---
-
 - name: Local Volume Provisioner | Ensure base dir is created on all hosts
   file:
-    path: "{{ item[1].host_dir }}"
+    path: "{{ local_volume_provisioner_storage_classes[item.1].host_dir }}"
     state: directory
     owner: root
     group: root
@@ -10,7 +9,7 @@
   delegate_to: "{{ item[0] }}"
   with_nested:
     - "{{ groups['k8s-cluster'] }}"
-    - "{{ local_volume_provisioner_storage_classes }}"
+    - "{{ local_volume_provisioner_storage_classes.keys() }}"
   failed_when: false
 
 - name: Local Volume Provisioner | Create addon dir

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2

@@ -1,3 +1,14 @@
+# Macro to convert camelCase dictionary keys to snake_case keys
+{%- macro convert_keys(mydict) %}
+  {% for key in mydict.keys() -%}
+    {% set key_split = key.split('_') -%}
+    {% set new_key = key_split[0] + key_split[1:]|map('capitalize')|join -%}
+    {% set value = mydict.pop(key) -%}
+    {{ mydict.__setitem__(new_key, value) -}}
+    {{ convert_keys(value) if value is mapping else None -}}
+  {% endfor -%}
+{% endmacro -%}
+
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -6,8 +17,9 @@ metadata:
   namespace: {{ local_volume_provisioner_namespace }}
 data:
   storageClassMap: |
-{% for class in local_volume_provisioner_storage_classes %}
+{% for class_name, storage_class in local_volume_provisioner_storage_classes.iteritems() %}
-    {{ class.name }}:
+    {{ class_name }}:
-      hostDir: {{ class.host_dir }}
+      {{- convert_keys(storage_class) }}
-      mountDir: {{ class.mount_dir }}
+      {{ storage_class | to_nice_yaml(indent=2) | indent(6) }}
-{% endfor %}
+{%- endfor %}
+

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2

@@ -44,17 +44,17 @@ spec:
             - name: local-volume-provisioner
               mountPath: /etc/provisioner/config
               readOnly: true
-{% for class in local_volume_provisioner_storage_classes %}
+{% for class_name, class_config in local_volume_provisioner_storage_classes.iteritems() %}
-            - name: {{ class.name }}
+            - name: local-volume-provisioner-hostpath-{{ class_name }}
-              mountPath: {{ class.mount_dir }}
+              mountPath: {{ class_config.mount_dir }}
               mountPropagation: "HostToContainer"
 {% endfor %}
       volumes:
         - name: local-volume-provisioner
           configMap:
             name: local-volume-provisioner
-{% for class in local_volume_provisioner_storage_classes %}
+{% for class_name, class_config in local_volume_provisioner_storage_classes.iteritems() %}
-        - name: {{ class.name }}
+        - name: local-volume-provisioner-hostpath-{{ class_name }}
           hostPath:
-            path: {{ class.host_dir }}
+            path: {{ class_config.host_dir }}
 {% endfor %}

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-psp.yml.j2

@@ -25,8 +25,8 @@ spec:
     - 'downwardAPI'
     - 'hostPath'
   allowedHostPaths:
-{% for class in local_volume_provisioner_storage_classes %}
+{% for class_name, class_config in local_volume_provisioner_storage_classes.iteritems() %}
-    - pathPrefix: "{{ class.host_dir }}"
+    - pathPrefix: "{{ class_config.host_dir }}"
       readOnly: false
 {% endfor %}
   hostNetwork: false

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2

@@ -1,9 +1,9 @@
-{% for class in local_volume_provisioner_storage_classes %}
+{% for class_name in local_volume_provisioner_storage_classes.keys() %}
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-  name: {{ class.name }}
+  name: {{ class_name }}
 provisioner: kubernetes.io/no-provisioner
 volumeBindingMode: WaitForFirstConsumer
-{% endfor %}
+{% endfor %}

roles/kubernetes/preinstall/tasks/0050-create_directories.yml

@@ -47,12 +47,14 @@
 
 - name: Create local volume provisioner directories
   file:
-    path: "{{ item.host_dir }}"
+    path: "{{ local_volume_provisioner_storage_classes[item.1].host_dir }}"
     state: directory
     owner: root
     group: root
     mode: 0700
-  with_items: "{{ local_volume_provisioner_storage_classes }}"
+  with_nested:
+    - "{{ groups['k8s-cluster'] }}"
+    - "{{ local_volume_provisioner_storage_classes.keys() }}"
   when:
     - inventory_hostname in groups['k8s-cluster']
     - local_volume_provisioner_enabled

roles/kubespray-defaults/defaults/main.yaml

@@ -314,10 +314,15 @@ kube_feature_gates: |-
   {%- endif %}
 
 # Local volume provisioner storage classes
-local_volume_provisioner_storage_classes:
+# Levarages Ansibles string to Python datatype casting. Otherwise the dict_key isn't substituted
-  - name: "{{ local_volume_provisioner_storage_class | default('local-storage') }}"
+# see https://github.com/ansible/ansible/issues/17324
-    host_dir: "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}"
+local_volume_provisioner_storage_classes: |
-    mount_dir: "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}"
+  {
+    "{{ local_volume_provisioner_storage_class | default('local-storage') }}": {
+     "host_dir": "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}",
+      "mount_dir": "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}"
+    }
+  }
 
 # weave's network password for encryption
 # if null then no network encryption