recator to use kube module, finally fix race condition in storage tasks

contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml

@@ -1,4 +1,3 @@
----
 # Bootstrap heketi
 - name: "Get state of heketi service, deployment and pods."
   register: "initial_heketi_state"
@@ -9,7 +8,7 @@
     - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0"
     - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0"
     - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0"
-  include_tasks: "setup/boot.yml"
+  include_tasks: "bootstrap/deploy.yml"
 
 # Prepare heketi topology
 - name: "Get heketi initial pod state."
@@ -27,11 +26,11 @@
   command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json"
 - name: "Load heketi topology."
   when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
-  include_tasks: "setup/topology.yml"
+  include_tasks: "bootstrap/topology.yml"
 
 # Provision heketi database volume
 - name: "Prepare heketi volumes."
-  include_tasks: "setup/volumes.yml"
+  include_tasks: "bootstrap/volumes.yml"
 
 # Prepare heketi storage
 - name: "Test heketi storage."
@@ -41,7 +40,7 @@
 - command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
   register: "job"
 - name: "Create heketi storage."
-  include_tasks: "setup/storage.yml"
+  include_tasks: "bootstrap/storage.yml"
   vars:
     secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
     endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
@@ -52,8 +51,7 @@
     - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
     - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
     - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
-# Finalize setup
+
+# Remove bootstrap heketi
 - name: "Tear down bootstrap."
-  include_tasks: "setup/tear-down-bootstrap.yml"
+  include_tasks: "bootstrap/tear-down.yml"
-- name: "Setup final heketi instance."
-  include_tasks: "setup/heketi.yml"

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml

@@ -0,0 +1,23 @@
+---
+- name: "Kubernetes Apps | Lay Down Heketi Bootstrap"
+  become: true
+  template: { src: "heketi-bootstrap.json.j2", dest: "{{ kube_config_dir }}/heketi-bootstrap.json" }
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi Bootstrap"
+  kube:
+    name: "GlusterFS"
+    filename: "{{ kube_config_dir }}/heketi-bootstrap.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"
+- name: "Wait for heketi bootstrap to complete."
+  changed_when: false
+  register: "initial_heketi_state"
+  vars:
+    initial_heketi_state: { stdout: "{}" }
+    pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]"
+    deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]"
+  command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
+  until:
+      - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'"
+      - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml

@@ -4,7 +4,10 @@
   changed_when: false
   register: "heketi_storage_state"
 - name: "Create heketi storage."
-  command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-storage.json"
+  kube:
+    name: "GlusterFS"
+    filename: "{{ kube_config_dir }}/heketi-storage.json"
+    state: "present"
   vars:
     secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
     endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
@@ -16,17 +19,17 @@
     - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
     - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
   register: "heketi_storage_result"
-- command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
+
-  register: "heketi_storage_state"
 - name: "Get state of heketi storage service, endpoint, secret and job."
   command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
   changed_when: false
   register: "heketi_storage_state"
   vars:
+    heketi_storage_state: { stdout: "{}" }
     secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
     endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
     service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
-    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.conditions[?type=='Complete'].status=='True']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.succeeded==1]"
   until:
     - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1"
     - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1"
@@ -34,5 +37,3 @@
     - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 1"
   retries: 60
   delay: 5
-# looks like there is some race condition that leads to "Database file did not appear, exiting.", can't figure out where
-- command: "sleep 10"

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml

@@ -13,6 +13,7 @@
 - name: "Load heketi topology."
   when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
   command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology load --json=/tmp/topology.json"
+  register: "load_heketi"
 - name: "Get heketi topology."
   register: "heketi_topology"
   command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json"

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml

@@ -31,12 +31,6 @@
   with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}"
   loop_control: { loop_var: "volume_id" }
   register: "volumes_information"
-- name: "debug heketi db vol."
-  with_items: "{{ volumes_information.results }}"
-  loop_control: { loop_var: "volume_information" }
-  vars: { volume: "{{ volume_information.stdout|from_json }}" }
-  when: "volume.name == 'heketidbstorage'"
-  debug: { var: "volume" }
 - name: "Test heketi database volume."
   set_fact: { heketi_database_volume_created: true }
   with_items: "{{ volumes_information.results }}"

contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml

@@ -0,0 +1,37 @@
+---
+- name: "Kubernetes Apps | Lay Down GlusterFS Daemonset"
+  template: { src: "glusterfs-daemonset.json.j2", dest: "{{ kube_config_dir }}/glusterfs-daemonset.json" }
+  become: true
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure GlusterFS daemonset"
+  kube:
+    name: "GlusterFS"
+    filename: "{{ kube_config_dir }}/glusterfs-daemonset.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"
+- name: "Kubernetes Apps | Wait for daemonset to become available."
+  register: "daemonset_state"
+  command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true"
+  changed_when: false
+  vars:
+    daemonset_state: { stdout: "{}" }
+    ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}"
+    desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}"
+  until: "ready == desired"
+  retries: 60
+  delay: 5
+
+- name: "Kubernetes Apps | Label GlusterFS nodes"
+  include_tasks: "glusterfs/label.yml"
+  with_items: "{{ groups['heketi-node'] }}"
+  loop_control:
+    loop_var: "node"
+
+- name: "Kubernetes Apps | Lay Down Heketi Service Account"
+  template: { src: "heketi-service-account.json.j2", dest: "{{ kube_config_dir }}/heketi-service-account.json" }
+  become: true
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi Service Account"
+  kube:
+    name: "GlusterFS"
+    filename: "{{ kube_config_dir }}/heketi-service-account.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"

contrib/network-storage/heketi/roles/provision/tasks/heketi.yml

@@ -0,0 +1,25 @@
+---
+- name: "Kubernetes Apps | Lay Down Heketi"
+  become: true
+  template: { src: "heketi-deployment.json.j2", dest: "{{ kube_config_dir }}/heketi-deployment.json" }
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi"
+  kube:
+    name: "GlusterFS"
+    filename: "{{ kube_config_dir }}/heketi-deployment.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"
+- name: "Ensure heketi is up and running."
+  changed_when: false
+  register: "heketi_state"
+  vars:
+    heketi_state: { stdout: "{}" }
+    pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]"
+    deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]"
+  command: "{{ bin_dir }}/kubectl get deployments,pods --selector=glusterfs --output=json"
+  until:
+      - "heketi_state.stdout|from_json|json_query(pods_query) == 'True'"
+      - "heketi_state.stdout|from_json|json_query(deployments_query) == 'True'"
+  retries: 60
+  delay: 5
+- set_fact:
+    heketi_pod_name: "{{ heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.name|[0]\") }}"

contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml

@@ -1,64 +0,0 @@
----
-- register: "daemonset_state"
-  command: "{{ bin_dir }}/kubectl get daemonset glusterfs -o=name --ignore-not-found=true"
-  changed_when: false
-- name: "Deploy the GlusterFS DaemonSet"
-  when: "daemonset_state.stdout == \"\""
-  command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/glusterfs-daemonset.json"
-- register: "daemonset_state"
-  command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true"
-  changed_when: false
-- name: "Wait for daemonset to become available."
-  register: "daemonset_state"
-  command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true"
-  changed_when: false
-  vars:
-    ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}"
-    desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}"
-  until: "ready == desired"
-  retries: 60
-  delay: 5
-
-- name: "Label Gluster nodes"
-  with_items: "{{ groups['heketi-node'] }}"
-  loop_control:
-    loop_var: "node"
-  include_tasks: "kubernetes/label.yml"
-
-- register: "service_account_state"
-  command: "{{ bin_dir }}/kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true"
-  changed_when: false
-- name: "Deploy the Heketi service account"
-  when: "service_account_state.stdout == \"\""
-  command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-service-account.json"
-- register: "service_account_state"
-  command: "{{ bin_dir }}/kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true"
-  changed_when: false
-- assert: { that: "service_account_state.stdout != \"\"", message: "Heketi service account is not present." }
-
-- register: "clusterrolebinding_state"
-  command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
-  changed_when: false
-- name: "Deploy cluster role binding."
-  when: "clusterrolebinding_state.stdout == \"\""
-  command: "{{ bin_dir }}/kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account"
-- register: "clusterrolebinding_state"
-  command: "{{ bin_dir }}/kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
-  changed_when: false
-- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." }
-
-- register: "secret_state"
-  command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
-  changed_when: false
-- name: "Render Heketi secret configuration."
-  become: true
-  template:
-    src: "heketi.json.j2"
-    dest: "{{ kube_config_dir }}/heketi.json"
-- name: "Deploy Heketi config secret"
-  when: "secret_state.stdout == \"\""
-  command: "{{ bin_dir }}/kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json"
-- register: "secret_state"
-  command: "{{ bin_dir }}/kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
-  changed_when: false
-- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." }

contrib/network-storage/heketi/roles/provision/tasks/main.yml

@@ -1,34 +1,27 @@
 ---
-- name: "Render configuration."
+- name: "Kubernetes Apps | GlusterFS"
-  become: true
+  include_tasks: "glusterfs.yml"
-  template: { src: "{{ item.file }}.j2", dest: "{{ kube_config_dir }}/{{ item.file }}" }
-  with_items:
-      - { file: "glusterfs-daemonset.json" }
-      - { file: "heketi-bootstrap.json" }
-      - { file: "heketi-deployment.json" }
-      - { file: "heketi-service-account.json" }
-- name: "Prepare kubernetes."
-  include_tasks: "kubernetes.yml"
 
-- name: "Test heketi setup."
+- name: "Kubernetes Apps | Heketi Secrets"
+  include_tasks: "secret.yml"
+
+- name: "Kubernetes Apps | Test Heketi"
   register: "heketi_service_state"
-  command: "{{ bin_dir }}/kubectl get service heketi -o=name --ignore-not-found=true"
+  command: "kubectl get service heketi-storage-endpoints -o=name --ignore-not-found=true"
   changed_when: false
 
-- name: "Setup heketi."
+- name: "Kubernetes Apps | Bootstrap Heketi"
   when: "heketi_service_state.stdout == \"\""
-  include_tasks: "setup.yml"
+  include_tasks: "bootstrap.yml"
 
-- name: "Test storage class."
+- name: "Kubernetes Apps | Heketi"
-  changed_when: false
+  include_tasks: "heketi.yml"
-  command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json"
+
-  register: "storageclass"
+- name: "Kubernetes Apps | Heketi Topology"
-- name: "Setup storage class."
+  include_tasks: "topology.yml"
-  when: "storageclass.stdout == \"\""
+
+- name: "Kubernetes Apps | Heketi Storage"
+  include_tasks: "storage.yml"
+
+- name: "Kubernetes Apps | Storage Class"
   include_tasks: "storageclass.yml"
-- name: "Test storage class."
-  changed_when: false
-  command: "{{ bin_dir }}/kubectl get storageclass gluster --ignore-not-found=true --output=json"
-  register: "storageclass"
-- name: "Ensure storage class is up."
-  assert: { that: "storageclass.stdout != \"\"" }

contrib/network-storage/heketi/roles/provision/tasks/secret.yml

@@ -0,0 +1,27 @@
+---
+- register: "clusterrolebinding_state"
+  command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Kubernetes Apps | Deploy cluster role binding."
+  when: "clusterrolebinding_state.stdout == \"\""
+  command: "kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account"
+- register: "clusterrolebinding_state"
+  command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." }
+
+- register: "secret_state"
+  command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
+  changed_when: false
+- name: "Render Heketi secret configuration."
+  become: true
+  template:
+    src: "heketi.json.j2"
+    dest: "{{ kube_config_dir }}/heketi.json"
+- name: "Deploy Heketi config secret"
+  when: "secret_state.stdout == \"\""
+  command: "kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json"
+- register: "secret_state"
+  command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
+  changed_when: false
+- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." }

contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml

@@ -1,27 +0,0 @@
----
-- name: "Get state of heketi service, deployment and pods."
-  register: "initial_heketi_state"
-  changed_when: false
-  command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
-- name: "Create Heketi initial service and deployment"
-  command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-bootstrap.json"
-  when:
-    - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0"
-    - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0"
-    - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0"
-- name: "Get state of heketi service, deployment and pods."
-  register: "initial_heketi_state"
-  changed_when: false
-  command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
-- name: "Wait for heketi bootstrap to complete."
-  changed_when: false
-  register: "initial_heketi_state"
-  vars:
-    pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]"
-    deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]"
-  command: "{{ bin_dir }}/kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
-  until:
-      - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'"
-      - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'"
-  retries: 60
-  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml

@@ -1,10 +0,0 @@
----
-- name: "Create long term Heketi instance."
-  command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/heketi-deployment.json"
-- name: "Get heketi deployment state."
-  register: "heketi_deployment_state"
-  command: "{{ bin_dir }}/kubectl get deployment heketi -o=name --ignore-not-found=true"
-  changed_when: false
-- name: "Ensure heketi is up and running."
-  assert: { that: "heketi_deployment_state.stdout != \"\"", message: "Heketi deployment did not succeed." }
-

contrib/network-storage/heketi/roles/provision/tasks/storage.yml

@@ -0,0 +1,11 @@
+---
+- name: "Kubernetes Apps | Lay Down Heketi Storage"
+  become: true
+  vars: { nodes: "{{ groups['heketi-node'] }}" }
+  template: { src: "heketi-storage.json.j2", dest: "{{ kube_config_dir }}/heketi-storage.json" }
+  register: "rendering"
+- name: "Kubernetes Apps | Install and configure Heketi Storage"
+  kube:
+    name: "GlusterFS"
+    filename: "{{ kube_config_dir }}/heketi-storage.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"

contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml

@@ -16,6 +16,9 @@
   template:
     src: "storageclass.yml.j2"
     dest: "{{ kube_config_dir }}/storageclass.yml"
-- name: "Setup storage class."
+  register: "rendering"
-  when: "storageclass.stdout == \"\""
+- name: "Kubernetes Apps | Install and configure Storace Class"
-  command: "{{ bin_dir }}/kubectl create -f {{ kube_config_dir }}/storageclass.yml"
+  kube:
+    name: "GlusterFS"
+    filename: "{{ kube_config_dir }}/storageclass.yml"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"

contrib/network-storage/heketi/roles/provision/tasks/topology.yml

@@ -0,0 +1,25 @@
+---
+- name: "Get heketi topology."
+  register: "heketi_topology"
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology info --json"
+- name: "Render heketi topology template."
+  become: true
+  vars: { nodes: "{{ groups['heketi-node'] }}" }
+  register: "rendering"
+  template:
+    src: "topology.json.j2"
+    dest: "{{ kube_config_dir }}/topology.json"
+- name: "Copy topology configuration into container."
+  when: "rendering.changed"
+  command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ heketi_pod_name }}:/tmp/topology.json"
+- name: "Load heketi topology."
+  when: "rendering.changed"
+  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology load --json=/tmp/topology.json"
+- name: "Get heketi topology."
+  register: "heketi_topology"
+  changed_when: false
+  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology info --json"
+  until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length"
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/templates/heketi-storage.json.j2

@@ -0,0 +1,54 @@
+{
+  "apiVersion": "v1",
+  "kind": "List",
+  "items": [
+    {
+      "kind": "Endpoints",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi-storage-endpoints",
+        "creationTimestamp": null
+      },
+      "subsets": [
+{% set nodeblocks = [] %}
+{% for node in nodes %}
+{% set nodeblock %}
+        {
+          "addresses": [
+            {
+              "ip": "{{ hostvars[node]['ansible_facts']['default_ipv4']['address'] }}"
+            }
+          ],
+          "ports": [
+            {
+              "port": 1
+            }
+          ]
+        }
+{% endset %}
+{% if nodeblocks.append(nodeblock) %}{% endif %}
+{% endfor %}
+{{ nodeblocks|join(',') }}
+      ]
+    },
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "heketi-storage-endpoints",
+        "creationTimestamp": null
+      },
+      "spec": {
+        "ports": [
+          {
+            "port": 1,
+            "targetPort": 0
+          }
+        ]
+      },
+      "status": {
+        "loadBalancer": {}
+      }
+    }
+  ]
+}