prevent some race conditions, increase over all time limits

contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml

@@ -6,9 +6,17 @@
   when: "daemonset_state.stdout == \"\""
   command: "kubectl create -f {{ role_path }}/glusterfs-daemonset.json"
 - register: "daemonset_state"
-  command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true"
+  command: "kubectl get daemonset glusterfs --output=json --ignore-not-found=true"
+- name: "Wait for daemonset to become available."
+  register: "daemonset_state"
+  command: "kubectl get daemonset glusterfs --output=json --ignore-not-found=true"
   changed_when: false
-- assert: { that: "daemonset_state.stdout != \"\"", message: "Daemonset glusterfs is not present." }
+  vars:
+    ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}"
+    desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}"
+  until: "ready == desired"
+  retries: 60
+  delay: 5
 
 - name: "Label Gluster nodes"
   with_items: "{{ groups['heketi-node'] }}"

contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml

@@ -30,5 +30,5 @@
   until:
       - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'"
       - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'"
-  retries: 10
-  delay: 10
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml

@@ -29,5 +29,5 @@
   uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true }
   register: "rest_hello_check"
   until: "rest_hello_check.content == \"Hello from Heketi\""
-  retries: 10
-  delay: 10
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml

@@ -15,6 +15,8 @@
     - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
     - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
     - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
+- command: "kubectl get secrets,endpoints,services,jobs --output=json"
+  register: "heketi_storage_state"
 - name: "Get state of heketi storage service, endpoint, secret and job."
   command: "kubectl get secrets,endpoints,services,jobs --output=json"
   changed_when: false
@@ -23,11 +25,11 @@
     secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
     endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
     service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
-    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']"
+    job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.active==0]"
   until:
     - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1"
     - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1"
     - "heketi_storage_state.stdout|from_json|json_query(service_query)|length > 0"
     - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 1"
-  retries: 10
-  delay: 10
+  retries: 60
+  delay: 5

contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml

@@ -10,5 +10,5 @@
   command: "kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json"
   register: "heketi_result"
   until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
-  retries: 10
+  retries: 60
   delay: 5

contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml

@@ -13,8 +13,6 @@
 - name: "Get heketi topology."
   register: "heketi_topology"
   command: "heketi-cli -s http://localhost:48080 topology info --json"
-- set_fact: { heketi_volumes: "{{ heketi_topology.stdout|from_json|json_query(\"clusters[*].volumes[?name=='heketidbstorage']\") }}" }
-- name: "Ensure heketi nodes are configured."
-  assert:
-    that: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length > 0"
-    msg: "Heketi topology missing."
+  until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length"
+  retries: 60
+  delay: 5