diff --git a/contrib/network-storage/heketi/roles/heketi.yml b/contrib/network-storage/heketi/roles/heketi.yml new file mode 100644 index 000000000..8ac74b980 --- /dev/null +++ b/contrib/network-storage/heketi/roles/heketi.yml @@ -0,0 +1,10 @@ +--- +- hosts: heketi-node + roles: + - { role: prepare } + +- hosts: localhost + tags: + - "provision" + roles: + - { role: provision } diff --git a/contrib/network-storage/heketi/roles/prepare/tasks/main.yml b/contrib/network-storage/heketi/roles/prepare/tasks/main.yml new file mode 100644 index 000000000..cb644364c --- /dev/null +++ b/contrib/network-storage/heketi/roles/prepare/tasks/main.yml @@ -0,0 +1,16 @@ +--- +- name: "Load lvm kernel modules" + become: true + with_items: + - "dm_snapshot" + - "dm_mirror" + - "dm_thin_pool" + modprobe: + name: "{{ item }}" + state: "present" + +- name: "Install glusterfs mount utils" + become: true + yum: + name: "glusterfs-fuse" + state: "present" diff --git a/contrib/network-storage/heketi/roles/provision/defaults/main.yml b/contrib/network-storage/heketi/roles/provision/defaults/main.yml new file mode 100644 index 000000000..df38376d1 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/defaults/main.yml @@ -0,0 +1,2 @@ +--- +artifacts_dir: "{{ ansible_inventory_sources[0] | dirname }}/artifacts" diff --git a/contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json b/contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json new file mode 100644 index 000000000..3c057fddd --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json @@ -0,0 +1,144 @@ +{ + "kind": "DaemonSet", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "glusterfs", + "labels": { + "glusterfs": "deployment" + }, + "annotations": { + "description": "GlusterFS Daemon Set", + "tags": "glusterfs" + } + }, + "spec": { + "template": { + "metadata": { + "name": "glusterfs", + "labels": { + "glusterfs-node": "daemonset" + } + }, + "spec": { + "nodeSelector": { + "storagenode" : "glusterfs" + }, + "hostNetwork": true, + "containers": [ + { + "image": "gluster/gluster-centos:latest", + "imagePullPolicy": "Always", + "name": "glusterfs", + "volumeMounts": [ + { + "name": "glusterfs-heketi", + "mountPath": "/var/lib/heketi" + }, + { + "name": "glusterfs-run", + "mountPath": "/run" + }, + { + "name": "glusterfs-lvm", + "mountPath": "/run/lvm" + }, + { + "name": "glusterfs-etc", + "mountPath": "/etc/glusterfs" + }, + { + "name": "glusterfs-logs", + "mountPath": "/var/log/glusterfs" + }, + { + "name": "glusterfs-config", + "mountPath": "/var/lib/glusterd" + }, + { + "name": "glusterfs-dev", + "mountPath": "/dev" + }, + { + "name": "glusterfs-cgroup", + "mountPath": "/sys/fs/cgroup" + } + ], + "securityContext": { + "capabilities": {}, + "privileged": true + }, + "readinessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 60, + "exec": { + "command": [ + "/bin/bash", + "-c", + "systemctl status glusterd.service" + ] + } + }, + "livenessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 60, + "exec": { + "command": [ + "/bin/bash", + "-c", + "systemctl status glusterd.service" + ] + } + } + } + ], + "volumes": [ + { + "name": "glusterfs-heketi", + "hostPath": { + "path": "/var/lib/heketi" + } + }, + { + "name": "glusterfs-run" + }, + { + "name": "glusterfs-lvm", + "hostPath": { + "path": "/run/lvm" + } + }, + { + "name": "glusterfs-etc", + "hostPath": { + "path": "/etc/glusterfs" + } + }, + { + "name": "glusterfs-logs", + "hostPath": { + "path": "/var/log/glusterfs" + } + }, + { + "name": "glusterfs-config", + "hostPath": { + "path": "/var/lib/glusterd" + } + }, + { + "name": "glusterfs-dev", + "hostPath": { + "path": "/dev" + } + }, + { + "name": "glusterfs-cgroup", + "hostPath": { + "path": "/sys/fs/cgroup" + } + } + ] + } + } + } +} diff --git a/contrib/network-storage/heketi/roles/provision/handlers/main.yml b/contrib/network-storage/heketi/roles/provision/handlers/main.yml new file mode 100644 index 000000000..9e876de17 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: "stop port forwarding" + command: "killall " diff --git a/contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json b/contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json new file mode 100644 index 000000000..cbd2bfa8d --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json @@ -0,0 +1,133 @@ +{ + "kind": "List", + "apiVersion": "v1", + "items": [ + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "deploy-heketi", + "labels": { + "glusterfs": "heketi-service", + "deploy-heketi": "support" + }, + "annotations": { + "description": "Exposes Heketi Service" + } + }, + "spec": { + "selector": { + "name": "deploy-heketi" + }, + "ports": [ + { + "name": "deploy-heketi", + "port": 8080, + "targetPort": 8080 + } + ] + } + }, + { + "kind": "Deployment", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "deploy-heketi", + "labels": { + "glusterfs": "heketi-deployment", + "deploy-heketi": "deployment" + }, + "annotations": { + "description": "Defines how to deploy Heketi" + } + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "name": "deploy-heketi", + "labels": { + "name": "deploy-heketi", + "glusterfs": "heketi-pod", + "deploy-heketi": "pod" + } + }, + "spec": { + "serviceAccountName": "heketi-service-account", + "containers": [ + { + "image": "heketi/heketi:dev", + "imagePullPolicy": "Always", + "name": "deploy-heketi", + "env": [ + { + "name": "HEKETI_EXECUTOR", + "value": "kubernetes" + }, + { + "name": "HEKETI_DB_PATH", + "value": "/var/lib/heketi/heketi.db" + }, + { + "name": "HEKETI_FSTAB", + "value": "/var/lib/heketi/fstab" + }, + { + "name": "HEKETI_SNAPSHOT_LIMIT", + "value": "14" + }, + { + "name": "HEKETI_KUBE_GLUSTER_DAEMONSET", + "value": "y" + } + ], + "ports": [ + { + "containerPort": 8080 + } + ], + "volumeMounts": [ + { + "name": "db", + "mountPath": "/var/lib/heketi" + }, + { + "name": "config", + "mountPath": "/etc/heketi" + } + ], + "readinessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 3, + "httpGet": { + "path": "/hello", + "port": 8080 + } + }, + "livenessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 30, + "httpGet": { + "path": "/hello", + "port": 8080 + } + } + } + ], + "volumes": [ + { + "name": "db" + }, + { + "name": "config", + "secret": { + "secretName": "heketi-config-secret" + } + } + ] + } + } + } + } + ] +} diff --git a/contrib/network-storage/heketi/roles/provision/heketi-deployment.json b/contrib/network-storage/heketi/roles/provision/heketi-deployment.json new file mode 100644 index 000000000..6670bc874 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/heketi-deployment.json @@ -0,0 +1,159 @@ +{ + "kind": "List", + "apiVersion": "v1", + "items": [ + { + "kind": "Secret", + "apiVersion": "v1", + "metadata": { + "name": "heketi-db-backup", + "labels": { + "glusterfs": "heketi-db", + "heketi": "db" + } + }, + "data": { + }, + "type": "Opaque" + }, + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "heketi", + "labels": { + "glusterfs": "heketi-service", + "deploy-heketi": "support" + }, + "annotations": { + "description": "Exposes Heketi Service" + } + }, + "spec": { + "selector": { + "name": "heketi" + }, + "ports": [ + { + "name": "heketi", + "port": 8080, + "targetPort": 8080 + } + ] + } + }, + { + "kind": "Deployment", + "apiVersion": "extensions/v1beta1", + "metadata": { + "name": "heketi", + "labels": { + "glusterfs": "heketi-deployment" + }, + "annotations": { + "description": "Defines how to deploy Heketi" + } + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "name": "heketi", + "labels": { + "name": "heketi", + "glusterfs": "heketi-pod" + } + }, + "spec": { + "serviceAccountName": "heketi-service-account", + "containers": [ + { + "image": "heketi/heketi:dev", + "imagePullPolicy": "Always", + "name": "heketi", + "env": [ + { + "name": "HEKETI_EXECUTOR", + "value": "kubernetes" + }, + { + "name": "HEKETI_DB_PATH", + "value": "/var/lib/heketi/heketi.db" + }, + { + "name": "HEKETI_FSTAB", + "value": "/var/lib/heketi/fstab" + }, + { + "name": "HEKETI_SNAPSHOT_LIMIT", + "value": "14" + }, + { + "name": "HEKETI_KUBE_GLUSTER_DAEMONSET", + "value": "y" + } + ], + "ports": [ + { + "containerPort": 8080 + } + ], + "volumeMounts": [ + { + "mountPath": "/backupdb", + "name": "heketi-db-secret" + }, + { + "name": "db", + "mountPath": "/var/lib/heketi" + }, + { + "name": "config", + "mountPath": "/etc/heketi" + } + ], + "readinessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 3, + "httpGet": { + "path": "/hello", + "port": 8080 + } + }, + "livenessProbe": { + "timeoutSeconds": 3, + "initialDelaySeconds": 30, + "httpGet": { + "path": "/hello", + "port": 8080 + } + } + } + ], + "volumes": [ + { + "name": "db", + "glusterfs": { + "endpoints": "heketi-storage-endpoints", + "path": "heketidbstorage" + } + }, + { + "name": "heketi-db-secret", + "secret": { + "secretName": "heketi-db-backup" + } + }, + { + "name": "config", + "secret": { + "secretName": "heketi-config-secret" + } + } + ] + } + } + } + } + ] +} diff --git a/contrib/network-storage/heketi/roles/provision/heketi-service-account.json b/contrib/network-storage/heketi/roles/provision/heketi-service-account.json new file mode 100644 index 000000000..1dbcb9e96 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/heketi-service-account.json @@ -0,0 +1,7 @@ +{ + "apiVersion": "v1", + "kind": "ServiceAccount", + "metadata": { + "name": "heketi-service-account" + } +} diff --git a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml new file mode 100644 index 000000000..d78ce8f2e --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml @@ -0,0 +1,54 @@ +--- +- register: "daemonset_state" + command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true" + changed_when: false +- name: "Deploy the GlusterFS DaemonSet" + when: "daemonset_state.stdout == \"\"" + command: "kubectl create -f {{ role_path }}/glusterfs-daemonset.json" +- register: "daemonset_state" + command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true" + changed_when: false +- assert: { that: "daemonset_state.stdout != \"\"", message: "Daemonset glusterfs is not present." } + +- name: "Label Gluster nodes" + with_items: "{{ groups['heketi-node'] }}" + loop_control: + loop_var: "node" + include_tasks: "kubernetes/label.yml" + +- register: "service_account_state" + command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" + changed_when: false +- name: "Deploy the Heketi service account" + when: "service_account_state.stdout == \"\"" + command: "kubectl create -f {{ role_path }}/heketi-service-account.json" +- register: "service_account_state" + command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true" + changed_when: false +- assert: { that: "service_account_state.stdout != \"\"", message: "Heketi service account is not present." } + +- register: "clusterrolebinding_state" + command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" + changed_when: false +- name: "Deploy cluster role binding." + when: "clusterrolebinding_state.stdout == \"\"" + command: "kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account" +- register: "clusterrolebinding_state" + command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true" + changed_when: false +- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." } + +- register: "secret_state" + command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" + changed_when: false +- name: "Render Heketi secret configuration." + template: + src: "heketi.json.j2" + dest: "{{ artifacts_dir }}/heketi.json" +- name: "Deploy Heketi config secret" + when: "secret_state.stdout == \"\"" + command: "kubectl create secret generic heketi-config-secret --from-file={{ artifacts_dir }}/heketi.json" +- register: "secret_state" + command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true" + changed_when: false +- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml new file mode 100644 index 000000000..7ff9ee4ae --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml @@ -0,0 +1,11 @@ +--- +- register: "label_present" + command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true" + changed_when: false +- name: "Assign storage label" + when: "label_present.stdout_lines|length == 0" + command: "kubectl label node {{ node }} storagenode=glusterfs" +- register: "label_present" + command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true" + changed_when: false +- assert: { that: "label_present|length > 0", msg: "Node {{ node }} has not been assigned with label storagenode=glusterfs." } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/main.yml b/contrib/network-storage/heketi/roles/provision/tasks/main.yml new file mode 100644 index 000000000..983f5e672 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/main.yml @@ -0,0 +1,26 @@ +--- +- name: "Prepare kubernetes." + include_tasks: "kubernetes.yml" + +- name: "Test heketi setup." + register: "heketi_service_state" + command: "kubectl get service heketi -o=name --ignore-not-found=true" + changed_when: false + +- name: "Setup heketi." + when: "heketi_service_state.stdout == \"\"" + include_tasks: "setup.yml" + +- name: "Test storage class." + changed_when: false + command: "kubectl get storageclass gluster --ignore-not-found=true --output=json" + register: "storageclass" +- name: "Setup storage class." + when: "storageclass.stdout == \"\"" + include_tasks: "storageclass.yml" +- name: "Test storage class." + changed_when: false + command: "kubectl get storageclass gluster --ignore-not-found=true --output=json" + register: "storageclass" +- name: "Ensure storage class is up." + assert: { that: "storageclass.stdout != \"\"" } diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup.yml new file mode 100644 index 000000000..1de5073a7 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup.yml @@ -0,0 +1,51 @@ +--- +- name: "Test REST endpoint." + uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true } + register: "rest_hello_check" + ignore_errors: true + +# Bootstrap heketi +- name: "Bootstrap heketi and start REST endpoint." + when: "rest_hello_check.content != \"Hello from Heketi\"" + include_tasks: "{{ item }}" + with_items: [ "setup/boot.yml", "setup/rest.yml" ] +- name: "Bootstrap heketi." + when: "rest_hello_check.content == \"Hello from Heketi\"" + include_tasks: "setup/boot.yml" + +# Prepare heketi topology +- name: "Test heketi topology." + changed_when: false + register: "heketi_topology" + command: "heketi-cli -s http://localhost:48080 topology info --json" +- name: "Load heketi topology." + when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" + include_tasks: "setup/topology.yml" + +# Provision heketi database volume +- name: "Prepare heketi volumes." + include_tasks: "setup/volumes.yml" + +# Prepare heketi storage +- name: "Test heketi storage." + command: "kubectl get secrets,endpoints,services,jobs --output=json" + changed_when: false + register: "heketi_storage_state" +- name: "Create heketi storage." + include_tasks: "setup/storage.yml" + vars: + secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" + endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" + service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']" + when: + - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" + +# Finalize setup +- name: "Tear down bootstrap." + include_tasks: "setup/tear-down-bootstrap.yml" +- name: "Setup final heketi instance." + include_tasks: "setup/heketi.yml" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml new file mode 100644 index 000000000..3de6922f1 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml @@ -0,0 +1,34 @@ +--- +- name: "Get state of heketi service, deployment and pods." + register: "initial_heketi_state" + changed_when: false + command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" +- name: "Create Heketi initial service and deployment" + command: "kubectl create -f {{ role_path }}/heketi-bootstrap.json" + when: + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0" + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0" + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0" +- name: "Get state of heketi service, deployment and pods." + register: "initial_heketi_state" + changed_when: false + command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" +- name: "Ensure heketi bootstrap environment exists." + assert: + that: + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service'].metadata.name\")).0 == 'deploy-heketi'" + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment'].metadata.name\")).0 == 'deploy-heketi'" + - "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.labels.name\")).0 == 'deploy-heketi'" + msg: "Heketi deployment did not succeed." +- name: "Wait for heketi bootstrap to complete." + changed_when: false + register: "initial_heketi_state" + vars: + pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]" + deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]" + command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json" + until: + - "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'" + - "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'" + retries: 10 + delay: 10 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml new file mode 100644 index 000000000..ff43568ca --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml @@ -0,0 +1,10 @@ +--- +- name: "Create long term Heketi instance." + command: "kubectl create -f {{ role_path }}/heketi-deployment.json" +- name: "Get heketi deployment state." + register: "heketi_deployment_state" + command: "kubectl get deployment heketi -o=name --ignore-not-found=true" + changed_when: false +- name: "Ensure heketi is up and running." + assert: { that: "heketi_deployment_state.stdout != \"\"", message: "Heketi deployment did not succeed." } + diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml new file mode 100644 index 000000000..b78395eeb --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml @@ -0,0 +1,38 @@ +--- +# Enable local REST-Interface +- name: "Get heketi initial pod state." + tags: ["test"] + register: "initial_heketi_pod" + command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json" + changed_when: false +- name: "Ensure heketi bootstrap pod is up." + tags: ["test"] + assert: + that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1" +- name: "Temporarily enable local port forwarding to heketi REST interface" + tags: ["test"] + vars: + initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}" + command: "kubectl port-forward {{ initial_heketi_pod_name }} 48080:8080" + async: 180 + poll: 0 + ignore_errors: "yes" + register: "heketi_port_forwarding" + changed_when: false +- name: "Ensure port forwarding is enabled." + tags: ["test"] + retries: 10 + delay: 5 + assert: + that: + - "heketi_port_forwarding.finished == 0" + - "heketi_port_forwarding.started == 1" + - "heketi_port_forwarding.failed == false" + msg: "Port forwarding does not work." +- name: "Test REST endpoint." + tags: ["test"] + uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true } + register: "rest_hello_check" + until: "rest_hello_check.content == \"Hello from Heketi\"" + retries: 10 + delay: 10 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml new file mode 100644 index 000000000..3a2d0df8a --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml @@ -0,0 +1,36 @@ +--- +- name: "Test heketi storage." + tags: ["test"] + command: "kubectl get secrets,endpoints,services,jobs --output=json" + changed_when: false + register: "heketi_storage_state" +- name: "Create heketi storage." + tags: ["test"] + command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json" + vars: + secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" + endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" + service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']" + when: + - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0" + - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0" +- name: "Get state of heketi storage service, endpoint, secret and job." + tags: ["test"] + command: "kubectl get secrets,endpoints,services,jobs --output=json" + changed_when: false + register: "heketi_storage_state" + vars: + secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']" + endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']" + service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']" + job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.conditions[?type=='Complete']]|[0].status.conditions|[0].status" + until: + - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1" + - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1" + - "heketi_storage_state.stdout|from_json|json_query(service_query)|length > 0" + - "heketi_storage_state.stdout|from_json|json_query(job_query) == 'True'" + retries: 10 + delay: 10 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml new file mode 100644 index 000000000..019f92dfa --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml @@ -0,0 +1,14 @@ +--- +- name: "Get existing Heketi deploy resources." + command: "kubectl get all --selector=\"deploy-heketi\" -o=json" + register: "heketi_resources" + changed_when: false +- name: "Delete bootstrap Heketi." + command: "kubectl delete all,service,jobs,deployment,secret --selector=\"deploy-heketi\"" + when: "heketi_resources.stdout|from_json|json_query('items[*]')|length > 0" +- name: "Ensure there is nothing left over." + command: "kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json" + register: "heketi_result" + until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0" + retries: 10 + delay: 5 diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml new file mode 100644 index 000000000..cc73fd62a --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml @@ -0,0 +1,20 @@ +--- +- name: "Get heketi topology." + register: "heketi_topology" + command: "heketi-cli -s http://localhost:48080 topology info --json" +- name: "Render heketi topology template." + vars: { nodes: "{{ groups['heketi-node'] }}" } + template: + src: "topology.json.j2" + dest: "{{ artifacts_dir }}/topology.json" +- name: "Load heketi topology." + when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0" + command: "heketi-cli -s http://localhost:48080 topology load --json={{ artifacts_dir }}/topology.json" +- name: "Get heketi topology." + register: "heketi_topology" + command: "heketi-cli -s http://localhost:48080 topology info --json" +- set_fact: { heketi_volumes: "{{ heketi_topology.stdout|from_json|json_query(\"clusters[*].volumes[?name=='heketidbstorage']\") }}" } +- name: "Ensure heketi nodes are configured." + assert: + that: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length > 0" + msg: "Heketi topology missing." diff --git a/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml new file mode 100644 index 000000000..d7b710c10 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml @@ -0,0 +1,20 @@ +--- +- name: "Get heketi volume ids." + command: "heketi-cli -s http://localhost:48080 volume list --json" + changed_when: false + register: "heketi_volumes" +- name: "Get heketi volumes." + changed_when: false + command: "heketi-cli -s http://localhost:48080 volume info {{ volume_id }} --json" + with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}" + loop_control: { loop_var: "volume_id" } + register: "volumes_information" +- name: "Test heketi database volume." + set_fact: { heketi_database_volume_exists: true } + with_items: "{{ volumes_information.results }}" + loop_control: { loop_var: "volume_information" } + vars: { volume: "{{ volume_information.stdout|from_json }}" } + when: "volume.name == 'heketidbstorage'" +- name: "Provision database volume." + command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json" + when: "heketi_database_volume_exists != true" diff --git a/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml new file mode 100644 index 000000000..d2d8e7725 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml @@ -0,0 +1,20 @@ +--- +- name: "Test storage class." + command: "kubectl get storageclass gluster --ignore-not-found=true --output=json" + register: "storageclass" + changed_when: false +- name: "Test heketi service." + command: "kubectl get service heketi --ignore-not-found=true --output=json" + register: "heketi_service" + changed_when: false +- name: "Ensure heketi service is available." + assert: { that: "heketi_service.stdout != \"\"" } +- name: "Render storage class configuration." + vars: + endpoint_address: "{{ (heketi_service.stdout|from_json).spec.clusterIP }}" + template: + src: "storageclass.yml.j2" + dest: "{{ artifacts_dir }}/storageclass.yml" +- name: "Setup storage class." + when: "storageclass.stdout == \"\"" + command: "kubectl create -f {{ artifacts_dir }}/storageclass.yml" diff --git a/contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 new file mode 100644 index 000000000..7ba980dca --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 @@ -0,0 +1,44 @@ +{ + "_port_comment": "Heketi Server Port Number", + "port": "8080", + + "_use_auth": "Enable JWT authorization. Please enable for deployment", + "use_auth": false, + + "_jwt": "Private keys for access", + "jwt": { + "_admin": "Admin has access to all APIs", + "admin": { + "key": "{{ heketi_admin_key }}" + }, + "_user": "User only has access to /volumes endpoint", + "user": { + "key": "{{ heketi_user_key }}" + } + }, + + "_glusterfs_comment": "GlusterFS Configuration", + "glusterfs": { + "_executor_comment": "Execute plugin. Possible choices: mock, kubernetes, ssh", + "executor": "kubernetes", + + "_db_comment": "Database file name", + "db": "/var/lib/heketi/heketi.db", + + "kubeexec": { + "rebalance_on_expansion": true + }, + + "sshexec": { + "rebalance_on_expansion": true, + "keyfile": "/etc/heketi/private_key", + "fstab": "/etc/fstab", + "port": "22", + "user": "root", + "sudo": false + } + }, + + "_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.", + "backup_db_to_kube_secret": false +} diff --git a/contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 b/contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 new file mode 100644 index 000000000..5eec1f5d1 --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: gluster +provisioner: kubernetes.io/glusterfs +parameters: + resturl: "http://{{ endpoint_address }}:8080" + restuser: "user" + restuserkey: "{{ heketi_user_key }}" diff --git a/contrib/network-storage/heketi/roles/provision/templates/topology.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/topology.json.j2 new file mode 100644 index 000000000..b0ac29d7b --- /dev/null +++ b/contrib/network-storage/heketi/roles/provision/templates/topology.json.j2 @@ -0,0 +1,34 @@ +{ + "clusters": [ + { + "nodes": [ +{% set nodeblocks = [] %} +{% for node in nodes %} +{% set nodeblock %} + { + "node": { + "hostnames": { + "manage": [ + "{{ node }}" + ], + "storage": [ + "{{ hostvars[node]['ansible_facts']['default_ipv4']['address'] }}" + ] + }, + "zone": 1 + }, + "devices": [ + { + "name": "{{ hostvars[node]['disk_volume_device_1'] }}", + "destroydata": false + } + ] + } +{% endset %} +{% if nodeblocks.append(nodeblock) %}{% endif %} +{% endfor %} +{{ nodeblocks|join(',') }} + ] + } + ] +}