C12s/c12s-kubespray
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add heketi/glusterfs as additional contributional network storage

Browse source
This commit is contained in:
Sascha Marcel Schmidt 2018-06-25 13:23:20 +02:00
parent 73a2a18006
commit 9d2fabc9b9
No known key found for this signature in database
GPG key ID: D1D6CE184437796D
23 changed files with 896 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
contrib/network-storage/heketi/roles/heketi.yml Normal file
View file

@ -0,0 +1,10 @@
---
- hosts: heketi-node
roles:
- { role: prepare }
- hosts: localhost
tags:
- "provision"
roles:
- { role: provision }

16
contrib/network-storage/heketi/roles/prepare/tasks/main.yml Normal file
View file

@ -0,0 +1,16 @@
---
- name: "Load lvm kernel modules"
become: true
with_items:
- "dm_snapshot"
- "dm_mirror"
- "dm_thin_pool"
modprobe:
name: "{{ item }}"
state: "present"
- name: "Install glusterfs mount utils"
become: true
yum:
name: "glusterfs-fuse"
state: "present"

2
contrib/network-storage/heketi/roles/provision/defaults/main.yml Normal file
View file

@ -0,0 +1,2 @@
---
artifacts_dir: "{{ ansible_inventory_sources[0] | dirname }}/artifacts"

144
contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json Normal file
View file

@ -0,0 +1,144 @@
{
"kind": "DaemonSet",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "glusterfs",
"labels": {
"glusterfs": "deployment"
},
"annotations": {
"description": "GlusterFS Daemon Set",
"tags": "glusterfs"
}
},
"spec": {
"template": {
"metadata": {
"name": "glusterfs",
"labels": {
"glusterfs-node": "daemonset"
}
},
"spec": {
"nodeSelector": {
"storagenode" : "glusterfs"
},
"hostNetwork": true,
"containers": [
{
"image": "gluster/gluster-centos:latest",
"imagePullPolicy": "Always",
"name": "glusterfs",
"volumeMounts": [
{
"name": "glusterfs-heketi",
"mountPath": "/var/lib/heketi"
},
{
"name": "glusterfs-run",
"mountPath": "/run"
},
{
"name": "glusterfs-lvm",
"mountPath": "/run/lvm"
},
{
"name": "glusterfs-etc",
"mountPath": "/etc/glusterfs"
},
{
"name": "glusterfs-logs",
"mountPath": "/var/log/glusterfs"
},
{
"name": "glusterfs-config",
"mountPath": "/var/lib/glusterd"
},
{
"name": "glusterfs-dev",
"mountPath": "/dev"
},
{
"name": "glusterfs-cgroup",
"mountPath": "/sys/fs/cgroup"
}
],
"securityContext": {
"capabilities": {},
"privileged": true
},
"readinessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 60,
"exec": {
"command": [
"/bin/bash",
"-c",
"systemctl status glusterd.service"
]
}
},
"livenessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 60,
"exec": {
"command": [
"/bin/bash",
"-c",
"systemctl status glusterd.service"
]
}
}
}
],
"volumes": [
{
"name": "glusterfs-heketi",
"hostPath": {
"path": "/var/lib/heketi"
}
},
{
"name": "glusterfs-run"
},
{
"name": "glusterfs-lvm",
"hostPath": {
"path": "/run/lvm"
}
},
{
"name": "glusterfs-etc",
"hostPath": {
"path": "/etc/glusterfs"
}
},
{
"name": "glusterfs-logs",
"hostPath": {
"path": "/var/log/glusterfs"
}
},
{
"name": "glusterfs-config",
"hostPath": {
"path": "/var/lib/glusterd"
}
},
{
"name": "glusterfs-dev",
"hostPath": {
"path": "/dev"
}
},
{
"name": "glusterfs-cgroup",
"hostPath": {
"path": "/sys/fs/cgroup"
}
}
]
}
}
}
}

3
contrib/network-storage/heketi/roles/provision/handlers/main.yml Normal file
View file

@ -0,0 +1,3 @@
---
- name: "stop port forwarding"
command: "killall "

133
contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json Normal file
View file

@ -0,0 +1,133 @@
{
"kind": "List",
"apiVersion": "v1",
"items": [
{
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": "deploy-heketi",
"labels": {
"glusterfs": "heketi-service",
"deploy-heketi": "support"
},
"annotations": {
"description": "Exposes Heketi Service"
}
},
"spec": {
"selector": {
"name": "deploy-heketi"
},
"ports": [
{
"name": "deploy-heketi",
"port": 8080,
"targetPort": 8080
}
]
}
},
{
"kind": "Deployment",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "deploy-heketi",
"labels": {
"glusterfs": "heketi-deployment",
"deploy-heketi": "deployment"
},
"annotations": {
"description": "Defines how to deploy Heketi"
}
},
"spec": {
"replicas": 1,
"template": {
"metadata": {
"name": "deploy-heketi",
"labels": {
"name": "deploy-heketi",
"glusterfs": "heketi-pod",
"deploy-heketi": "pod"
}
},
"spec": {
"serviceAccountName": "heketi-service-account",
"containers": [
{
"image": "heketi/heketi:dev",
"imagePullPolicy": "Always",
"name": "deploy-heketi",
"env": [
{
"name": "HEKETI_EXECUTOR",
"value": "kubernetes"
},
{
"name": "HEKETI_DB_PATH",
"value": "/var/lib/heketi/heketi.db"
},
{
"name": "HEKETI_FSTAB",
"value": "/var/lib/heketi/fstab"
},
{
"name": "HEKETI_SNAPSHOT_LIMIT",
"value": "14"
},
{
"name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
"value": "y"
}
],
"ports": [
{
"containerPort": 8080
}
],
"volumeMounts": [
{
"name": "db",
"mountPath": "/var/lib/heketi"
},
{
"name": "config",
"mountPath": "/etc/heketi"
}
],
"readinessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 3,
"httpGet": {
"path": "/hello",
"port": 8080
}
},
"livenessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 30,
"httpGet": {
"path": "/hello",
"port": 8080
}
}
}
],
"volumes": [
{
"name": "db"
},
{
"name": "config",
"secret": {
"secretName": "heketi-config-secret"
}
}
]
}
}
}
}
]
}

159
contrib/network-storage/heketi/roles/provision/heketi-deployment.json Normal file
View file

@ -0,0 +1,159 @@
{
"kind": "List",
"apiVersion": "v1",
"items": [
{
"kind": "Secret",
"apiVersion": "v1",
"metadata": {
"name": "heketi-db-backup",
"labels": {
"glusterfs": "heketi-db",
"heketi": "db"
}
},
"data": {
},
"type": "Opaque"
},
{
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": "heketi",
"labels": {
"glusterfs": "heketi-service",
"deploy-heketi": "support"
},
"annotations": {
"description": "Exposes Heketi Service"
}
},
"spec": {
"selector": {
"name": "heketi"
},
"ports": [
{
"name": "heketi",
"port": 8080,
"targetPort": 8080
}
]
}
},
{
"kind": "Deployment",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "heketi",
"labels": {
"glusterfs": "heketi-deployment"
},
"annotations": {
"description": "Defines how to deploy Heketi"
}
},
"spec": {
"replicas": 1,
"template": {
"metadata": {
"name": "heketi",
"labels": {
"name": "heketi",
"glusterfs": "heketi-pod"
}
},
"spec": {
"serviceAccountName": "heketi-service-account",
"containers": [
{
"image": "heketi/heketi:dev",
"imagePullPolicy": "Always",
"name": "heketi",
"env": [
{
"name": "HEKETI_EXECUTOR",
"value": "kubernetes"
},
{
"name": "HEKETI_DB_PATH",
"value": "/var/lib/heketi/heketi.db"
},
{
"name": "HEKETI_FSTAB",
"value": "/var/lib/heketi/fstab"
},
{
"name": "HEKETI_SNAPSHOT_LIMIT",
"value": "14"
},
{
"name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
"value": "y"
}
],
"ports": [
{
"containerPort": 8080
}
],
"volumeMounts": [
{
"mountPath": "/backupdb",
"name": "heketi-db-secret"
},
{
"name": "db",
"mountPath": "/var/lib/heketi"
},
{
"name": "config",
"mountPath": "/etc/heketi"
}
],
"readinessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 3,
"httpGet": {
"path": "/hello",
"port": 8080
}
},
"livenessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 30,
"httpGet": {
"path": "/hello",
"port": 8080
}
}
}
],
"volumes": [
{
"name": "db",
"glusterfs": {
"endpoints": "heketi-storage-endpoints",
"path": "heketidbstorage"
}
},
{
"name": "heketi-db-secret",
"secret": {
"secretName": "heketi-db-backup"
}
},
{
"name": "config",
"secret": {
"secretName": "heketi-config-secret"
}
}
]
}
}
}
}
]
}

7
contrib/network-storage/heketi/roles/provision/heketi-service-account.json Normal file
View file

@ -0,0 +1,7 @@
{
"apiVersion": "v1",
"kind": "ServiceAccount",
"metadata": {
"name": "heketi-service-account"
}
}

54
contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml Normal file
View file

@ -0,0 +1,54 @@
---
- register: "daemonset_state"
command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true"
changed_when: false
- name: "Deploy the GlusterFS DaemonSet"
when: "daemonset_state.stdout == \"\""
command: "kubectl create -f {{ role_path }}/glusterfs-daemonset.json"
- register: "daemonset_state"
command: "kubectl get daemonset glusterfs -o=name --ignore-not-found=true"
changed_when: false
- assert: { that: "daemonset_state.stdout != \"\"", message: "Daemonset glusterfs is not present." }
- name: "Label Gluster nodes"
with_items: "{{ groups['heketi-node'] }}"
loop_control:
loop_var: "node"
include_tasks: "kubernetes/label.yml"
- register: "service_account_state"
command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true"
changed_when: false
- name: "Deploy the Heketi service account"
when: "service_account_state.stdout == \"\""
command: "kubectl create -f {{ role_path }}/heketi-service-account.json"
- register: "service_account_state"
command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true"
changed_when: false
- assert: { that: "service_account_state.stdout != \"\"", message: "Heketi service account is not present." }
- register: "clusterrolebinding_state"
command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
changed_when: false
- name: "Deploy cluster role binding."
when: "clusterrolebinding_state.stdout == \"\""
command: "kubectl create clusterrolebinding heketi-gluster-admin --clusterrole=edit --serviceaccount=default:heketi-service-account"
- register: "clusterrolebinding_state"
command: "kubectl get clusterrolebinding heketi-gluster-admin -o=name --ignore-not-found=true"
changed_when: false
- assert: { that: "clusterrolebinding_state.stdout != \"\"", message: "Cluster role binding is not present." }
- register: "secret_state"
command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
changed_when: false
- name: "Render Heketi secret configuration."
template:
src: "heketi.json.j2"
dest: "{{ artifacts_dir }}/heketi.json"
- name: "Deploy Heketi config secret"
when: "secret_state.stdout == \"\""
command: "kubectl create secret generic heketi-config-secret --from-file={{ artifacts_dir }}/heketi.json"
- register: "secret_state"
command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
changed_when: false
- assert: { that: "secret_state.stdout != \"\"", message: "Heketi config secret is not present." }

11
contrib/network-storage/heketi/roles/provision/tasks/kubernetes/label.yml Normal file
View file

@ -0,0 +1,11 @@
---
- register: "label_present"
command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true"
changed_when: false
- name: "Assign storage label"
when: "label_present.stdout_lines|length == 0"
command: "kubectl label node {{ node }} storagenode=glusterfs"
- register: "label_present"
command: "kubectl get node --selector=storagenode=glusterfs,kubernetes.io/hostname={{ node }} --ignore-not-found=true"
changed_when: false
- assert: { that: "label_present|length > 0", msg: "Node {{ node }} has not been assigned with label storagenode=glusterfs." }

26
contrib/network-storage/heketi/roles/provision/tasks/main.yml Normal file
View file

@ -0,0 +1,26 @@
---
- name: "Prepare kubernetes."
include_tasks: "kubernetes.yml"
- name: "Test heketi setup."
register: "heketi_service_state"
command: "kubectl get service heketi -o=name --ignore-not-found=true"
changed_when: false
- name: "Setup heketi."
when: "heketi_service_state.stdout == \"\""
include_tasks: "setup.yml"
- name: "Test storage class."
changed_when: false
command: "kubectl get storageclass gluster --ignore-not-found=true --output=json"
register: "storageclass"
- name: "Setup storage class."
when: "storageclass.stdout == \"\""
include_tasks: "storageclass.yml"
- name: "Test storage class."
changed_when: false
command: "kubectl get storageclass gluster --ignore-not-found=true --output=json"
register: "storageclass"
- name: "Ensure storage class is up."
assert: { that: "storageclass.stdout != \"\"" }

51
contrib/network-storage/heketi/roles/provision/tasks/setup.yml Normal file
View file

@ -0,0 +1,51 @@
---
- name: "Test REST endpoint."
uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true }
register: "rest_hello_check"
ignore_errors: true
# Bootstrap heketi
- name: "Bootstrap heketi and start REST endpoint."
when: "rest_hello_check.content != \"Hello from Heketi\""
include_tasks: "{{ item }}"
with_items: [ "setup/boot.yml", "setup/rest.yml" ]
- name: "Bootstrap heketi."
when: "rest_hello_check.content == \"Hello from Heketi\""
include_tasks: "setup/boot.yml"
# Prepare heketi topology
- name: "Test heketi topology."
changed_when: false
register: "heketi_topology"
command: "heketi-cli -s http://localhost:48080 topology info --json"
- name: "Load heketi topology."
when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
include_tasks: "setup/topology.yml"
# Provision heketi database volume
- name: "Prepare heketi volumes."
include_tasks: "setup/volumes.yml"
# Prepare heketi storage
- name: "Test heketi storage."
command: "kubectl get secrets,endpoints,services,jobs --output=json"
changed_when: false
register: "heketi_storage_state"
- name: "Create heketi storage."
include_tasks: "setup/storage.yml"
vars:
secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']"
when:
- "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0"
- "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
- "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
- "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
# Finalize setup
- name: "Tear down bootstrap."
include_tasks: "setup/tear-down-bootstrap.yml"
- name: "Setup final heketi instance."
include_tasks: "setup/heketi.yml"

34
contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml Normal file
View file

@ -0,0 +1,34 @@
---
- name: "Get state of heketi service, deployment and pods."
register: "initial_heketi_state"
changed_when: false
command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
- name: "Create Heketi initial service and deployment"
command: "kubectl create -f {{ role_path }}/heketi-bootstrap.json"
when:
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0"
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0"
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0"
- name: "Get state of heketi service, deployment and pods."
register: "initial_heketi_state"
changed_when: false
command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
- name: "Ensure heketi bootstrap environment exists."
assert:
that:
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service'].metadata.name\")).0 == 'deploy-heketi'"
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment'].metadata.name\")).0 == 'deploy-heketi'"
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.labels.name\")).0 == 'deploy-heketi'"
msg: "Heketi deployment did not succeed."
- name: "Wait for heketi bootstrap to complete."
changed_when: false
register: "initial_heketi_state"
vars:
pods_query: "items[?kind=='Pod'].status.conditions|[0][?type=='Ready'].status|[0]"
deployments_query: "items[?kind=='Deployment'].status.conditions|[0][?type=='Available'].status|[0]"
command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
until:
- "initial_heketi_state.stdout|from_json|json_query(pods_query) == 'True'"
- "initial_heketi_state.stdout|from_json|json_query(deployments_query) == 'True'"
retries: 10
delay: 10

10
contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml Normal file
View file

@ -0,0 +1,10 @@
---
- name: "Create long term Heketi instance."
command: "kubectl create -f {{ role_path }}/heketi-deployment.json"
- name: "Get heketi deployment state."
register: "heketi_deployment_state"
command: "kubectl get deployment heketi -o=name --ignore-not-found=true"
changed_when: false
- name: "Ensure heketi is up and running."
assert: { that: "heketi_deployment_state.stdout != \"\"", message: "Heketi deployment did not succeed." }

38
contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml Normal file
View file

@ -0,0 +1,38 @@
---
# Enable local REST-Interface
- name: "Get heketi initial pod state."
tags: ["test"]
register: "initial_heketi_pod"
command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json"
changed_when: false
- name: "Ensure heketi bootstrap pod is up."
tags: ["test"]
assert:
that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1"
- name: "Temporarily enable local port forwarding to heketi REST interface"
tags: ["test"]
vars:
initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}"
command: "kubectl port-forward {{ initial_heketi_pod_name }} 48080:8080"
async: 180
poll: 0
ignore_errors: "yes"
register: "heketi_port_forwarding"
changed_when: false
- name: "Ensure port forwarding is enabled."
tags: ["test"]
retries: 10
delay: 5
assert:
that:
- "heketi_port_forwarding.finished == 0"
- "heketi_port_forwarding.started == 1"
- "heketi_port_forwarding.failed == false"
msg: "Port forwarding does not work."
- name: "Test REST endpoint."
tags: ["test"]
uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true }
register: "rest_hello_check"
until: "rest_hello_check.content == \"Hello from Heketi\""
retries: 10
delay: 10

36
contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml Normal file
View file

@ -0,0 +1,36 @@
---
- name: "Test heketi storage."
tags: ["test"]
command: "kubectl get secrets,endpoints,services,jobs --output=json"
changed_when: false
register: "heketi_storage_state"
- name: "Create heketi storage."
tags: ["test"]
command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json"
vars:
secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job']"
when:
- "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 0"
- "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
- "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
- "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
- name: "Get state of heketi storage service, endpoint, secret and job."
tags: ["test"]
command: "kubectl get secrets,endpoints,services,jobs --output=json"
changed_when: false
register: "heketi_storage_state"
vars:
secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.conditions[?type=='Complete']]|[0].status.conditions|[0].status"
until:
- "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1"
- "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1"
- "heketi_storage_state.stdout|from_json|json_query(service_query)|length > 0"
- "heketi_storage_state.stdout|from_json|json_query(job_query) == 'True'"
retries: 10
delay: 10

14
contrib/network-storage/heketi/roles/provision/tasks/setup/tear-down-bootstrap.yml Normal file
View file

@ -0,0 +1,14 @@
---
- name: "Get existing Heketi deploy resources."
command: "kubectl get all --selector=\"deploy-heketi\" -o=json"
register: "heketi_resources"
changed_when: false
- name: "Delete bootstrap Heketi."
command: "kubectl delete all,service,jobs,deployment,secret --selector=\"deploy-heketi\""
when: "heketi_resources.stdout|from_json|json_query('items[*]')|length > 0"
- name: "Ensure there is nothing left over."
command: "kubectl get all,service,jobs,deployment,secret --selector=\"deploy-heketi\" -o=json"
register: "heketi_result"
until: "heketi_result.stdout|from_json|json_query('items[*]')|length == 0"
retries: 10
delay: 5

20
contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml Normal file
View file

@ -0,0 +1,20 @@
---
- name: "Get heketi topology."
register: "heketi_topology"
command: "heketi-cli -s http://localhost:48080 topology info --json"
- name: "Render heketi topology template."
vars: { nodes: "{{ groups['heketi-node'] }}" }
template:
src: "topology.json.j2"
dest: "{{ artifacts_dir }}/topology.json"
- name: "Load heketi topology."
when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
command: "heketi-cli -s http://localhost:48080 topology load --json={{ artifacts_dir }}/topology.json"
- name: "Get heketi topology."
register: "heketi_topology"
command: "heketi-cli -s http://localhost:48080 topology info --json"
- set_fact: { heketi_volumes: "{{ heketi_topology.stdout|from_json|json_query(\"clusters[*].volumes[?name=='heketidbstorage']\") }}" }
- name: "Ensure heketi nodes are configured."
assert:
that: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length > 0"
msg: "Heketi topology missing."

20
contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml Normal file
View file

@ -0,0 +1,20 @@
---
- name: "Get heketi volume ids."
command: "heketi-cli -s http://localhost:48080 volume list --json"
changed_when: false
register: "heketi_volumes"
- name: "Get heketi volumes."
changed_when: false
command: "heketi-cli -s http://localhost:48080 volume info {{ volume_id }} --json"
with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}"
loop_control: { loop_var: "volume_id" }
register: "volumes_information"
- name: "Test heketi database volume."
set_fact: { heketi_database_volume_exists: true }
with_items: "{{ volumes_information.results }}"
loop_control: { loop_var: "volume_information" }
vars: { volume: "{{ volume_information.stdout|from_json }}" }
when: "volume.name == 'heketidbstorage'"
- name: "Provision database volume."
command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json"
when: "heketi_database_volume_exists != true"

20
contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml Normal file
View file

@ -0,0 +1,20 @@
---
- name: "Test storage class."
command: "kubectl get storageclass gluster --ignore-not-found=true --output=json"
register: "storageclass"
changed_when: false
- name: "Test heketi service."
command: "kubectl get service heketi --ignore-not-found=true --output=json"
register: "heketi_service"
changed_when: false
- name: "Ensure heketi service is available."
assert: { that: "heketi_service.stdout != \"\"" }
- name: "Render storage class configuration."
vars:
endpoint_address: "{{ (heketi_service.stdout|from_json).spec.clusterIP }}"
template:
src: "storageclass.yml.j2"
dest: "{{ artifacts_dir }}/storageclass.yml"
- name: "Setup storage class."
when: "storageclass.stdout == \"\""
command: "kubectl create -f {{ artifacts_dir }}/storageclass.yml"

44
contrib/network-storage/heketi/roles/provision/templates/heketi.json.j2 Normal file
View file

@ -0,0 +1,44 @@
{
"_port_comment": "Heketi Server Port Number",
"port": "8080",
"_use_auth": "Enable JWT authorization. Please enable for deployment",
"use_auth": false,
"_jwt": "Private keys for access",
"jwt": {
"_admin": "Admin has access to all APIs",
"admin": {
"key": "{{ heketi_admin_key }}"
},
"_user": "User only has access to /volumes endpoint",
"user": {
"key": "{{ heketi_user_key }}"
}
},
"_glusterfs_comment": "GlusterFS Configuration",
"glusterfs": {
"_executor_comment": "Execute plugin. Possible choices: mock, kubernetes, ssh",
"executor": "kubernetes",
"_db_comment": "Database file name",
"db": "/var/lib/heketi/heketi.db",
"kubeexec": {
"rebalance_on_expansion": true
},
"sshexec": {
"rebalance_on_expansion": true,
"keyfile": "/etc/heketi/private_key",
"fstab": "/etc/fstab",
"port": "22",
"user": "root",
"sudo": false
}
},
"_backup_db_to_kube_secret": "Backup the heketi database to a Kubernetes secret when running in Kubernetes. Default is off.",
"backup_db_to_kube_secret": false
}

10
contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2 Normal file
View file

@ -0,0 +1,10 @@
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: gluster
provisioner: kubernetes.io/glusterfs
parameters:
resturl: "http://{{ endpoint_address }}:8080"
restuser: "user"
restuserkey: "{{ heketi_user_key }}"

34
contrib/network-storage/heketi/roles/provision/templates/topology.json.j2 Normal file
View file

@ -0,0 +1,34 @@
{
"clusters": [
{
"nodes": [
{% set nodeblocks = [] %}
{% for node in nodes %}
{% set nodeblock %}
{
"node": {
"hostnames": {
"manage": [
"{{ node }}"
],
"storage": [
"{{ hostvars[node]['ansible_facts']['default_ipv4']['address'] }}"
]
},
"zone": 1
},
"devices": [
{
"name": "{{ hostvars[node]['disk_volume_device_1'] }}",
"destroydata": false
}
]
}
{% endset %}
{% if nodeblocks.append(nodeblock) %}{% endif %}
{% endfor %}
{{ nodeblocks|join(',') }}
]
}
]
}