C12s/c12s-kubespray
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

run kubectl on one of the masters

Browse source
This commit is contained in:
Sascha Marcel Schmidt 2018-07-25 16:42:30 +02:00
parent 306a6a751f
commit c42397d7db
No known key found for this signature in database
GPG key ID: D1D6CE184437796D
17 changed files with 55 additions and 510 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
contrib/network-storage/heketi/heketi.yml
View file

@ -3,7 +3,7 @@
roles:
- { role: prepare }
- hosts: localhost
- hosts: kube-master[0]
tags:
- "provision"
roles:

1
contrib/network-storage/heketi/roles/provision/defaults/main.yml
View file

@ -1,2 +1 @@
---
artifacts_dir: "{{ ansible_inventory_sources[0] | dirname }}/artifacts"

144
contrib/network-storage/heketi/roles/provision/glusterfs-daemonset.json
View file

@ -1,144 +0,0 @@
{
"kind": "DaemonSet",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "glusterfs",
"labels": {
"glusterfs": "deployment"
},
"annotations": {
"description": "GlusterFS Daemon Set",
"tags": "glusterfs"
}
},
"spec": {
"template": {
"metadata": {
"name": "glusterfs",
"labels": {
"glusterfs-node": "daemonset"
}
},
"spec": {
"nodeSelector": {
"storagenode" : "glusterfs"
},
"hostNetwork": true,
"containers": [
{
"image": "gluster/gluster-centos:gluster4u0_centos7",
"imagePullPolicy": "Always",
"name": "glusterfs",
"volumeMounts": [
{
"name": "glusterfs-heketi",
"mountPath": "/var/lib/heketi"
},
{
"name": "glusterfs-run",
"mountPath": "/run"
},
{
"name": "glusterfs-lvm",
"mountPath": "/run/lvm"
},
{
"name": "glusterfs-etc",
"mountPath": "/etc/glusterfs"
},
{
"name": "glusterfs-logs",
"mountPath": "/var/log/glusterfs"
},
{
"name": "glusterfs-config",
"mountPath": "/var/lib/glusterd"
},
{
"name": "glusterfs-dev",
"mountPath": "/dev"
},
{
"name": "glusterfs-cgroup",
"mountPath": "/sys/fs/cgroup"
}
],
"securityContext": {
"capabilities": {},
"privileged": true
},
"readinessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 60,
"exec": {
"command": [
"/bin/bash",
"-c",
"systemctl status glusterd.service"
]
}
},
"livenessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 60,
"exec": {
"command": [
"/bin/bash",
"-c",
"systemctl status glusterd.service"
]
}
}
}
],
"volumes": [
{
"name": "glusterfs-heketi",
"hostPath": {
"path": "/var/lib/heketi"
}
},
{
"name": "glusterfs-run"
},
{
"name": "glusterfs-lvm",
"hostPath": {
"path": "/run/lvm"
}
},
{
"name": "glusterfs-etc",
"hostPath": {
"path": "/etc/glusterfs"
}
},
{
"name": "glusterfs-logs",
"hostPath": {
"path": "/var/log/glusterfs"
}
},
{
"name": "glusterfs-config",
"hostPath": {
"path": "/var/lib/glusterd"
}
},
{
"name": "glusterfs-dev",
"hostPath": {
"path": "/dev"
}
},
{
"name": "glusterfs-cgroup",
"hostPath": {
"path": "/sys/fs/cgroup"
}
}
]
}
}
}
}

133
contrib/network-storage/heketi/roles/provision/heketi-bootstrap.json
View file

@ -1,133 +0,0 @@
{
"kind": "List",
"apiVersion": "v1",
"items": [
{
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": "deploy-heketi",
"labels": {
"glusterfs": "heketi-service",
"deploy-heketi": "support"
},
"annotations": {
"description": "Exposes Heketi Service"
}
},
"spec": {
"selector": {
"name": "deploy-heketi"
},
"ports": [
{
"name": "deploy-heketi",
"port": 8080,
"targetPort": 8080
}
]
}
},
{
"kind": "Deployment",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "deploy-heketi",
"labels": {
"glusterfs": "heketi-deployment",
"deploy-heketi": "deployment"
},
"annotations": {
"description": "Defines how to deploy Heketi"
}
},
"spec": {
"replicas": 1,
"template": {
"metadata": {
"name": "deploy-heketi",
"labels": {
"name": "deploy-heketi",
"glusterfs": "heketi-pod",
"deploy-heketi": "pod"
}
},
"spec": {
"serviceAccountName": "heketi-service-account",
"containers": [
{
"image": "heketi/heketi:7",
"imagePullPolicy": "Always",
"name": "deploy-heketi",
"env": [
{
"name": "HEKETI_EXECUTOR",
"value": "kubernetes"
},
{
"name": "HEKETI_DB_PATH",
"value": "/var/lib/heketi/heketi.db"
},
{
"name": "HEKETI_FSTAB",
"value": "/var/lib/heketi/fstab"
},
{
"name": "HEKETI_SNAPSHOT_LIMIT",
"value": "14"
},
{
"name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
"value": "y"
}
],
"ports": [
{
"containerPort": 8080
}
],
"volumeMounts": [
{
"name": "db",
"mountPath": "/var/lib/heketi"
},
{
"name": "config",
"mountPath": "/etc/heketi"
}
],
"readinessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 3,
"httpGet": {
"path": "/hello",
"port": 8080
}
},
"livenessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 30,
"httpGet": {
"path": "/hello",
"port": 8080
}
}
}
],
"volumes": [
{
"name": "db"
},
{
"name": "config",
"secret": {
"secretName": "heketi-config-secret"
}
}
]
}
}
}
}
]
}

159
contrib/network-storage/heketi/roles/provision/heketi-deployment.json
View file

@ -1,159 +0,0 @@
{
"kind": "List",
"apiVersion": "v1",
"items": [
{
"kind": "Secret",
"apiVersion": "v1",
"metadata": {
"name": "heketi-db-backup",
"labels": {
"glusterfs": "heketi-db",
"heketi": "db"
}
},
"data": {
},
"type": "Opaque"
},
{
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": "heketi",
"labels": {
"glusterfs": "heketi-service",
"deploy-heketi": "support"
},
"annotations": {
"description": "Exposes Heketi Service"
}
},
"spec": {
"selector": {
"name": "heketi"
},
"ports": [
{
"name": "heketi",
"port": 8080,
"targetPort": 8080
}
]
}
},
{
"kind": "Deployment",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "heketi",
"labels": {
"glusterfs": "heketi-deployment"
},
"annotations": {
"description": "Defines how to deploy Heketi"
}
},
"spec": {
"replicas": 1,
"template": {
"metadata": {
"name": "heketi",
"labels": {
"name": "heketi",
"glusterfs": "heketi-pod"
}
},
"spec": {
"serviceAccountName": "heketi-service-account",
"containers": [
{
"image": "heketi/heketi:7",
"imagePullPolicy": "Always",
"name": "heketi",
"env": [
{
"name": "HEKETI_EXECUTOR",
"value": "kubernetes"
},
{
"name": "HEKETI_DB_PATH",
"value": "/var/lib/heketi/heketi.db"
},
{
"name": "HEKETI_FSTAB",
"value": "/var/lib/heketi/fstab"
},
{
"name": "HEKETI_SNAPSHOT_LIMIT",
"value": "14"
},
{
"name": "HEKETI_KUBE_GLUSTER_DAEMONSET",
"value": "y"
}
],
"ports": [
{
"containerPort": 8080
}
],
"volumeMounts": [
{
"mountPath": "/backupdb",
"name": "heketi-db-secret"
},
{
"name": "db",
"mountPath": "/var/lib/heketi"
},
{
"name": "config",
"mountPath": "/etc/heketi"
}
],
"readinessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 3,
"httpGet": {
"path": "/hello",
"port": 8080
}
},
"livenessProbe": {
"timeoutSeconds": 3,
"initialDelaySeconds": 30,
"httpGet": {
"path": "/hello",
"port": 8080
}
}
}
],
"volumes": [
{
"name": "db",
"glusterfs": {
"endpoints": "heketi-storage-endpoints",
"path": "heketidbstorage"
}
},
{
"name": "heketi-db-secret",
"secret": {
"secretName": "heketi-db-backup"
}
},
{
"name": "config",
"secret": {
"secretName": "heketi-config-secret"
}
}
]
}
}
}
}
]
}

7
contrib/network-storage/heketi/roles/provision/heketi-service-account.json
View file

@ -1,7 +0,0 @@
{
"apiVersion": "v1",
"kind": "ServiceAccount",
"metadata": {
"name": "heketi-service-account"
}
}

9
contrib/network-storage/heketi/roles/provision/tasks/kubernetes.yml
View file

@ -4,7 +4,7 @@
changed_when: false
- name: "Deploy the GlusterFS DaemonSet"
when: "daemonset_state.stdout == \"\""
command: "kubectl create -f {{ role_path }}/glusterfs-daemonset.json"
command: "kubectl create -f {{ kube_config_dir }}/glusterfs-daemonset.json"
- register: "daemonset_state"
command: "kubectl get daemonset glusterfs --output=json --ignore-not-found=true"
changed_when: false
@ -30,7 +30,7 @@
changed_when: false
- name: "Deploy the Heketi service account"
when: "service_account_state.stdout == \"\""
command: "kubectl create -f {{ role_path }}/heketi-service-account.json"
command: "kubectl create -f {{ kube_config_dir }}/heketi-service-account.json"
- register: "service_account_state"
command: "kubectl get serviceaccount heketi-service-account -o=name --ignore-not-found=true"
changed_when: false
@ -51,12 +51,13 @@
command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
changed_when: false
- name: "Render Heketi secret configuration."
become: true
template:
src: "heketi.json.j2"
dest: "{{ artifacts_dir }}/heketi.json"
dest: "{{ kube_config_dir }}/heketi.json"
- name: "Deploy Heketi config secret"
when: "secret_state.stdout == \"\""
command: "kubectl create secret generic heketi-config-secret --from-file={{ artifacts_dir }}/heketi.json"
command: "kubectl create secret generic heketi-config-secret --from-file={{ kube_config_dir }}/heketi.json"
- register: "secret_state"
command: "kubectl get secret heketi-config-secret -o=name --ignore-not-found=true"
changed_when: false

8
contrib/network-storage/heketi/roles/provision/tasks/main.yml
View file

@ -1,4 +1,12 @@
---
- name: "Render configuration."
become: true
template: { src: "{{ item.file }}.j2", dest: "{{ kube_config_dir }}/{{ item.file }}" }
with_items:
- { file: "glusterfs-daemonset.json" }
- { file: "heketi-bootstrap.json" }
- { file: "heketi-deployment.json" }
- { file: "heketi-service-account.json" }
- name: "Prepare kubernetes."
include_tasks: "kubernetes.yml"

18
contrib/network-storage/heketi/roles/provision/tasks/setup.yml
View file

@ -10,16 +10,21 @@
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0"
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod']\"))|length == 0"
include_tasks: "setup/boot.yml"
- name: "Test rest endpoint port."
wait_for: { host: "localhost", port: "48080", state: "absent" }
- name: "Start REST endpoint."
include_tasks: "setup/rest.yml"
# Prepare heketi topology
- name: "Get heketi initial pod state."
register: "initial_heketi_pod"
command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json"
changed_when: false
- name: "Ensure heketi bootstrap pod is up."
assert:
that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1"
- set_fact:
initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}"
- name: "Test heketi topology."
changed_when: false
register: "heketi_topology"
command: "heketi-cli -s http://localhost:48080 topology info --json"
command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json"
- name: "Load heketi topology."
when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
include_tasks: "setup/topology.yml"
@ -33,6 +38,8 @@
command: "kubectl get secrets,endpoints,services,jobs --output=json"
changed_when: false
register: "heketi_storage_state"
- command: "kubectl get secrets,endpoints,services,jobs --output=json"
register: "job"
- name: "Create heketi storage."
include_tasks: "setup/storage.yml"
vars:
@ -45,7 +52,6 @@
- "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
- "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
- "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
# Finalize setup
- name: "Tear down bootstrap."
include_tasks: "setup/tear-down-bootstrap.yml"

9
contrib/network-storage/heketi/roles/provision/tasks/setup/boot.yml
View file

@ -4,7 +4,7 @@
changed_when: false
command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
- name: "Create Heketi initial service and deployment"
command: "kubectl create -f {{ role_path }}/heketi-bootstrap.json"
command: "kubectl create -f {{ kube_config_dir }}/heketi-bootstrap.json"
when:
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service']\"))|length == 0"
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment']\"))|length == 0"
@ -13,13 +13,6 @@
register: "initial_heketi_state"
changed_when: false
command: "kubectl get services,deployments,pods --selector=deploy-heketi --output=json"
- name: "Ensure heketi bootstrap environment exists."
assert:
that:
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Service'].metadata.name\")).0 == 'deploy-heketi'"
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Deployment'].metadata.name\")).0 == 'deploy-heketi'"
- "(initial_heketi_state.stdout|from_json|json_query(\"items[?kind=='Pod'].metadata.labels.name\")).0 == 'deploy-heketi'"
msg: "Heketi deployment did not succeed."
- name: "Wait for heketi bootstrap to complete."
changed_when: false
register: "initial_heketi_state"

2
contrib/network-storage/heketi/roles/provision/tasks/setup/heketi.yml
View file

@ -1,6 +1,6 @@
---
- name: "Create long term Heketi instance."
command: "kubectl create -f {{ role_path }}/heketi-deployment.json"
command: "kubectl create -f {{ kube_config_dir }}/heketi-deployment.json"
- name: "Get heketi deployment state."
register: "heketi_deployment_state"
command: "kubectl get deployment heketi -o=name --ignore-not-found=true"

33
contrib/network-storage/heketi/roles/provision/tasks/setup/rest.yml
View file

@ -1,33 +0,0 @@
---
# Enable local REST-Interface
- name: "Get heketi initial pod state."
register: "initial_heketi_pod"
command: "kubectl get pods --selector=deploy-heketi=pod,glusterfs=heketi-pod,name=deploy-heketi --output=json"
changed_when: false
- name: "Ensure heketi bootstrap pod is up."
assert:
that: "(initial_heketi_pod.stdout|from_json|json_query('items[*]'))|length == 1"
- name: "Temporarily enable local port forwarding to heketi REST interface"
vars:
initial_heketi_pod_name: "{{ initial_heketi_pod.stdout|from_json|json_query(\"items[*].metadata.name|[0]\") }}"
command: "kubectl port-forward {{ initial_heketi_pod_name }} 48080:8080"
async: 600
poll: 0
ignore_errors: "yes"
register: "heketi_port_forwarding"
changed_when: false
- name: "Ensure port forwarding is enabled."
retries: 10
delay: 5
assert:
that:
- "heketi_port_forwarding.finished == 0"
- "heketi_port_forwarding.started == 1"
- "heketi_port_forwarding.failed == false"
msg: "Port forwarding does not work."
- name: "Test REST endpoint."
uri: { url: "http://localhost:48080/hello", method: "GET", return_content: true }
register: "rest_hello_check"
until: "rest_hello_check.content == \"Hello from Heketi\""
retries: 60
delay: 5

2
contrib/network-storage/heketi/roles/provision/tasks/setup/storage.yml
View file

@ -4,7 +4,7 @@
changed_when: false
register: "heketi_storage_state"
- name: "Create heketi storage."
command: "kubectl create -f {{ artifacts_dir }}/heketi-storage.json"
command: "kubectl create -f {{ kube_config_dir }}/heketi-storage.json"
vars:
secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"

11
contrib/network-storage/heketi/roles/provision/tasks/setup/topology.yml
View file

@ -1,18 +1,21 @@
---
- name: "Get heketi topology."
register: "heketi_topology"
command: "heketi-cli -s http://localhost:48080 topology info --json"
command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json"
- name: "Render heketi topology template."
become: true
vars: { nodes: "{{ groups['heketi-node'] }}" }
template:
src: "topology.json.j2"
dest: "{{ artifacts_dir }}/topology.json"
dest: "{{ kube_config_dir }}/topology.json"
- name: "Copy topology configuration into container."
command: "kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json"
- name: "Load heketi topology."
when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
command: "heketi-cli -s http://localhost:48080 topology load --json={{ artifacts_dir }}/topology.json"
command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology load --json=/tmp/topology.json"
- name: "Get heketi topology."
register: "heketi_topology"
command: "heketi-cli -s http://localhost:48080 topology info --json"
command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli topology info --json"
until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length"
retries: 60
delay: 5

20
contrib/network-storage/heketi/roles/provision/tasks/setup/volumes.yml
View file

@ -1,11 +1,11 @@
---
- name: "Get heketi volume ids."
command: "heketi-cli -s http://localhost:48080 volume list --json"
command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume list --json"
changed_when: false
register: "heketi_volumes"
- name: "Get heketi volumes."
changed_when: false
command: "heketi-cli -s http://localhost:48080 volume info {{ volume_id }} --json"
command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume info {{ volume_id }} --json"
with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}"
loop_control: { loop_var: "volume_id" }
register: "volumes_information"
@ -16,19 +16,27 @@
vars: { volume: "{{ volume_information.stdout|from_json }}" }
when: "volume.name == 'heketidbstorage'"
- name: "Provision database volume."
command: "heketi-cli -s http://localhost:48080 setup-openshift-heketi-storage"
args: { chdir: "{{ artifacts_dir }}" }
command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli setup-openshift-heketi-storage"
when: "heketi_database_volume_exists is undefined"
- name: "Copy configuration from pod."
become: true
command: "/usr/local/bin/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage.json"
- name: "Get heketi volume ids."
command: "heketi-cli -s http://localhost:48080 volume list --json"
command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume list --json"
changed_when: false
register: "heketi_volumes"
- name: "Get heketi volumes."
changed_when: false
command: "heketi-cli -s http://localhost:48080 volume info {{ volume_id }} --json"
command: "kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli volume info {{ volume_id }} --json"
with_items: "{{ heketi_volumes.stdout|from_json|json_query(\"volumes[*]\") }}"
loop_control: { loop_var: "volume_id" }
register: "volumes_information"
- name: "debug heketi db vol."
with_items: "{{ volumes_information.results }}"
loop_control: { loop_var: "volume_information" }
vars: { volume: "{{ volume_information.stdout|from_json }}" }
when: "volume.name == 'heketidbstorage'"
debug: { var: "volume" }
- name: "Test heketi database volume."
set_fact: { heketi_database_volume_created: true }
with_items: "{{ volumes_information.results }}"

5
contrib/network-storage/heketi/roles/provision/tasks/storageclass.yml
View file

@ -10,11 +10,12 @@
- name: "Ensure heketi service is available."
assert: { that: "heketi_service.stdout != \"\"" }
- name: "Render storage class configuration."
become: true
vars:
endpoint_address: "{{ (heketi_service.stdout|from_json).spec.clusterIP }}"
template:
src: "storageclass.yml.j2"
dest: "{{ artifacts_dir }}/storageclass.yml"
dest: "{{ kube_config_dir }}/storageclass.yml"
- name: "Setup storage class."
when: "storageclass.stdout == \"\""
command: "kubectl create -f {{ artifacts_dir }}/storageclass.yml"
command: "kubectl create -f {{ kube_config_dir }}/storageclass.yml"

2
contrib/network-storage/heketi/roles/provision/templates/storageclass.yml.j2
View file

@ -3,6 +3,8 @@ apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: gluster
annotations:
storageclass.beta.kubernetes.io/is-default-class: "true"
provisioner: kubernetes.io/glusterfs
parameters:
resturl: "http://{{ endpoint_address }}:8080"