Adds documentation on how to use it from terraform

cluster.yml

@@ -12,7 +12,7 @@
   any_errors_fatal: true
   gather_facts: true
 
-- hosts: all
+- hosts: all:!network-storage
   any_errors_fatal: true
   roles:
     - { role: kubernetes/preinstall, tags: preinstall }

contrib/network-storage/glusterfs/README.md

@@ -0,0 +1,92 @@
+# Deploying a Kargo Kubernetes Cluster with GlusterFS
+
+You can either deploy using Ansible on its own by supplying your own inventory file or by using Terraform to create the VMs and then providing a dynamic inventory to Ansible. The following two sections are self-contained, you don't need to go through one to use the other. So, if you want to provision with Terraform, you can skip the **Using an Ansible inventory** section, and if you want to provision with a pre-built ansible inventory, you can neglect the **Using Terraform and Ansible**  section.
+
+## Using an Ansible inventory
+
+In the same directory of this ReadMe file you should find a file named `inventory.example` which contains an example setup. Please note that, additionally to the Kubernetes nodes/masters, we define a set of machines for GlusterFS and we add them to the group `[gfs-cluster]`, which in turn is added to the larger `[network-storage]` group as a child group.
+
+Change that file to reflect your local setup (adding more machines or removing them and setting the adequate ip numbers), and save it to `inventory/k8s_gfs_inventory`. Make sure that the settings on `inventory/group_vars/all.yml` make sense with your deployment. Then execute change to the kargo root folder, and execute (supposing that the machines are all using ubuntu):
+
+```
+ansible-playbook -b --become-user=root -i inventory/k8s_gfs_inventory --user=ubuntu ./cluster.yml
+```
+
+This will provision your Kubernetes cluster. Then, to provision and configure the GlusterFS cluster, from the same directory execute:
+
+```
+ansible-playbook -b --become-user=root -i inventory/k8s_gfs_inventory --user=ubuntu ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If your machines are not using Ubuntu, you need to change the `--user=ubuntu` to the correct user. Alternatively, if your Kubernetes machines are using one OS and your GlusterFS a different one, you can instead specify the `ansible_ssh_user=<correct-user>` variable in the inventory file that you just created, for each machine/VM:
+
+```
+k8s-master-1 ansible_ssh_host=192.168.0.147 ip=192.168.0.147 ansible_ssh_user=core
+k8s-master-node-1 ansible_ssh_host=192.168.0.148 ip=192.168.0.148 ansible_ssh_user=core
+k8s-master-node-2 ansible_ssh_host=192.168.0.146 ip=192.168.0.146 ansible_ssh_user=core
+```
+
+## Using Terraform and Ansible
+
+First step is to fill in a `my-kargo-gluster-cluster.tfvars` file with the specification desired for your cluster. An example with all required variables would look like:
+
+```
+cluster_name = "cluster1"
+number_of_k8s_masters = "1"
+number_of_k8s_masters_no_floating_ip = "2"
+number_of_k8s_nodes_no_floating_ip = "0"
+number_of_k8s_nodes = "0"
+public_key_path = "~/.ssh/my-desired-key.pub"
+image = "Ubuntu 16.04"
+ssh_user = "ubuntu"
+flavor_k8s_node = "node-flavor-id-in-your-openstack"
+flavor_k8s_master = "master-flavor-id-in-your-openstack"
+network_name = "k8s-network"
+floatingip_pool = "net_external"
+
+# GlusterFS variables
+flavor_gfs_node = "gluster-flavor-id-in-your-openstack"
+image_gfs = "Ubuntu 16.04"
+number_of_gfs_nodes_no_floating_ip = "3"
+gfs_volume_size_in_gb = "50"
+ssh_user_gfs = "ubuntu"
+```
+
+As explained in the general terraform/openstack guide, you need to source your OpenStack credentials file, add your ssh-key to the ssh-agent and setup environment variables for terraform:
+
+```
+$ source ~/.stackrc
+$ eval $(ssh-agent -s)
+$ ssh-add ~/.ssh/my-desired-key
+$ echo Setting up Terraform creds && \
+  export TF_VAR_username=${OS_USERNAME} && \
+  export TF_VAR_password=${OS_PASSWORD} && \
+  export TF_VAR_tenant=${OS_TENANT_NAME} && \
+  export TF_VAR_auth_url=${OS_AUTH_URL}
+```
+
+Then, standing on the kargo directory (root base of the Git checkout), issue the following terraform command to create the VMs for the cluster:
+
+```
+terraform apply -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kargo-gluster-cluster.tfvars contrib/terraform/openstack
+```
+
+This will create both your Kubernetes and Gluster VMs. Make sure that the ansible file `contrib/terraform/openstack/group_vars/all.yml` includes any ansible variable that you want to setup (like, for instance, the type of machine for bootstrapping).
+
+Then, provision your Kubernetes (Kargo) cluster with the following ansible call:
+
+```
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./cluster.yml
+```
+
+Finally, provision the glusterfs nodes and add the Persistent Volume setup for GlusterFS in Kubernetes through the following ansible call:
+
+```
+ansible-playbook -b --become-user=root -i contrib/terraform/openstack/hosts ./contrib/network-storage/glusterfs/glusterfs.yml
+```
+
+If you need to destroy the cluster, you can run:
+
+```
+terraform destroy -state=contrib/terraform/openstack/terraform.tfstate -var-file=my-kargo-gluster-cluster.tfvars contrib/terraform/openstack
+```

contrib/network-storage/glusterfs/glusterfs.yml

@@ -0,0 +1,17 @@
+---
+- hosts: all
+  gather_facts: true
+
+- hosts: gfs-cluster
+  roles:
+    - { role: glusterfs/server }
+
+- hosts: k8s-cluster
+  roles:
+    - { role: glusterfs/client }
+
+- hosts: kube-master[0]
+  roles:
+    - { role: kubernetes-pv/lib }
+    - { role: kubernetes-pv }
+

contrib/network-storage/glusterfs/inventory.example

@@ -0,0 +1,44 @@
+# ## Configure 'ip' variable to bind kubernetes services on a
+# ## different ip than the default iface
+# node1 ansible_ssh_host=95.54.0.12  # ip=10.3.0.1
+# node2 ansible_ssh_host=95.54.0.13  # ip=10.3.0.2
+# node3 ansible_ssh_host=95.54.0.14  # ip=10.3.0.3
+# node4 ansible_ssh_host=95.54.0.15  # ip=10.3.0.4
+# node5 ansible_ssh_host=95.54.0.16  # ip=10.3.0.5
+# node6 ansible_ssh_host=95.54.0.17  # ip=10.3.0.6
+#
+# ## GlusterFS nodes
+# ## Set disk_volume_device_1 to desired device for gluster brick, if different to /dev/vdb (default).
+# ## As in the previous case, you can set ip to give direct communication on internal IPs
+# gfs_node1 ansible_ssh_host=95.54.0.18 # disk_volume_device_1=/dev/vdc  ip=10.3.0.7
+# gfs_node2 ansible_ssh_host=95.54.0.19 # disk_volume_device_1=/dev/vdc  ip=10.3.0.8 
+# gfs_node1 ansible_ssh_host=95.54.0.20 # disk_volume_device_1=/dev/vdc  ip=10.3.0.9 
+
+# [kube-master]
+# node1
+# node2
+
+# [etcd]
+# node1
+# node2
+# node3
+
+# [kube-node]
+# node2
+# node3
+# node4
+# node5
+# node6
+
+# [k8s-cluster:children]
+# kube-node
+# kube-master
+
+# [gfs-cluster]
+# gfs_node1
+# gfs_node2
+# gfs_node3
+
+# [network-storage:children]
+# gfs-cluster
+

contrib/network-storage/glusterfs/roles/glusterfs/client/defaults/main.yml

@@ -2,7 +2,7 @@
 # For Ubuntu.
 glusterfs_default_release: ""
 glusterfs_ppa_use: yes
-glusterfs_ppa_version: "3.5"
+glusterfs_ppa_version: "3.8"
 
 # Gluster configuration.
 gluster_mount_dir: /mnt/gluster

contrib/network-storage/glusterfs/roles/glusterfs/server/defaults/main.yml

@@ -2,10 +2,12 @@
 # For Ubuntu.
 glusterfs_default_release: ""
 glusterfs_ppa_use: yes
-glusterfs_ppa_version: "3.5"
+glusterfs_ppa_version: "3.8"
 
 # Gluster configuration.
 gluster_mount_dir: /mnt/gluster
 gluster_volume_node_mount_dir: /mnt/xfs-drive-gluster
 gluster_brick_dir: "{{ gluster_volume_node_mount_dir }}/brick"
 gluster_brick_name: gluster
+# Default device to mount for xfs formatting, terraform overrides this by setting the variable in the inventory.
+disk_volume_device_1: /dev/vdb

contrib/network-storage/glusterfs/roles/glusterfs/server/tasks/main.yml

@@ -14,7 +14,7 @@
 
 # Format external volumes in xfs
 - name: Format volumes in xfs
-  filesystem: fstype=xfs dev=/dev/vdb
+  filesystem: "fstype=xfs dev={{ disk_volume_device_1 }}"
 
 # Mount external volumes
 - name: mounting new xfs filesystem
@@ -42,7 +42,7 @@
         name: "{{ gluster_brick_name }}"
         brick: "{{ gluster_brick_dir }}"
         replicas: "{{ groups['gfs-cluster'] | length }}"
-        cluster: "{{ groups['gfs-cluster'] | map('extract', hostvars, ['ip']) | join(',') }}"
+        cluster: "{% for item in groups['gfs-cluster'] -%}{{ hostvars[item]['ip']|default(hostvars[item].ansible_default_ipv4['address']) }}{% if not loop.last %},{% endif %}{%- endfor %}"
         host: "{{ inventory_hostname }}"
         force: yes
   run_once: true
@@ -50,7 +50,7 @@
 - name: Mount glusterfs to retrieve disk size
   mount:
     name: "{{ gluster_mount_dir }}"
-    src: "{{ hostvars[groups['gfs-cluster'][0]]['ip'] }}:/gluster" 
+    src: "{{ ip|default(ansible_default_ipv4['address']) }}:/gluster" 
     fstype: glusterfs
     opts: "defaults,_netdev"
     state: mounted
@@ -66,12 +66,17 @@
      gluster_disk_size_gb: "{{ (mounts_data.ansible_facts.ansible_mounts | selectattr('mount', 'equalto', gluster_mount_dir) | map(attribute='size_total') | first | int / (1024*1024*1024)) | int }}"
   when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
 
+- name: Create file on GlusterFS
+  template:
+      dest: "{{ gluster_mount_dir }}/.test-file.txt"
+      src: test-file.txt
+  when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
 
 - name: Unmount glusterfs
   mount:
     name: "{{ gluster_mount_dir }}"
     fstype: glusterfs
-    src: "{{ hostvars[groups['gfs-cluster'][0]]['ip'] }}:/gluster"
+    src: "{{ ip|default(ansible_default_ipv4['address']) }}:/gluster"
     state: unmounted
   when: groups['gfs-cluster'] is defined and inventory_hostname == groups['gfs-cluster'][0]
 

contrib/network-storage/glusterfs/roles/glusterfs/server/templates/test-file.txt

@@ -0,0 +1 @@
+test file

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/tasks/main.yaml

@@ -0,0 +1,20 @@
+---
+- name: Kubernetes Apps | Lay Down k8s GlusterFS Endpoint and PV
+  template: src={{item.file}} dest=/etc/kubernetes/{{item.dest}}
+  with_items:
+          - { file: glusterfs-kubernetes-endpoint.json.j2, type: ep, dest: glusterfs-kubernetes-endpoint.json}
+          - { file: glusterfs-kubernetes-pv.yml.j2, type: pv, dest: glusterfs-kubernetes-pv.yml}
+ow to deploy it.
+  register: gluster_pv
+  when: inventory_hostname == groups['kube-master'][0] and groups['gfs-cluster'] is defined and hostvars[groups['gfs-cluster'][0]].gluster_disk_size_gb is defined
+
+- name: Kubernetes Apps | Set GlusterFS endpoint and PV
+  kube:
+    name: glusterfs
+    namespace: default
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "{{item.item.type}}"
+    filename: "/etc/kubernetes/{{item.item.dest}}"
+    state: "{{item.changed | ternary('latest','present') }}"
+  with_items: "{{ gluster_pv.results }}"
+  when: inventory_hostname == groups['kube-master'][0] and groups['gfs-cluster'] is defined

contrib/network-storage/glusterfs/roles/kubernetes-pv/ansible/templates/glusterfs-kubernetes-endpoint.json.j2

@@ -8,8 +8,8 @@
     {% for host in groups['gfs-cluster'] %}
     {
       "addresses": [
-        { 
-          "ip": "{{hostvars[host]['ip']}}"
+        {
+          "ip": "{{hostvars[host]['ip']|default(hostvars[host].ansible_default_ipv4['address'])}}"
         }
       ],
       "ports": [
@@ -21,4 +21,3 @@
     {% endfor %}
   ]
 }
-

contrib/network-storage/glusterfs/roles/kubernetes-pv/lib

@@ -0,0 +1 @@
+../../../../../roles/kubernetes-apps/lib

contrib/network-storage/glusterfs/roles/kubernetes-pv/meta/main.yaml

@@ -0,0 +1,2 @@
+dependencies:
+  - {role: kubernetes-pv/ansible, tags: apps}

contrib/terraform/openstack/README.md

@@ -83,6 +83,21 @@ number_of_k8s_nodes = "0"
 ```
 This will provision one VM as master using a floating ip, two additional masters using no floating ips (these will only have private ips inside your tenancy) and one VM as node, again without a floating ip.
 
+Additionally, now the terraform based installation supports provisioning of a GlusterFS shared file system based on a separate set of VMs, running either a Debian or RedHat based set of VMs. To enable this, you need to add to your `my-terraform-vars.tfvars` the following variables:
+
+```
+# Flavour depends on your openstack installation, you can get available flavours through `nova list-flavors`
+flavor_gfs_node = "af659280-5b8a-42b5-8865-a703775911da"
+# This is the name of an image already available in your openstack installation.
+image_gfs = "Ubuntu 15.10"
+number_of_gfs_nodes_no_floating_ip = "3"
+# This is the size of the non-ephemeral volumes to be attached to store the GlusterFS bricks.
+gfs_volume_size_in_gb = "50"
+# The user needed for the image choosen for GlusterFS.
+ssh_user_gfs = "ubuntu"
+```
+
+If these variables are provided, this will give rise to a new ansible group called `gfs-cluster`, for which we have added ansible roles to execute in the ansible provisioning step. If you are using CoreOS, these GlusterFS VM necessarily need to be either Debian or RedHat based VMs, CoreOS cannot serve GlusterFS, but can connect to it through binaries available on hyperkube v1.4.3_coreos.0 or higher.
 
 
 # Provision a Kubernetes Cluster on OpenStack

contrib/terraform/openstack/kubespray.tf

@@ -70,7 +70,7 @@ resource "openstack_compute_instance_v2" "k8s_master" {
         ssh_user = "${var.ssh_user}"
         kubespray_groups = "etcd,kube-master,kube-node,k8s-cluster"
     }
-    
+
 }
 
 
@@ -126,7 +126,7 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" {
         kubespray_groups = "kube-node,k8s-cluster"
     }
     provisioner "local-exec" {
-	command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/k8s-cluster.yml"        
+	command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/k8s-cluster.yml"
     }
 }
 
@@ -149,13 +149,13 @@ resource "openstack_compute_instance_v2" "glusterfs_node_no_floating_ip" {
     security_groups = ["${openstack_compute_secgroup_v2.k8s.name}" ]
     metadata = {
         ssh_user = "${var.ssh_user_gfs}"
-        kubespray_groups = "gfs-cluster"
+        kubespray_groups = "gfs-cluster,network-storage"
     }
     volume {
         volume_id = "${element(openstack_blockstorage_volume_v2.glusterfs_volume.*.id, count.index)}"
     }
     provisioner "local-exec" {
-	command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/gfs-cluster.yml"        
+	command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/gfs-cluster.yml"
     }
 }
 

roles/kubernetes-apps/ansible/tasks/main.yaml

@@ -18,26 +18,6 @@
   with_items: "{{ manifests.results }}"
   when: inventory_hostname == groups['kube-master'][0]
 
-- name: Kubernetes Apps | Lay Down k8s GlusterFS Endpoint and PV
-  template: src={{item.file}} dest=/etc/kubernetes/{{item.dest}}
-  with_items:
-          - {file: glusterfs-kubernetes-pv.yml.j2, type: pv, dest: glusterfs-kubernetes-pv.yml}
-          - {file: glusterfs-kubernetes-endpoint.json.j2, type: ep, dest: glusterfs-kubernetes-endpoint.json} 
-  register: gluster_pv
-  when: inventory_hostname == groups['kube-master'][0] and groups['gfs-cluster'] is defined and hostvars[groups['gfs-cluster'][0]].gluster_disk_size_gb is defined
-
-- name: Kubernetes Apps | Set GlusterFS endpoint and PV
-  kube:
-    name: glusterfs
-    namespace: default
-    kubectl: "{{bin_dir}}/kubectl"
-    resource: "{{item.item.type}}"
-    filename: "/etc/kubernetes/{{item.item.dest}}"
-    state: "{{item.changed | ternary('latest','present') }}"
-  with_items: "{{ gluster_pv.results }}"
-  when: inventory_hostname == groups['kube-master'][0] and groups['gfs-cluster'] is defined
-
-
 - include: tasks/calico-policy-controller.yml
   when: ( enable_network_policy is defined and enable_network_policy == True ) or
     ( kube_network_plugin == 'canal' )