Add cephfs_provisioner Support for Kubespray

extra_playbooks/build-cephfs-provisioner.yml

@@ -0,0 +1,54 @@
+---
+
+- hosts: localhost
+  tasks:
+    - name: CephFS Provisioner | Install pip packages
+      pip:
+        name: "{{ item.name }}"
+        version: "{{ item.version }}"
+        state: "{{ item.state }}"
+      with_items:
+        - { state: "present", name: "docker", version: "2.7.0" }
+        - { state: "present", name: "docker-compose", version: "1.18.0" }
+
+    - name: CephFS Provisioner | Check Go version
+      shell: |
+        go version
+      ignore_errors: yes
+      register: go_version_result
+
+    - name: CephFS Provisioner | Install Go 1.9
+      shell: |
+        add-apt-repository -y ppa:gophers/archive
+        apt-get update
+        apt-get install -y golang-1.9
+        ln -fs /usr/lib/go-1.9/bin/* /usr/local/bin/
+      when: 'go_version_result.rc != 0 or "go version go1.9" not in go_version_result.stdout'
+
+    - name: CephFS Provisioner | Check if image exists
+      shell: |
+        docker image list | grep 'cephfs-provisioner'
+      ignore_errors: yes
+      register: check_image_result
+
+    - block:
+        - name: CephFS Provisioner | Clone repo
+          git:
+            repo: https://github.com/kubernetes-incubator/external-storage.git
+            dest: "~/go/src/github.com/kubernetes-incubator"
+            version: 92295a30
+            clone: no
+            update: yes
+            
+        - name: CephFS Provisioner | Build image
+          shell: |
+            cd ~/go/src/github.com/kubernetes-incubator/external-storage
+            REGISTRY=quay.io/kubespray/ VERSION=92295a30 make ceph/cephfs
+
+        - name: CephFS Provisioner | Push image
+          docker_image:
+            name: quay.io/kubespray/cephfs-provisioner:92295a30
+            push: yes
+          retries: 10
+
+      when: check_image_result.rc != 0

inventory/sample/group_vars/k8s-cluster.yml

@@ -173,6 +173,17 @@ registry_enabled: false
 local_volumes_enabled: false
 local_volume_provisioner_enabled: "{{ local_volumes_enabled }}"
 
+# CephFS provisioner deployment
+cephfs_provisioner_enabled: false
+# cephfs_provisioner_namespace: "{{ system_namespace }}"
+# cephfs_provisioner_cluster: ceph
+# cephfs_provisioner_monitors:
+#   - 172.24.0.1:6789
+#   - 172.24.0.2:6789
+#   - 172.24.0.3:6789
+# cephfs_provisioner_admin_id: admin
+# cephfs_provisioner_secret: secret
+
 # Add Persistent Volumes Storage Class for corresponding cloud provider ( OpenStack is only supported now )
 persistent_volumes_enabled: false
 

roles/kubernetes-apps/cephfs_provisioner/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+cephfs_provisioner_image_repo: quay.io/kubespray/cephfs-provisioner
+cephfs_provisioner_image_tag: 92295a30
+
+cephfs_provisioner_namespace: "{{ system_namespace }}"
+cephfs_provisioner_cluster: ceph
+cephfs_provisioner_monitors: []
+cephfs_provisioner_admin_id: admin
+cephfs_provisioner_secret: secret

roles/kubernetes-apps/cephfs_provisioner/tasks/main.yml

@@ -0,0 +1,36 @@
+---
+
+- name: CephFS Provisioner | Create addon dir
+  file:
+    path: "{{ kube_config_dir }}/addons/cephfs_provisioner"
+    owner: root
+    group: root
+    mode: 0755
+    recurse: true
+
+- name: CephFS Provisioner | Create manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.file }}"
+  with_items:
+    - { name: cephfs-provisioner-sa, file: cephfs-provisioner-sa.yml, type: sa }
+    - { name: cephfs-provisioner-role, file: cephfs-provisioner-role.yml, type: role }
+    - { name: cephfs-provisioner-rolebinding, file: cephfs-provisioner-rolebinding.yml, type: rolebinding }
+    - { name: cephfs-provisioner-clusterrole, file: cephfs-provisioner-clusterrole.yml, type: clusterrole }
+    - { name: cephfs-provisioner-clusterrolebinding, file: cephfs-provisioner-clusterrolebinding.yml, type: clusterrolebinding }
+    - { name: cephfs-provisioner-deploy, file: cephfs-provisioner-deploy.yml, type: deploy }
+    - { name: cephfs-provisioner-secret, file: cephfs-provisioner-secret.yml, type: secret }
+    - { name: cephfs-provisioner-sc, file: cephfs-provisioner-sc.yml, type: sc }
+  register: cephfs_manifests
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: CephFS Provisioner | Apply manifests
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "{{ system_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ cephfs_manifests.results }}"
+  when: inventory_hostname == groups['kube-master'][0]

roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2

@@ -0,0 +1,22 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cephfs-provisioner
+  namespace: {{ system_namespace }}
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "create", "delete"]

roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrolebinding.yml.j2

@@ -0,0 +1,14 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cephfs-provisioner
+  namespace: {{ cephfs_provisioner_namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: cephfs-provisioner
+    namespace: {{ cephfs_provisioner_namespace }}
+roleRef:
+  kind: ClusterRole
+  name: cephfs-provisioner
+  apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-deploy.yml.j2

@@ -0,0 +1,26 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: cephfs-provisioner
+  namespace: {{ cephfs_provisioner_namespace }}
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: cephfs-provisioner
+    spec:
+      containers:
+        - name: cephfs-provisioner
+          image: {{ cephfs_provisioner_image_repo }}:{{ cephfs_provisioner_image_tag }}
+          env:
+            - name: PROVISIONER_NAME
+              value: ceph.com/cephfs
+          command:
+            - "/usr/local/bin/cephfs-provisioner"
+          args:
+            - "-id=cephfs-provisioner-1"
+      serviceAccount: cephfs-provisioner

roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-role.yml.j2

@@ -0,0 +1,10 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: cephfs-provisioner
+  namespace: {{ cephfs_provisioner_namespace }}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["create", "get", "delete"]

roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2

@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: cephfs-provisioner
+  namespace: {{ cephfs_provisioner_namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: cephfs-provisioner
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: cephfs-provisioner

roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sa.yml.j2

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cephfs-provisioner
+  namespace: {{ cephfs_provisioner_namespace }}

roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sc.yml.j2

@@ -0,0 +1,12 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: cephfs
+provisioner: ceph.com/cephfs
+parameters:
+  cluster: {{ cephfs_provisioner_cluster }}
+  monitors: {{ cephfs_provisioner_monitors | join(',') }}
+  adminId: {{ cephfs_provisioner_admin_id }}
+  adminSecretName: cephfs-provisioner-{{ cephfs_provisioner_admin_id }}-secret
+  adminSecretNamespace: {{ cephfs_provisioner_namespace }}

roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-secret.yml.j2

@@ -0,0 +1,9 @@
+---
+kind: Secret
+apiVersion: v1
+metadata:
+  name: cephfs-provisioner-{{ cephfs_provisioner_admin_id }}-secret
+  namespace: {{ cephfs_provisioner_namespace }}
+type: Opaque
+data:
+  secret: {{ cephfs_provisioner_secret | b64encode }}

roles/kubernetes-apps/meta/main.yml

@@ -34,6 +34,13 @@ dependencies:
       - local_volume_provisioner
       - storage
 
+  - role: kubernetes-apps/cephfs_provisioner
+    when: cephfs_provisioner_enabled
+    tags:
+      - apps
+      - cephfs_provisioner
+      - storage
+
   # istio role should be last because it takes a long time to initialize and
   # will cause timeouts trying to start other addons.
   - role: kubernetes-apps/istio