From 206e24448b7c27756ab83f5a85445900172a9ab2 Mon Sep 17 00:00:00 2001 From: Wong Hoi Sing Edison Date: Tue, 13 Feb 2018 09:55:59 +0800 Subject: [PATCH] CephFS Provisioner Addon Fixup --- .../cephfs_provisioner/README.md | 78 +++++++++++++++++++ .../cephfs_provisioner/defaults/main.yml | 0 .../cephfs_provisioner/tasks/main.yml | 7 +- .../cephfs-provisioner-clusterrole.yml.j2 | 2 +- ...phfs-provisioner-clusterrolebinding.yml.j2 | 0 .../templates/cephfs-provisioner-ns.yml.j2 | 7 ++ .../templates/cephfs-provisioner-role.yml.j2 | 0 .../cephfs-provisioner-rolebinding.yml.j2 | 1 + .../templates/cephfs-provisioner-rs.yml.j2} | 21 +++-- .../templates/cephfs-provisioner-sa.yml.j2 | 0 .../templates/cephfs-provisioner-sc.yml.j2 | 0 .../cephfs-provisioner-secret.yml.j2 | 0 .../external_provisioner/meta/main.yml | 7 ++ roles/kubernetes-apps/meta/main.yml | 7 -- 14 files changed, 113 insertions(+), 17 deletions(-) create mode 100644 roles/kubernetes-apps/external_provisioner/cephfs_provisioner/README.md rename roles/kubernetes-apps/{ => external_provisioner}/cephfs_provisioner/defaults/main.yml (100%) rename roles/kubernetes-apps/{ => external_provisioner}/cephfs_provisioner/tasks/main.yml (86%) rename roles/kubernetes-apps/{ => external_provisioner}/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2 (92%) rename roles/kubernetes-apps/{ => external_provisioner}/cephfs_provisioner/templates/cephfs-provisioner-clusterrolebinding.yml.j2 (100%) create mode 100644 roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-ns.yml.j2 rename roles/kubernetes-apps/{ => external_provisioner}/cephfs_provisioner/templates/cephfs-provisioner-role.yml.j2 (100%) rename roles/kubernetes-apps/{ => external_provisioner}/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2 (85%) rename roles/kubernetes-apps/{cephfs_provisioner/templates/cephfs-provisioner-deploy.yml.j2 => external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-rs.yml.j2} (52%) rename roles/kubernetes-apps/{ => external_provisioner}/cephfs_provisioner/templates/cephfs-provisioner-sa.yml.j2 (100%) rename roles/kubernetes-apps/{ => external_provisioner}/cephfs_provisioner/templates/cephfs-provisioner-sc.yml.j2 (100%) rename roles/kubernetes-apps/{ => external_provisioner}/cephfs_provisioner/templates/cephfs-provisioner-secret.yml.j2 (100%) diff --git a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/README.md b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/README.md new file mode 100644 index 000000000..5b338a793 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/README.md @@ -0,0 +1,78 @@ +CephFS Volume Provisioner for Kubernetes 1.5+ +============================================= + +[![Docker Repository on Quay](https://quay.io/repository/external_storage/cephfs-provisioner/status "Docker Repository on Quay")](https://quay.io/repository/external_storage/cephfs-provisioner) + +Using Ceph volume client + +Development +----------- + +Compile the provisioner + +``` console +make +``` + +Make the container image and push to the registry + +``` console +make push +``` + +Test instruction +---------------- + +- Start Kubernetes local cluster + +See https://kubernetes.io/. + +- Create a Ceph admin secret + +``` bash +ceph auth get client.admin 2>&1 |grep "key = " |awk '{print $3'} |xargs echo -n > /tmp/secret +kubectl create ns cephfs +kubectl create secret generic ceph-secret-admin --from-file=/tmp/secret --namespace=cephfs +``` + +- Start CephFS provisioner + +The following example uses `cephfs-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity. + +``` bash +docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host cephfs-provisioner /usr/local/bin/cephfs-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=cephfs-provisioner-1 +``` + +Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md). + +- Create a CephFS Storage Class + +Replace Ceph monitor's IP in example/class.yaml with your own and create storage class: + +``` bash +kubectl create -f example/class.yaml +``` + +- Create a claim + +``` bash +kubectl create -f example/claim.yaml +``` + +- Create a Pod using the claim + +``` bash +kubectl create -f example/test-pod.yaml +``` + +Known limitations +----------------- + +- Kernel CephFS doesn't work with SELinux, setting SELinux label in Pod's securityContext will not work. +- Kernel CephFS doesn't support quota or capacity, capacity requested by PVC is not enforced or validated. +- Currently each Ceph user created by the provisioner has `allow r` MDS cap to permit CephFS mount. + +Acknowledgement +--------------- + +Inspired by CephFS Manila provisioner and conversation with John Spray diff --git a/roles/kubernetes-apps/cephfs_provisioner/defaults/main.yml b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/defaults/main.yml similarity index 100% rename from roles/kubernetes-apps/cephfs_provisioner/defaults/main.yml rename to roles/kubernetes-apps/external_provisioner/cephfs_provisioner/defaults/main.yml diff --git a/roles/kubernetes-apps/cephfs_provisioner/tasks/main.yml b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml similarity index 86% rename from roles/kubernetes-apps/cephfs_provisioner/tasks/main.yml rename to roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml index 6e854f05e..c1fdc624c 100644 --- a/roles/kubernetes-apps/cephfs_provisioner/tasks/main.yml +++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml @@ -3,22 +3,23 @@ - name: CephFS Provisioner | Create addon dir file: path: "{{ kube_config_dir }}/addons/cephfs_provisioner" + state: directory owner: root group: root mode: 0755 - recurse: true - name: CephFS Provisioner | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.file }}" with_items: + - { name: cephfs-provisioner-ns, file: cephfs-provisioner-ns.yml, type: ns } - { name: cephfs-provisioner-sa, file: cephfs-provisioner-sa.yml, type: sa } - { name: cephfs-provisioner-role, file: cephfs-provisioner-role.yml, type: role } - { name: cephfs-provisioner-rolebinding, file: cephfs-provisioner-rolebinding.yml, type: rolebinding } - { name: cephfs-provisioner-clusterrole, file: cephfs-provisioner-clusterrole.yml, type: clusterrole } - { name: cephfs-provisioner-clusterrolebinding, file: cephfs-provisioner-clusterrolebinding.yml, type: clusterrolebinding } - - { name: cephfs-provisioner-deploy, file: cephfs-provisioner-deploy.yml, type: deploy } + - { name: cephfs-provisioner-rs, file: cephfs-provisioner-rs.yml, type: rs } - { name: cephfs-provisioner-secret, file: cephfs-provisioner-secret.yml, type: secret } - { name: cephfs-provisioner-sc, file: cephfs-provisioner-sc.yml, type: sc } register: cephfs_manifests @@ -27,7 +28,7 @@ - name: CephFS Provisioner | Apply manifests kube: name: "{{ item.item.name }}" - namespace: "{{ system_namespace }}" + namespace: "{{ cephfs_provisioner_namespace }}" kubectl: "{{ bin_dir }}/kubectl" resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.item.file }}" diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2 similarity index 92% rename from roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2 rename to roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2 index 272db0f70..e714c3cb2 100644 --- a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2 +++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2 @@ -3,7 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cephfs-provisioner - namespace: {{ system_namespace }} + namespace: {{ cephfs_provisioner_namespace }} rules: - apiGroups: [""] resources: ["persistentvolumes"] diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrolebinding.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-clusterrolebinding.yml.j2 similarity index 100% rename from roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrolebinding.yml.j2 rename to roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-clusterrolebinding.yml.j2 diff --git a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-ns.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-ns.yml.j2 new file mode 100644 index 000000000..2a2a67cf6 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-ns.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ cephfs_provisioner_namespace }} + labels: + name: {{ cephfs_provisioner_namespace }} diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-role.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-role.yml.j2 similarity index 100% rename from roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-role.yml.j2 rename to roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-role.yml.j2 diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2 similarity index 85% rename from roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2 rename to roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2 index f84ed32ba..01ab87b7d 100644 --- a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2 +++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2 @@ -7,6 +7,7 @@ metadata: subjects: - kind: ServiceAccount name: cephfs-provisioner + namespace: {{ cephfs_provisioner_namespace }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-deploy.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-rs.yml.j2 similarity index 52% rename from roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-deploy.yml.j2 rename to roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-rs.yml.j2 index bfe211754..976f29c05 100644 --- a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-deploy.yml.j2 +++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-rs.yml.j2 @@ -1,21 +1,28 @@ --- -apiVersion: extensions/v1beta1 -kind: Deployment +apiVersion: apps/v1 +kind: ReplicaSet metadata: - name: cephfs-provisioner + name: cephfs-provisioner-v{{ cephfs_provisioner_image_tag }} namespace: {{ cephfs_provisioner_namespace }} + labels: + k8s-app: cephfs-provisioner + version: v{{ cephfs_provisioner_image_tag }} spec: replicas: 1 - strategy: - type: Recreate + selector: + matchLabels: + k8s-app: cephfs-provisioner + version: v{{ cephfs_provisioner_image_tag }} template: metadata: labels: - app: cephfs-provisioner + k8s-app: cephfs-provisioner + version: v{{ cephfs_provisioner_image_tag }} spec: containers: - name: cephfs-provisioner image: {{ cephfs_provisioner_image_repo }}:{{ cephfs_provisioner_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} env: - name: PROVISIONER_NAME value: ceph.com/cephfs @@ -23,4 +30,6 @@ spec: - "/usr/local/bin/cephfs-provisioner" args: - "-id=cephfs-provisioner-1" +{% if rbac_enabled %} serviceAccount: cephfs-provisioner +{% endif %} diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sa.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-sa.yml.j2 similarity index 100% rename from roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sa.yml.j2 rename to roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-sa.yml.j2 diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sc.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-sc.yml.j2 similarity index 100% rename from roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-sc.yml.j2 rename to roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-sc.yml.j2 diff --git a/roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-secret.yml.j2 b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-secret.yml.j2 similarity index 100% rename from roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-secret.yml.j2 rename to roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-secret.yml.j2 diff --git a/roles/kubernetes-apps/external_provisioner/meta/main.yml b/roles/kubernetes-apps/external_provisioner/meta/main.yml index 3daa461d8..b520922d6 100644 --- a/roles/kubernetes-apps/external_provisioner/meta/main.yml +++ b/roles/kubernetes-apps/external_provisioner/meta/main.yml @@ -6,3 +6,10 @@ dependencies: - apps - local-volume-provisioner - external-provisioner + + - role: kubernetes-apps/external_provisioner/cephfs_provisioner + when: cephfs_provisioner_enabled + tags: + - apps + - cephfs-provisioner + - external-provisioner diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index fa3b1f1a6..fca51a3b6 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -27,13 +27,6 @@ dependencies: - apps - registry - - role: kubernetes-apps/cephfs_provisioner - when: cephfs_provisioner_enabled - tags: - - apps - - cephfs_provisioner - - storage - # istio role should be last because it takes a long time to initialize and # will cause timeouts trying to start other addons. - role: kubernetes-apps/istio