Merge pull request #2333 from hswong3i/cephfs_provisioner_fixup

CephFS Provisioner Addon Fixup
This commit is contained in:
Andreas Krüger 2018-03-23 11:57:11 +01:00 committed by GitHub
commit f07734596e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 113 additions and 17 deletions

View file

@ -0,0 +1,78 @@
CephFS Volume Provisioner for Kubernetes 1.5+
=============================================
[![Docker Repository on Quay](https://quay.io/repository/external_storage/cephfs-provisioner/status "Docker Repository on Quay")](https://quay.io/repository/external_storage/cephfs-provisioner)
Using Ceph volume client
Development
-----------
Compile the provisioner
``` console
make
```
Make the container image and push to the registry
``` console
make push
```
Test instruction
----------------
- Start Kubernetes local cluster
See <a href="https://kubernetes.io/" class="uri" class="uri">https://kubernetes.io/</a>.
- Create a Ceph admin secret
``` bash
ceph auth get client.admin 2>&1 |grep "key = " |awk '{print $3'} |xargs echo -n > /tmp/secret
kubectl create ns cephfs
kubectl create secret generic ceph-secret-admin --from-file=/tmp/secret --namespace=cephfs
```
- Start CephFS provisioner
The following example uses `cephfs-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
``` bash
docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host cephfs-provisioner /usr/local/bin/cephfs-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=cephfs-provisioner-1
```
Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
- Create a CephFS Storage Class
Replace Ceph monitor's IP in <a href="example/class.yaml" class="uri" class="uri">example/class.yaml</a> with your own and create storage class:
``` bash
kubectl create -f example/class.yaml
```
- Create a claim
``` bash
kubectl create -f example/claim.yaml
```
- Create a Pod using the claim
``` bash
kubectl create -f example/test-pod.yaml
```
Known limitations
-----------------
- Kernel CephFS doesn't work with SELinux, setting SELinux label in Pod's securityContext will not work.
- Kernel CephFS doesn't support quota or capacity, capacity requested by PVC is not enforced or validated.
- Currently each Ceph user created by the provisioner has `allow r` MDS cap to permit CephFS mount.
Acknowledgement
---------------
Inspired by CephFS Manila provisioner and conversation with John Spray

View file

@ -3,22 +3,23 @@
- name: CephFS Provisioner | Create addon dir
file:
path: "{{ kube_config_dir }}/addons/cephfs_provisioner"
state: directory
owner: root
group: root
mode: 0755
recurse: true
- name: CephFS Provisioner | Create manifests
template:
src: "{{ item.file }}.j2"
dest: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.file }}"
with_items:
- { name: cephfs-provisioner-ns, file: cephfs-provisioner-ns.yml, type: ns }
- { name: cephfs-provisioner-sa, file: cephfs-provisioner-sa.yml, type: sa }
- { name: cephfs-provisioner-role, file: cephfs-provisioner-role.yml, type: role }
- { name: cephfs-provisioner-rolebinding, file: cephfs-provisioner-rolebinding.yml, type: rolebinding }
- { name: cephfs-provisioner-clusterrole, file: cephfs-provisioner-clusterrole.yml, type: clusterrole }
- { name: cephfs-provisioner-clusterrolebinding, file: cephfs-provisioner-clusterrolebinding.yml, type: clusterrolebinding }
- { name: cephfs-provisioner-deploy, file: cephfs-provisioner-deploy.yml, type: deploy }
- { name: cephfs-provisioner-rs, file: cephfs-provisioner-rs.yml, type: rs }
- { name: cephfs-provisioner-secret, file: cephfs-provisioner-secret.yml, type: secret }
- { name: cephfs-provisioner-sc, file: cephfs-provisioner-sc.yml, type: sc }
register: cephfs_manifests
@ -27,7 +28,7 @@
- name: CephFS Provisioner | Apply manifests
kube:
name: "{{ item.item.name }}"
namespace: "{{ system_namespace }}"
namespace: "{{ cephfs_provisioner_namespace }}"
kubectl: "{{ bin_dir }}/kubectl"
resource: "{{ item.item.type }}"
filename: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.item.file }}"

View file

@ -3,7 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: cephfs-provisioner
namespace: {{ system_namespace }}
namespace: {{ cephfs_provisioner_namespace }}
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]

View file

@ -0,0 +1,7 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: {{ cephfs_provisioner_namespace }}
labels:
name: {{ cephfs_provisioner_namespace }}

View file

@ -7,6 +7,7 @@ metadata:
subjects:
- kind: ServiceAccount
name: cephfs-provisioner
namespace: {{ cephfs_provisioner_namespace }}
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role

View file

@ -1,21 +1,28 @@
---
apiVersion: extensions/v1beta1
kind: Deployment
apiVersion: apps/v1
kind: ReplicaSet
metadata:
name: cephfs-provisioner
name: cephfs-provisioner-v{{ cephfs_provisioner_image_tag }}
namespace: {{ cephfs_provisioner_namespace }}
labels:
k8s-app: cephfs-provisioner
version: v{{ cephfs_provisioner_image_tag }}
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
k8s-app: cephfs-provisioner
version: v{{ cephfs_provisioner_image_tag }}
template:
metadata:
labels:
app: cephfs-provisioner
k8s-app: cephfs-provisioner
version: v{{ cephfs_provisioner_image_tag }}
spec:
containers:
- name: cephfs-provisioner
image: {{ cephfs_provisioner_image_repo }}:{{ cephfs_provisioner_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }}
env:
- name: PROVISIONER_NAME
value: ceph.com/cephfs
@ -23,4 +30,6 @@ spec:
- "/usr/local/bin/cephfs-provisioner"
args:
- "-id=cephfs-provisioner-1"
{% if rbac_enabled %}
serviceAccount: cephfs-provisioner
{% endif %}

View file

@ -6,3 +6,10 @@ dependencies:
- apps
- local-volume-provisioner
- external-provisioner
- role: kubernetes-apps/external_provisioner/cephfs_provisioner
when: cephfs_provisioner_enabled
tags:
- apps
- cephfs-provisioner
- external-provisioner

View file

@ -27,13 +27,6 @@ dependencies:
- apps
- registry
- role: kubernetes-apps/cephfs_provisioner
when: cephfs_provisioner_enabled
tags:
- apps
- cephfs_provisioner
- storage
# istio role should be last because it takes a long time to initialize and
# will cause timeouts trying to start other addons.
- role: kubernetes-apps/istio