diff --git a/README.md b/README.md index df7da6d6d..689ae31b2 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ Supported Components - [weave](https://github.com/weaveworks/weave) v2.5.1 - Application - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11 + - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11 - [cert-manager](https://github.com/jetstack/cert-manager) v0.5.2 - [coredns](https://github.com/coredns/coredns) v1.5.0 - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.21.0 diff --git a/inventory/sample/group_vars/k8s-cluster/addons.yml b/inventory/sample/group_vars/k8s-cluster/addons.yml index cf8726ef3..f91dc64f1 100644 --- a/inventory/sample/group_vars/k8s-cluster/addons.yml +++ b/inventory/sample/group_vars/k8s-cluster/addons.yml @@ -56,6 +56,25 @@ cephfs_provisioner_enabled: false # cephfs_provisioner_claim_root: /volumes # cephfs_provisioner_deterministic_names: true +# RBD provisioner deployment +rbd_provisioner_enabled: false +# rbd_provisioner_namespace: rbd-provisioner +# rbd_provisioner_replicas: 2 +# rbd_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789" +# rbd_provisioner_pool: kube +# rbd_provisioner_admin_id: admin +# rbd_provisioner_secret_name: ceph-secret-admin +# rbd_provisioner_secret: ceph-key-admin +# rbd_provisioner_user_id: kube +# rbd_provisioner_user_secret_name: ceph-secret-user +# rbd_provisioner_user_secret: ceph-key-user +# rbd_provisioner_user_secret_namespace: rbd-provisioner +# rbd_provisioner_fs_type: ext4 +# rbd_provisioner_image_format: "2" +# rbd_provisioner_image_features: layering +# rbd_provisioner_storage_class: rbd +# rbd_provisioner_reclaim_policy: Delete + # Nginx ingress controller deployment ingress_nginx_enabled: false # ingress_nginx_host_network: false diff --git a/roles/bootstrap-os/tasks/main.yml b/roles/bootstrap-os/tasks/main.yml index 5733c7db4..8b57c7a34 100644 --- a/roles/bootstrap-os/tasks/main.yml +++ b/roles/bootstrap-os/tasks/main.yml @@ -59,3 +59,10 @@ filter: ansible_hostname when: - hostname_changed.changed + +- name: "Install ceph-commmon package" + package: + name: + - ceph-common + state: latest + when: rbd_provisioner_enabled|default(false) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index db38cc7c6..6d2eb86ed 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -244,6 +244,8 @@ local_volume_provisioner_image_repo: "quay.io/external_storage/local-volume-prov local_volume_provisioner_image_tag: "v2.1.0" cephfs_provisioner_image_repo: "quay.io/external_storage/cephfs-provisioner" cephfs_provisioner_image_tag: "v2.1.0-k8s1.11" +rbd_provisioner_image_repo: "quay.io/external_storage/rbd-provisioner" +rbd_provisioner_image_tag: "v2.1.1-k8s1.11" local_path_provisioner_image_repo: "rancher/local-path-provisioner" local_path_provisioner_image_tag: "v0.0.2" ingress_nginx_controller_image_repo: "quay.io/kubernetes-ingress-controller/nginx-ingress-controller" @@ -632,6 +634,15 @@ downloads: groups: - kube-node + rbd_provisioner: + enabled: "{{ rbd_provisioner_enabled }}" + container: true + repo: "{{ rbd_provisioner_image_repo }}" + tag: "{{ rbd_provisioner_image_tag }}" + sha256: "{{ rbd_provisioner_digest_checksum|default(None) }}" + groups: + - kube-node + local_path_provisioner: enabled: "{{ local_volume_provisioner_enabled }}" container: true diff --git a/roles/kubernetes-apps/external_provisioner/meta/main.yml b/roles/kubernetes-apps/external_provisioner/meta/main.yml index 8b3dbdee1..19fe8ba48 100644 --- a/roles/kubernetes-apps/external_provisioner/meta/main.yml +++ b/roles/kubernetes-apps/external_provisioner/meta/main.yml @@ -16,6 +16,12 @@ dependencies: - cephfs-provisioner - external-provisioner + - role: kubernetes-apps/external_provisioner/rbd_provisioner + when: rbd_provisioner_enabled + tags: + - apps + - rbd-provisioner + - external-provisioner - role: kubernetes-apps/external_provisioner/local_path_provisioner when: local_path_provisioner_enabled tags: diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/README.md b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/README.md new file mode 100644 index 000000000..ef844380e --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/README.md @@ -0,0 +1,79 @@ +# RBD Volume Provisioner for Kubernetes 1.5+ + +`rbd-provisioner` is an out-of-tree dynamic provisioner for Kubernetes 1.5+. +You can use it quickly & easily deploy ceph RBD storage that works almost +anywhere. + +It works just like in-tree dynamic provisioner. For more information on how +dynamic provisioning works, see [the docs](http://kubernetes.io/docs/user-guide/persistent-volumes/) +or [this blog post](http://blog.kubernetes.io/2016/10/dynamic-provisioning-and-storage-in-kubernetes.html). + +## Development + +Compile the provisioner + +```console +make +``` + +Make the container image and push to the registry + +```console +make push +``` + +## Test instruction + +* Start Kubernetes local cluster + +See https://kubernetes.io/. + +* Create a Ceph admin secret + +```bash +ceph auth get client.admin 2>&1 |grep "key = " |awk '{print $3'} |xargs echo -n > /tmp/secret +kubectl create secret generic ceph-admin-secret --from-file=/tmp/secret --namespace=kube-system +``` + +* Create a Ceph pool and a user secret + +```bash +ceph osd pool create kube 8 8 +ceph auth add client.kube mon 'allow r' osd 'allow rwx pool=kube' +ceph auth get-key client.kube > /tmp/secret +kubectl create secret generic ceph-secret --from-file=/tmp/secret --namespace=kube-system +``` + +* Start RBD provisioner + +The following example uses `rbd-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity. + +```bash +docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host quay.io/external_storage/rbd-provisioner /usr/local/bin/rbd-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=rbd-provisioner-1 +``` + +Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md). + +* Create a RBD Storage Class + +Replace Ceph monitor's IP in [examples/class.yaml](examples/class.yaml) with your own and create storage class: + +```bash +kubectl create -f examples/class.yaml +``` + +* Create a claim + +```bash +kubectl create -f examples/claim.yaml +``` + +* Create a Pod using the claim + +```bash +kubectl create -f examples/test-pod.yaml +``` + +## Acknowledgements + +- This provisioner is extracted from [Kubernetes core](https://github.com/kubernetes/kubernetes) with some modifications for this project. diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml new file mode 100644 index 000000000..2dc104093 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml @@ -0,0 +1,17 @@ +--- +rbd_provisioner_namespace: "rbd-provisioner" +rbd_provisioner_replicas: 2 +rbd_provisioner_monitors: ~ +rbd_provisioner_pool: kube +rbd_provisioner_admin_id: admin +rbd_provisioner_secret_name: ceph-secret-admin +rbd_provisioner_secret_token: ceph-key-admin +rbd_provisioner_user_id: kube +rbd_provisioner_user_secret_name: ceph-secret-user +rbd_provisioner_user_secret_token: ceph-key-user +rbd_provisioner_user_secret_namespace: rbd-provisioner +rbd_provisioner_fs_type: ext4 +rbd_provisioner_image_format: "2" +rbd_provisioner_image_features: layering +rbd_provisioner_storage_class: rbd +rbd_provisioner_reclaim_policy: Delete diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/tasks/main.yml b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/tasks/main.yml new file mode 100644 index 000000000..7c09168b2 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/tasks/main.yml @@ -0,0 +1,79 @@ +--- + +- name: RBD Provisioner | Remove legacy addon dir and manifests + file: + path: "{{ kube_config_dir }}/addons/rbd_provisioner" + state: absent + when: + - inventory_hostname == groups['kube-master'][0] + tags: + - upgrade + +- name: RBD Provisioner | Remove legacy namespace + shell: | + {{ bin_dir }}/kubectl delete namespace {{ rbd_provisioner_namespace }} + ignore_errors: yes + when: + - inventory_hostname == groups['kube-master'][0] + tags: + - upgrade + +- name: RBD Provisioner | Remove legacy storageclass + shell: | + {{ bin_dir }}/kubectl delete storageclass {{ rbd_provisioner_storage_class }} + ignore_errors: yes + when: + - inventory_hostname == groups['kube-master'][0] + tags: + - upgrade + +- name: RBD Provisioner | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/rbd_provisioner" + state: directory + owner: root + group: root + mode: 0755 + when: + - inventory_hostname == groups['kube-master'][0] + +- name: RBD Provisioner | Templates list + set_fact: + rbd_provisioner_templates: + - { name: 00-namespace, file: 00-namespace.yml, type: ns } + - { name: secret-rbd-provisioner, file: secret-rbd-provisioner.yml, type: secret } + - { name: sa-rbd-provisioner, file: sa-rbd-provisioner.yml, type: sa } + - { name: clusterrole-rbd-provisioner, file: clusterrole-rbd-provisioner.yml, type: clusterrole } + - { name: clusterrolebinding-rbd-provisioner, file: clusterrolebinding-rbd-provisioner.yml, type: clusterrolebinding } + - { name: role-rbd-provisioner, file: role-rbd-provisioner.yml, type: role } + - { name: rolebinding-rbd-provisioner, file: rolebinding-rbd-provisioner.yml, type: rolebinding } + - { name: deploy-rbd-provisioner, file: deploy-rbd-provisioner.yml, type: deploy } + - { name: sc-rbd-provisioner, file: sc-rbd-provisioner.yml, type: sc } + rbd_provisioner_templates_for_psp: + - { name: psp-rbd-provisioner, file: psp-rbd-provisioner.yml, type: psp } + +- name: RBD Provisioner | Append extra templates to RBD Provisioner Templates list for PodSecurityPolicy + set_fact: + rbd_provisioner_templates: "{{ rbd_provisioner_templates_for_psp + rbd_provisioner_templates }}" + when: + - podsecuritypolicy_enabled + - rbd_provisioner_namespace != "kube-system" + +- name: RBD Provisioner | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/rbd_provisioner/{{ item.file }}" + with_items: "{{ rbd_provisioner_templates }}" + register: rbd_provisioner_manifests + when: inventory_hostname == groups['kube-master'][0] + +- name: RBD Provisioner | Apply manifests + kube: + name: "{{ item.item.name }}" + namespace: "{{ rbd_provisioner_namespace }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/rbd_provisioner/{{ item.item.file }}" + state: "latest" + with_items: "{{ rbd_provisioner_manifests.results }}" + when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/00-namespace.yml.j2 b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/00-namespace.yml.j2 new file mode 100644 index 000000000..8bec2b5eb --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/00-namespace.yml.j2 @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ rbd_provisioner_namespace }} + labels: + name: {{ rbd_provisioner_namespace }} diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrole-rbd-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrole-rbd-provisioner.yml.j2 new file mode 100644 index 000000000..8fc7e4b9d --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrole-rbd-provisioner.yml.j2 @@ -0,0 +1,30 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create", "update", "patch"] + - apiGroups: [""] + resources: ["services"] + resourceNames: ["kube-dns","coredns"] + verbs: ["list", "get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "delete"] + - apiGroups: ["policy"] + resourceNames: ["rbd-provisioner"] + resources: ["podsecuritypolicies"] + verbs: ["use"] diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrolebinding-rbd-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrolebinding-rbd-provisioner.yml.j2 new file mode 100644 index 000000000..ae9e6c525 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrolebinding-rbd-provisioner.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: rbd-provisioner +subjects: + - kind: ServiceAccount + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} +roleRef: + kind: ClusterRole + name: rbd-provisioner + apiGroup: rbac.authorization.k8s.io diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/deploy-rbd-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/deploy-rbd-provisioner.yml.j2 new file mode 100644 index 000000000..0d66bac88 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/deploy-rbd-provisioner.yml.j2 @@ -0,0 +1,42 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} + labels: + app: rbd-provisioner + version: {{ rbd_provisioner_image_tag }} +spec: + replicas: {{ rbd_provisioner_replicas }} + strategy: + type: Recreate + selector: + matchLabels: + app: rbd-provisioner + version: {{ rbd_provisioner_image_tag }} + template: + metadata: + labels: + app: rbd-provisioner + version: {{ rbd_provisioner_image_tag }} + spec: +{% if kube_version is version('v1.11.1', '>=') %} + priorityClassName: {% if rbd_provisioner_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{''}} +{% endif %} + serviceAccount: rbd-provisioner + containers: + - name: rbd-provisioner + image: {{ rbd_provisioner_image_repo }}:{{ rbd_provisioner_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: PROVISIONER_NAME + value: ceph.com/rbd + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + command: + - "/usr/local/bin/rbd-provisioner" + args: + - "-id=${POD_NAME}" diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/psp-rbd-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/psp-rbd-provisioner.yml.j2 new file mode 100644 index 000000000..947675514 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/psp-rbd-provisioner.yml.j2 @@ -0,0 +1,45 @@ +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: rbd-provisioner + annotations: + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default' +{% if apparmor_enabled %} + apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' +{% endif %} + labels: + kubernetes.io/cluster-service: 'true' + addonmanager.kubernetes.io/mode: Reconcile +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/role-rbd-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/role-rbd-provisioner.yml.j2 new file mode 100644 index 000000000..d8dbbf990 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/role-rbd-provisioner.yml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list", "watch", "create", "update", "patch"] diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/rolebinding-rbd-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/rolebinding-rbd-provisioner.yml.j2 new file mode 100644 index 000000000..fcae1cce0 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/rolebinding-rbd-provisioner.yml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} +subjects: + - kind: ServiceAccount + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rbd-provisioner diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sa-rbd-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sa-rbd-provisioner.yml.j2 new file mode 100644 index 000000000..c4dce6450 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sa-rbd-provisioner.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rbd-provisioner + namespace: {{ rbd_provisioner_namespace }} diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sc-rbd-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sc-rbd-provisioner.yml.j2 new file mode 100644 index 000000000..9fea17a94 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sc-rbd-provisioner.yml.j2 @@ -0,0 +1,19 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ rbd_provisioner_storage_class }} +provisioner: ceph.com/rbd +reclaimPolicy: {{ rbd_provisioner_reclaim_policy }} +parameters: + monitors: {{ rbd_provisioner_monitors }} + adminId: {{ rbd_provisioner_admin_id }} + adminSecretNamespace: {{ rbd_provisioner_namespace }} + adminSecretName: {{ rbd_provisioner_secret_name }} + pool: {{ rbd_provisioner_pool }} + userId: {{ rbd_provisioner_user_id }} + userSecretNamespace: {{ rbd_provisioner_user_secret_namespace }} + userSecretName: {{ rbd_provisioner_user_secret_name }} + fsType: "{{ rbd_provisioner_fs_type }}" + imageFormat: "{{ rbd_provisioner_image_format }}" + imageFeatures: {{ rbd_provisioner_image_features }} diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/secret-rbd-provisioner.yml.j2 b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/secret-rbd-provisioner.yml.j2 new file mode 100644 index 000000000..a3b66d634 --- /dev/null +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/secret-rbd-provisioner.yml.j2 @@ -0,0 +1,18 @@ +--- +kind: Secret +apiVersion: v1 +metadata: + name: {{ rbd_provisioner_secret_name }} + namespace: {{ rbd_provisioner_namespace }} +type: Opaque +data: + secret: {{ rbd_provisioner_secret | b64encode }} +--- +kind: Secret +apiVersion: v1 +metadata: + name: {{ rbd_provisioner_user_secret_name }} + namespace: {{ rbd_provisioner_user_secret_namespace }} +type: Opaque +data: + key: {{ rbd_provisioner_user_secret | b64encode }} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 205362c3e..5c91f022d 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -278,6 +278,7 @@ enable_network_policy: true local_volume_provisioner_enabled: "{{ local_volumes_enabled | default('false') }}" persistent_volumes_enabled: false cephfs_provisioner_enabled: false +rbd_provisioner_enabled: false ingress_nginx_enabled: false cert_manager_enabled: false expand_persistent_volumes: false