UpCloud integration (#8653)

* [upcloud] add upcloud csi-driver

* Option to use ansible_host as api ip for kubueconfig
This commit is contained in:
Robin Wallace 2022-04-12 00:13:23 +02:00 committed by GitHub
parent 9dced7133c
commit d7254eead6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 565 additions and 0 deletions

View file

@ -65,6 +65,11 @@ resource "upcloud_server" "master" {
network = upcloud_network.private.id network = upcloud_network.private.id
} }
# Ignore volumes created by csi-driver
lifecycle {
ignore_changes = [storage_devices]
}
dynamic "storage_devices" { dynamic "storage_devices" {
for_each = { for_each = {
for disk_key_name, disk in upcloud_storage.additional_disks : for disk_key_name, disk in upcloud_storage.additional_disks :
@ -114,6 +119,11 @@ resource "upcloud_server" "worker" {
network = upcloud_network.private.id network = upcloud_network.private.id
} }
# Ignore volumes created by csi-driver
lifecycle {
ignore_changes = [storage_devices]
}
dynamic "storage_devices" { dynamic "storage_devices" {
for_each = { for_each = {
for disk_key_name, disk in upcloud_storage.additional_disks : for disk_key_name, disk in upcloud_storage.additional_disks :

View file

@ -0,0 +1,17 @@
## Repo for UpClouds csi-driver: https://github.com/UpCloudLtd/upcloud-csi
## To use UpClouds CSI plugin to provision volumes set this value to true
## Remember to set UPCLOUD_USERNAME and UPCLOUD_PASSWORD
# upcloud_csi_enabled: true
# upcloud_csi_controller_replicas: 1
## Override used image tags
# upcloud_csi_provisioner_image_tag: "v3.1.0"
# upcloud_csi_attacher_image_tag: "v3.4.0"
# upcloud_csi_resizer_image_tag: "v1.4.0"
# upcloud_csi_plugin_image_tag: "alpha"
# upcloud_csi_node_image_tag: "v2.5.0"
# upcloud_tolerations: []
## Storage class options
# expand_persistent_volumes: true
# storage_classes:
# - name: standard
# is_default: true

View file

@ -238,6 +238,8 @@ podsecuritypolicy_enabled: false
# Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts # Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts
# kubeconfig_localhost: false # kubeconfig_localhost: false
# Use ansible_host as external api ip when copying over kubeconfig.
# kubeconfig_localhost_ansible_host: false
# Download kubectl onto the host that runs Ansible in {{ bin_dir }} # Download kubectl onto the host that runs Ansible in {{ bin_dir }}
# kubectl_localhost: false # kubectl_localhost: false

View file

@ -0,0 +1,10 @@
---
upcloud_csi_controller_replicas: 1
upcloud_csi_provisioner_image_tag: "v3.1.0"
upcloud_csi_attacher_image_tag: "v3.4.0"
upcloud_csi_resizer_image_tag: "v1.4.0"
upcloud_csi_plugin_image_tag: "alpha"
upcloud_csi_node_image_tag: "v2.5.0"
upcloud_username: "{{ lookup('env','UPCLOUD_USERNAME') }}"
upcloud_password: "{{ lookup('env','UPCLOUD_PASSWORD') }}"
upcloud_tolerations: []

View file

@ -0,0 +1,43 @@
---
- name: UpCloud CSI Driver | Check if UPCLOUD_USERNAME exists
fail:
msg: "UpCloud username is missing. Env UPCLOUD_USERNAME is mandatory"
when: upcloud_username is not defined or not upcloud_username
tags: upcloud-csi-driver
- name: UpCloud CSI Driver | Check if UPCLOUD_PASSWORD exists
fail:
msg: "UpCloud password is missing. Env UPCLOUD_PASSWORD is mandatory"
when:
- upcloud_username is defined
- upcloud_username|length > 0
- upcloud_password is not defined or not upcloud_password
tags: upcloud-csi-driver
- name: UpCloud CSI Driver | Generate Manifests
template:
src: "{{ item.file }}.j2"
dest: "{{ kube_config_dir }}/{{ item.file }}"
with_items:
- {name: upcloud-csi-cred-secret, file: upcloud-csi-cred-secret.yml}
- {name: upcloud-csi-setup, file: upcloud-csi-setup.yml}
- {name: upcloud-csi-controller, file: upcloud-csi-controller.yml}
- {name: upcloud-csi-node, file: upcloud-csi-node.yml}
- {name: upcloud-csi-driver, file: upcloud-csi-driver.yml}
register: upcloud_csi_manifests
when: inventory_hostname == groups['kube_control_plane'][0]
tags: upcloud-csi-driver
- name: UpCloud CSI Driver | Apply Manifests
kube:
kubectl: "{{ bin_dir }}/kubectl"
filename: "{{ kube_config_dir }}/{{ item.item.file }}"
state: "latest"
with_items:
- "{{ upcloud_csi_manifests.results }}"
when:
- inventory_hostname == groups['kube_control_plane'][0]
- not item is skipped
loop_control:
label: "{{ item.item.file }}"
tags: upcloud-csi-driver

View file

@ -0,0 +1,96 @@
kind: StatefulSet
apiVersion: apps/v1
metadata:
name: csi-upcloud-controller
namespace: kube-system
spec:
serviceName: "csi-upcloud"
replicas: {{ upcloud_csi_controller_replicas }}
selector:
matchLabels:
app: csi-upcloud-controller
template:
metadata:
labels:
app: csi-upcloud-controller
role: csi-upcloud
spec:
priorityClassName: system-cluster-critical
serviceAccount: csi-upcloud-controller-sa
containers:
- name: csi-provisioner
image: k8s.gcr.io/sig-storage/csi-provisioner:{{ upcloud_csi_provisioner_image_tag }}
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"
- "--timeout=60s"
env:
- name: ADDRESS
value: /var/lib/csi/sockets/pluginproxy/csi.sock
imagePullPolicy: "Always"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/csi/sockets/pluginproxy/
- name: csi-attacher
image: k8s.gcr.io/sig-storage/csi-attacher:{{ upcloud_csi_attacher_image_tag }}
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
- "--timeout=30s"
env:
- name: ADDRESS
value: /var/lib/csi/sockets/pluginproxy/csi.sock
imagePullPolicy: "Always"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/csi/sockets/pluginproxy/
- name: csi-resizer
image: k8s.gcr.io/sig-storage/csi-resizer:{{ upcloud_csi_resizer_image_tag }}
args:
- "--v=5"
- "--timeout=45s"
- "--csi-address=$(ADDRESS)"
- "--handle-volume-inuse-error=true"
env:
- name: ADDRESS
value: /var/lib/csi/sockets/pluginproxy/csi.sock
imagePullPolicy: "Always"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/csi/sockets/pluginproxy/
- name: csi-upcloud-plugin
image: ghcr.io/upcloudltd/upcloud-csi:{{ upcloud_csi_plugin_image_tag }}
args:
- "--endpoint=$(CSI_ENDPOINT)"
- "--nodehost=$(NODE_ID)"
- "--username=$(UPCLOUD_USERNAME)"
- "--password=$(UPCLOUD_PASSWORD)"
- "--url=$(UPCLOUD_API_URL)"
env:
- name: CSI_ENDPOINT
value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
- name: UPCLOUD_API_URL
value: https://api.upcloud.com/
- name: UPCLOUD_USERNAME
valueFrom:
secretKeyRef:
name: upcloud
key: username
- name: UPCLOUD_PASSWORD
valueFrom:
secretKeyRef:
name: upcloud
key: password
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
imagePullPolicy: "Always"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/csi/sockets/pluginproxy/
imagePullSecrets:
- name: regcred
volumes:
- name: socket-dir
emptyDir: { }

View file

@ -0,0 +1,9 @@
---
apiVersion: v1
kind: Secret
metadata:
name: upcloud
namespace: kube-system
stringData:
username: {{ upcloud_username }}
password: {{ upcloud_password }}

View file

@ -0,0 +1,7 @@
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
name: storage.csi.upcloud.com
spec:
attachRequired: true
podInfoOnMount: true

View file

@ -0,0 +1,113 @@
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: csi-upcloud-node
namespace: kube-system
spec:
selector:
matchLabels:
app: csi-upcloud-node
template:
metadata:
labels:
app: csi-upcloud-node
role: csi-upcloud
spec:
priorityClassName: system-node-critical
serviceAccount: csi-upcloud-node-sa
hostNetwork: true
containers:
- name: csi-node-driver-registrar
image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:{{ upcloud_csi_node_image_tag }}
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
- "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
lifecycle:
preStop:
exec:
command:
[
"/bin/sh",
"-c",
"rm -rf /registration/storage.csi.upcloud.com /registration/storage.csi.upcloud.com-reg.sock",
]
env:
- name: ADDRESS
value: /csi/csi.sock
- name: DRIVER_REG_SOCK_PATH
value: /var/lib/kubelet/plugins/storage.csi.upcloud.com/csi.sock
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: plugin-dir
mountPath: /csi/
- name: registration-dir
mountPath: /registration/
- name: csi-upcloud-plugin
image: ghcr.io/upcloudltd/upcloud-csi:alpha
args:
- "--endpoint=$(CSI_ENDPOINT)"
- "--nodehost=$(NODE_ID)"
- "--username=$(UPCLOUD_USERNAME)"
- "--password=$(UPCLOUD_PASSWORD)"
- "--url=$(UPCLOUD_API_URL)"
env:
- name: CSI_ENDPOINT
value: unix:///csi/csi.sock
- name: UPCLOUD_API_URL
value: https://api.upcloud.com/
- name: UPCLOUD_USERNAME
valueFrom:
secretKeyRef:
name: upcloud
key: username
- name: UPCLOUD_PASSWORD
valueFrom:
secretKeyRef:
name: upcloud
key: password
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
imagePullPolicy: "Always"
securityContext:
privileged: true
capabilities:
add: [ "SYS_ADMIN" ]
allowPrivilegeEscalation: true
volumeMounts:
- name: plugin-dir
mountPath: /csi
- name: pods-mount-dir
mountPath: /var/lib/kubelet
# needed so that any mounts setup inside this container are
# propagated back to the host machine.
mountPropagation: "Bidirectional"
- name: device-dir
mountPath: /dev
imagePullSecrets:
- name: regcred
volumes:
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins_registry/
type: DirectoryOrCreate
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins/storage.csi.upcloud.com
type: DirectoryOrCreate
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet
type: Directory
- name: device-dir
hostPath:
path: /dev
{% if upcloud_tolerations %}
tolerations:
{{ upcloud_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}
{% endif %}

View file

@ -0,0 +1,205 @@
kind: ServiceAccount
apiVersion: v1
metadata:
name: csi-upcloud-controller-sa
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-upcloud-provisioner-role
rules:
- apiGroups: [ "" ]
resources: [ "secrets" ]
verbs: [ "get", "list" ]
- apiGroups: [ "" ]
resources: [ "persistentvolumes" ]
verbs: [ "get", "list", "watch", "create", "delete" ]
- apiGroups: [ "" ]
resources: [ "persistentvolumeclaims" ]
verbs: [ "get", "list", "watch", "update" ]
- apiGroups: [ "storage.k8s.io" ]
resources: [ "storageclasses" ]
verbs: [ "get", "list", "watch" ]
- apiGroups: [ "storage.k8s.io" ]
resources: [ "csinodes" ]
verbs: [ "get", "list", "watch" ]
- apiGroups: [ "" ]
resources: [ "events" ]
verbs: [ "list", "watch", "create", "update", "patch" ]
- apiGroups: [ "snapshot.storage.k8s.io" ]
resources: [ "volumesnapshots" ]
verbs: [ "get", "list" ]
- apiGroups: [ "snapshot.storage.k8s.io" ]
resources: [ "volumesnapshotcontents" ]
verbs: [ "get", "list" ]
- apiGroups: [ "" ]
resources: [ "nodes" ]
verbs: [ "get", "list", "watch" ]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-upcloud-provisioner-binding
subjects:
- kind: ServiceAccount
name: csi-upcloud-controller-sa
namespace: kube-system
roleRef:
kind: ClusterRole
name: csi-upcloud-provisioner-role
apiGroup: rbac.authorization.k8s.io
---
# Attacher must be able to work with PVs, nodes and VolumeAttachments
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-upcloud-attacher-role
rules:
- apiGroups: [ "" ]
resources: [ "persistentvolumes" ]
verbs: [ "get", "list", "watch", "update", "patch" ]
- apiGroups: [ "" ]
resources: [ "nodes" ]
verbs: [ "get", "list", "watch" ]
- apiGroups: [ "storage.k8s.io" ]
resources: [ "csinodes" ]
verbs: [ "get", "list", "watch" ]
- apiGroups: [ "storage.k8s.io" ]
resources: [ "volumeattachments" ]
verbs: [ "get", "list", "watch", "update", "patch" ]
- apiGroups: [ "storage.k8s.io" ]
resources: [ "volumeattachments/status" ]
verbs: [ "get", "list", "watch", "update", "patch" ]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-upcloud-attacher-binding
subjects:
- kind: ServiceAccount
name: csi-upcloud-controller-sa
namespace: kube-system
roleRef:
kind: ClusterRole
name: csi-upcloud-attacher-role
apiGroup: rbac.authorization.k8s.io
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-upcloud-snapshotter-role
rules:
- apiGroups: [ "" ]
resources: [ "persistentvolumes" ]
verbs: [ "get", "list", "watch" ]
- apiGroups: [ "" ]
resources: [ "persistentvolumeclaims" ]
verbs: [ "get", "list", "watch" ]
- apiGroups: [ "storage.k8s.io" ]
resources: [ "storageclasses" ]
verbs: [ "get", "list", "watch" ]
- apiGroups: [ "" ]
resources: [ "events" ]
verbs: [ "list", "watch", "create", "update", "patch" ]
- apiGroups: [ "" ]
resources: [ "secrets" ]
verbs: [ "get", "list" ]
- apiGroups: [ "snapshot.storage.k8s.io" ]
resources: [ "volumesnapshotclasses" ]
verbs: [ "get", "list", "watch" ]
- apiGroups: [ "snapshot.storage.k8s.io" ]
resources: [ "volumesnapshotcontents" ]
verbs: [ "create", "get", "list", "watch", "update", "delete" ]
- apiGroups: [ "snapshot.storage.k8s.io" ]
resources: [ "volumesnapshots" ]
verbs: [ "get", "list", "watch", "update" ]
- apiGroups: [ "apiextensions.k8s.io" ]
resources: [ "customresourcedefinitions" ]
verbs: [ "create", "list", "watch", "delete" ]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-upcloud-snapshotter-binding
subjects:
- kind: ServiceAccount
name: csi-upcloud-controller-sa
namespace: kube-system
roleRef:
kind: ClusterRole
name: csi-upcloud-snapshotter-role
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-upcloud-node-sa
namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-upcloud-node-driver-registrar-role
namespace: kube-system
rules:
- apiGroups: [ "" ]
resources: [ "events" ]
verbs: [ "get", "list", "watch", "create", "update", "patch" ]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-upcloud-node-driver-registrar-binding
subjects:
- kind: ServiceAccount
name: csi-upcloud-node-sa
namespace: kube-system
roleRef:
kind: ClusterRole
name: csi-upcloud-node-driver-registrar-role
apiGroup: rbac.authorization.k8s.io
---
# Resizer must be able to work with PVCs, PVs, SCs.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-upcloud-resizer-role
rules:
- apiGroups: [ "" ]
resources: [ "persistentvolumes" ]
verbs: [ "get", "list", "watch", "update", "patch" ]
- apiGroups: [ "" ]
resources: [ "persistentvolumeclaims" ]
verbs: [ "get", "list", "watch" ]
- apiGroups: [ "" ]
resources: [ "persistentvolumeclaims/status" ]
verbs: [ "update", "patch" ]
- apiGroups: [ "" ]
resources: [ "events" ]
verbs: [ "list", "watch", "create", "update", "patch" ]
- apiGroups: [ "" ]
resources: [ "pods" ]
verbs: [ "watch", "list" ]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-upcloud-resizer-binding
subjects:
- kind: ServiceAccount
name: csi-upcloud-controller-sa
namespace: kube-system
roleRef:
kind: ClusterRole
name: csi-upcloud-resizer-role
apiGroup: rbac.authorization.k8s.io

View file

@ -65,6 +65,13 @@ dependencies:
- gcp-pd-csi-driver - gcp-pd-csi-driver
- csi-driver - csi-driver
- role: kubernetes-apps/csi_driver/upcloud
when:
- upcloud_csi_enabled
tags:
- upcloud-csi-driver
- csi-driver
- role: kubernetes-apps/csi_driver/vsphere - role: kubernetes-apps/csi_driver/vsphere
when: when:
- vsphere_csi_enabled - vsphere_csi_enabled

View file

@ -34,3 +34,10 @@ dependencies:
tags: tags:
- persistent_volumes_gcp_pd_csi - persistent_volumes_gcp_pd_csi
- gcp-pd-csi-driver - gcp-pd-csi-driver
- role: kubernetes-apps/persistent_volumes/upcloud-csi
when:
- upcloud_csi_enabled
tags:
- persistent_volumes_upcloud_csi
- upcloud-csi-driver

View file

@ -0,0 +1,5 @@
---
expand_persistent_volumes: true
storage_classes:
- name: standard
is_default: true

View file

@ -0,0 +1,19 @@
---
- name: Kubernetes Persistent Volumes | Copy UpCloud CSI Storage Class template
template:
src: "upcloud-csi-storage-class.yml.j2"
dest: "{{ kube_config_dir }}/upcloud-csi-storage-class.yml"
register: manifests
when:
- inventory_hostname == groups['kube_control_plane'][0]
- name: Kubernetes Persistent Volumes | Add UpCloud CSI Storage Class
kube:
name: upcloud-csi
kubectl: "{{ bin_dir }}/kubectl"
resource: StorageClass
filename: "{{ kube_config_dir }}/upcloud-csi-storage-class.yml"
state: "latest"
when:
- inventory_hostname == groups['kube_control_plane'][0]
- manifests.changed

View file

@ -0,0 +1,11 @@
{% for class in storage_classes %}
---
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: "{{ class.name }}"
annotations:
storageclass.kubernetes.io/is-default-class: "{{ class.is_default | default(false) | ternary("true","false") }}"
provisioner: storage.csi.upcloud.com
allowVolumeExpansion: {{ expand_persistent_volumes }}
{% endfor %}

View file

@ -1,5 +1,6 @@
--- ---
kubeconfig_localhost: false kubeconfig_localhost: false
kubeconfig_localhost_ansible_host: false
kubectl_localhost: false kubectl_localhost: false
artifacts_dir: "{{ inventory_dir }}/artifacts" artifacts_dir: "{{ inventory_dir }}/artifacts"

View file

@ -4,6 +4,8 @@
external_apiserver_address: >- external_apiserver_address: >-
{%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined -%} {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined -%}
{{ loadbalancer_apiserver.address }} {{ loadbalancer_apiserver.address }}
{%- elif kubeconfig_localhost_ansible_host is defined and kubeconfig_localhost_ansible_host -%}
{{ hostvars[groups['kube_control_plane'][0]].ansible_host }}
{%- else -%} {%- else -%}
{{ kube_apiserver_access_address }} {{ kube_apiserver_access_address }}
{%- endif -%} {%- endif -%}

View file

@ -403,6 +403,7 @@ aws_ebs_csi_enabled: false
azure_csi_enabled: false azure_csi_enabled: false
gcp_pd_csi_enabled: false gcp_pd_csi_enabled: false
vsphere_csi_enabled: false vsphere_csi_enabled: false
upcloud_csi_enabled: false
csi_snapshot_controller_enabled: false csi_snapshot_controller_enabled: false
persistent_volumes_enabled: false persistent_volumes_enabled: false
cephfs_provisioner_enabled: false cephfs_provisioner_enabled: false