C12s/c12s-kubespray
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

External OpenStack Cloud Controller Manager implementation (#5491)

Browse source 
* External OpenStack Cloud Controller Manager implementation

* Adding controller image tag

* Minor fixes

* Restructuring the external cloud controller to work with KubeADM
This commit is contained in:
Ali Sanhaji 2020-02-18 13:47:28 +01:00 committed by GitHub
parent 277b347604
commit 646fd5f47b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 473 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cluster.yml
View file

@ -97,6 +97,7 @@
any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
roles:
- { role: kubespray-defaults}
- { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
- { role: kubernetes-apps/network_plugin, tags: network }
- { role: kubernetes-apps/policy_controller, tags: policy-controller }
- { role: kubernetes-apps/ingress_controller, tags: ingress-controller }

7
inventory/sample/group_vars/all/all.yml
View file

@ -51,10 +51,13 @@ loadbalancer_apiserver_healthcheck_port: 8081
## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', or 'external'
## When openstack is used make sure to source in the openstack credentials
## like you would do when using openstack-client before starting the playbook.
## Note: The 'external' cloud provider is not supported.
## TODO(riverzhang): https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/#running-cloud-controller-manager
# cloud_provider:
## When cloud_provider is set to 'external', you can set the cloud controller to deploy
## Supported cloud controllers are: 'openstack'
## When openstack is used make sure to source in the openstack credentials
# external_cloud_provider:
## Set these proxy values in order to update package manager and docker daemon to use proxies
# http_proxy: ""
# https_proxy: ""

15
inventory/sample/group_vars/all/openstack.yml
View file

@ -15,6 +15,21 @@
# openstack_lbaas_monitor_timeout: "30s"
# openstack_lbaas_monitor_max_retries: "3"
## Values for the external OpenStack Cloud Controller
# external_openstack_lbaas_network_id: "Neutron network ID to create LBaaS VIP"
# external_openstack_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP"
# external_openstack_lbaas_floating_network_id: "Neutron network ID to get floating IP from"
# external_openstack_lbaas_floating_subnet_id: "Neutron subnet ID to get floating IP from"
# external_openstack_lbaas_use_octavia: true
# external_openstack_lbaas_method: "ROUND_ROBIN"
# external_openstack_lbaas_create_monitor: false
# external_openstack_lbaas_monitor_delay: "1m"
# external_openstack_lbaas_monitor_timeout: "30s"
# external_openstack_lbaas_monitor_max_retries: "3"
## The tag of the external OpenStack Cloud Controller image
# external_openstack_cloud_controller_image_tag: "latest"
## To use Cinder CSI plugin to provision volumes set this value to true
## Make sure to source in the openstack credentials
# cinder_csi_enabled: true

12
roles/kubernetes-apps/external_cloud_controller/meta/main.yml Normal file
View file

@ -0,0 +1,12 @@
---
dependencies:
- role: kubernetes-apps/external_cloud_controller/openstack
when:
- cloud_provider is defined
- cloud_provider == "external"
- external_cloud_provider is defined
- external_cloud_provider == "openstack"
- inventory_hostname == groups['kube-master'][0]
tags:
- external-cloud-controller
- external-openstack

15
roles/kubernetes-apps/external_cloud_controller/openstack/defaults/main.yml Normal file
View file

@ -0,0 +1,15 @@
---
# The external cloud controller will need credentials to access
# openstack apis. Per default these values will be
# read from the environment.
external_openstack_auth_url: "{{ lookup('env','OS_AUTH_URL') }}"
external_openstack_username: "{{ lookup('env','OS_USERNAME') }}"
external_openstack_password: "{{ lookup('env','OS_PASSWORD') }}"
external_openstack_region: "{{ lookup('env','OS_REGION_NAME') }}"
external_openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID'),true) }}"
external_openstack_tenant_name: "{{ lookup('env','OS_TENANT_NAME') }}"
external_openstack_domain_name: "{{ lookup('env','OS_USER_DOMAIN_NAME') }}"
external_openstack_domain_id: "{{ lookup('env','OS_USER_DOMAIN_ID') }}"
external_openstack_cacert: "{{ lookup('env','OS_CACERT') }}"
external_openstack_cloud_controller_image_tag: "latest"

58
roles/kubernetes-apps/external_cloud_controller/openstack/tasks/main.yml Normal file
View file

@ -0,0 +1,58 @@
---
- include_tasks: openstack-credential-check.yml
tags: external-openstack
- name: External OpenStack Cloud Controller | Write cacert file
copy:
src: "{{ external_openstack_cacert }}"
dest: "{{ kube_config_dir }}/external-openstack-cacert.pem"
group: "{{ kube_cert_group }}"
mode: 0640
when:
- inventory_hostname in groups['k8s-cluster']
- external_openstack_cacert is defined
- external_openstack_cacert | length > 0
tags: external-openstack
- name: External OpenStack Cloud Controller | Write External OpenStack cloud-config
template:
src: "external-openstack-cloud-config.j2"
dest: "{{ kube_config_dir }}/external_openstack_cloud_config"
group: "{{ kube_cert_group }}"
mode: 0640
when: inventory_hostname == groups['kube-master'][0]
tags: external-openstack
- name: External OpenStack Cloud Controller | Get base64 cloud-config
slurp:
src: "{{ kube_config_dir }}/external_openstack_cloud_config"
register: external_openstack_cloud_config_secret
when: inventory_hostname == groups['kube-master'][0]
tags: external-openstack
- name: External OpenStack Cloud Controller | Generate Manifests
template:
src: "{{ item.file }}.j2"
dest: "{{ kube_config_dir }}/{{ item.file }}"
with_items:
- {name: external-openstack-cloud-config-secret, file: external-openstack-cloud-config-secret.yml}
- {name: external-openstack-cloud-controller-manager-roles, file: external-openstack-cloud-controller-manager-roles.yml}
- {name: external-openstack-cloud-controller-manager-role-bindings, file: external-openstack-cloud-controller-manager-role-bindings.yml}
- {name: external-openstack-cloud-controller-manager-ds, file: external-openstack-cloud-controller-manager-ds.yml}
register: external_openstack_manifests
when: inventory_hostname == groups['kube-master'][0]
tags: external-openstack
- name: External OpenStack Cloud Controller | Apply Manifests
kube:
kubectl: "{{ bin_dir }}/kubectl"
filename: "{{ kube_config_dir }}/{{ item.item.file }}"
state: "latest"
with_items:
- "{{ external_openstack_manifests.results }}"
when:
- inventory_hostname == groups['kube-master'][0]
- not item is skipped
loop_control:
label: "{{ item.item.file }}"
tags: external-openstack

34
roles/kubernetes-apps/external_cloud_controller/openstack/tasks/openstack-credential-check.yml Normal file
View file

@ -0,0 +1,34 @@
---
- name: External OpenStack Cloud Controller | check external_openstack_auth_url value
fail:
msg: "external_openstack_auth_url is missing"
when: external_openstack_auth_url is not defined or not external_openstack_auth_url
- name: External OpenStack Cloud Controller | check external_openstack_username value
fail:
msg: "external_openstack_username is missing"
when: external_openstack_username is not defined or not external_openstack_username
- name: External OpenStack Cloud Controller | check external_openstack_password value
fail:
msg: "external_openstack_password is missing"
when: external_openstack_password is not defined or not external_openstack_password
- name: External OpenStack Cloud Controller | check external_openstack_region value
fail:
msg: "external_openstack_region is missing"
when: external_openstack_region is not defined or not external_openstack_region
- name: External OpenStack Cloud Controller | check external_openstack_tenant_id value
fail:
msg: "one of external_openstack_tenant_id or external_openstack_tenant_name must be specified"
when:
- external_openstack_tenant_id is not defined or not external_openstack_tenant_id
- external_openstack_tenant_name is not defined
- name: External OpenStack Cloud Controller | check external_openstack_tenant_name value
fail:
msg: "one of external_openstack_tenant_id or external_openstack_tenant_name must be specified"
when:
- external_openstack_tenant_name is not defined or not external_openstack_tenant_name
- external_openstack_tenant_id is not defined

10
roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config-secret.yml.j2 Normal file
View file

@ -0,0 +1,10 @@
# This YAML file contains secret objects,
# which are necessary to run external openstack cloud controller.
kind: Secret
apiVersion: v1
metadata:
name: external-openstack-cloud-config
namespace: kube-system
data:
cloud.conf: {{ external_openstack_cloud_config_secret.content }}

41
roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-config.j2 Normal file
View file

@ -0,0 +1,41 @@
[Global]
auth-url="{{ external_openstack_auth_url }}"
username="{{ external_openstack_username }}"
password="{{ external_openstack_password }}"
region="{{ external_openstack_region }}"
{% if external_openstack_tenant_id is defined and external_openstack_tenant_id != "" %}
tenant-id="{{ external_openstack_tenant_id }}"
{% endif %}
{% if external_openstack_tenant_name is defined and external_openstack_tenant_name != "" %}
tenant-name="{{ external_openstack_tenant_name }}"
{% endif %}
{% if external_openstack_domain_name is defined and external_openstack_domain_name != "" %}
domain-name="{{ external_openstack_domain_name }}"
{% elif external_openstack_domain_id is defined and external_openstack_domain_id != "" %}
domain-id ="{{ external_openstack_domain_id }}"
{% endif %}
{% if external_openstack_cacert is defined and external_openstack_cacert != "" %}
ca-file="{{ kube_config_dir }}/external-openstack-cacert.pem"
{% endif %}
[LoadBalancer]
use-octavia={{ external_openstack_lbaas_use_octavia }}
create-monitor={{ openstack_lbaas_create_monitor }}
monitor-delay={{ openstack_lbaas_monitor_delay }}
monitor-timeout={{ openstack_lbaas_monitor_timeout }}
monitor-max-retries={{ openstack_lbaas_monitor_max_retries }}
{% if external_openstack_lbaas_method is defined %}
lb-method={{ external_openstack_lbaas_method }}
{% endif %}
{% if external_openstack_lbaas_network_id is defined %}
network-id={{ external_openstack_lbaas_network_id }}
{% endif %}
{% if external_openstack_lbaas_subnet_id is defined %}
subnet-id={{ external_openstack_lbaas_subnet_id }}
{% endif %}
{% if external_openstack_lbaas_floating_network_id is defined %}
floating-network-id={{ external_openstack_lbaas_floating_network_id }}
{% endif %}
{% if external_openstack_lbaas_flaoting_subnet_id is defined %}
floating-subnet-id={{ external_openstack_lbaas_floating_subnet_id }}
{% endif %}

92
roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2 Normal file
View file

@ -0,0 +1,92 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cloud-controller-manager
namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: openstack-cloud-controller-manager
namespace: kube-system
labels:
k8s-app: openstack-cloud-controller-manager
spec:
selector:
matchLabels:
k8s-app: openstack-cloud-controller-manager
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
k8s-app: openstack-cloud-controller-manager
spec:
nodeSelector:
node-role.kubernetes.io/master: ""
securityContext:
runAsUser: 1001
tolerations:
- key: node.cloudprovider.kubernetes.io/uninitialized
value: "true"
effect: NoSchedule
- key: node-role.kubernetes.io/master
effect: NoSchedule
serviceAccountName: cloud-controller-manager
containers:
- name: openstack-cloud-controller-manager
image: {{ docker_image_repo }}/k8scloudprovider/openstack-cloud-controller-manager:{{ external_openstack_cloud_controller_image_tag }}
args:
- /bin/openstack-cloud-controller-manager
- --v=1
- --cloud-config=$(CLOUD_CONFIG)
- --cloud-provider=openstack
- --use-service-account-credentials=true
- --address=127.0.0.1
volumeMounts:
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/config
name: cloud-config-volume
readOnly: true
{% if external_openstack_cacert is defined and external_openstack_cacert != "" %}
- mountPath: {{ kube_config_dir }}/external-openstack-cacert.pem
name: openstack-cacert
readOnly: true
{% endif %}
- mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
name: flexvolume-dir
resources:
requests:
cpu: 200m
env:
- name: CLOUD_CONFIG
value: /etc/config/cloud.conf
hostNetwork: true
volumes:
- hostPath:
path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
type: DirectoryOrCreate
name: flexvolume-dir
- hostPath:
path: /etc/kubernetes/pki
type: DirectoryOrCreate
name: k8s-certs
- hostPath:
path: /etc/ssl/certs
type: DirectoryOrCreate
name: ca-certs
- name: cloud-config-volume
secret:
secretName: external-openstack-cloud-config
{% if external_openstack_cacert is defined and external_openstack_cacert != "" %}
- hostPath:
path: {{ kube_config_dir }}/external-openstack-cacert.pem
type: FileOrCreate
name: openstack-cacert
{% endif %}

40
roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-role-bindings.yml.j2 Normal file
View file

@ -0,0 +1,40 @@
apiVersion: v1
items:
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:cloud-node-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:cloud-node-controller
subjects:
- kind: ServiceAccount
name: cloud-node-controller
namespace: kube-system
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:pvl-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:pvl-controller
subjects:
- kind: ServiceAccount
name: pvl-controller
namespace: kube-system
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:cloud-controller-manager
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:cloud-controller-manager
subjects:
- kind: ServiceAccount
name: cloud-controller-manager
namespace: kube-system
kind: List
metadata: {}

129
roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-roles.yml.j2 Normal file
View file

@ -0,0 +1,129 @@
apiVersion: v1
items:
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:cloud-controller-manager
rules:
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- get
- create
- update
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
- apiGroups:
- ""
resources:
- nodes
verbs:
- '*'
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
- apiGroups:
- ""
resources:
- services
verbs:
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- serviceaccounts
verbs:
- create
- get
- apiGroups:
- ""
resources:
- persistentvolumes
verbs:
- '*'
- apiGroups:
- ""
resources:
- endpoints
verbs:
- create
- get
- list
- watch
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- list
- get
- watch
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:cloud-node-controller
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- '*'
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:pvl-controller
rules:
- apiGroups:
- ""
resources:
- persistentvolumes
verbs:
- '*'
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
kind: List
metadata: {}

14
roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2
View file

@ -21,6 +21,10 @@ nodeRegistration:
taints: []
{% endif %}
criSocket: {{ cri_socket }}
{% if cloud_provider is defined and cloud_provider in ["external"] %}
kubeletExtraArgs:
cloud-provider: external
{% endif %}
---
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
@ -170,12 +174,10 @@ apiServer:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
cloud-provider: {{cloud_provider}}
cloud-config: {{ kube_config_dir }}/cloud_config
{% elif cloud_provider is defined and cloud_provider in ["external"] %}
cloud-config: {{ kube_config_dir }}/cloud_config
{% endif %}
{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or kube_webhook_token_auth|default(false) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) or apiserver_extra_volumes or ssl_ca_dirs|length %}
extraVolumes:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
- name: cloud-config
hostPath: {{ kube_config_dir }}/cloud_config
mountPath: {{ kube_config_dir }}/cloud_config
@ -244,20 +246,18 @@ controllerManager:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
cloud-provider: {{cloud_provider}}
cloud-config: {{ kube_config_dir }}/cloud_config
{% elif cloud_provider is defined and cloud_provider in ["external"] %}
cloud-config: {{ kube_config_dir }}/cloud_config
{% endif %}
{% if kube_network_plugin is defined and kube_network_plugin not in ["cloud"] %}
configure-cloud-routes: "false"
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] or controller_manager_extra_volumes %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] or controller_manager_extra_volumes %}
extraVolumes:
{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
- name: openstackcacert
hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
- name: cloud-config
hostPath: {{ kube_config_dir }}/cloud_config
mountPath: {{ kube_config_dir }}/cloud_config

14
roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2
View file

@ -24,6 +24,10 @@ nodeRegistration:
taints: []
{% endif %}
criSocket: {{ cri_socket }}
{% if cloud_provider is defined and cloud_provider in ["external"] %}
kubeletExtraArgs:
cloud-provider: external
{% endif %}
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
@ -173,12 +177,10 @@ apiServer:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
cloud-provider: {{ cloud_provider }}
cloud-config: {{ kube_config_dir }}/cloud_config
{% elif cloud_provider is defined and cloud_provider in ["external"] %}
cloud-config: {{ kube_config_dir }}/cloud_config
{% endif %}
{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or kube_webhook_token_auth|default(false) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) or apiserver_extra_volumes or ssl_ca_dirs|length %}
extraVolumes:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
- name: cloud-config
hostPath: {{ kube_config_dir }}/cloud_config
mountPath: {{ kube_config_dir }}/cloud_config
@ -247,20 +249,18 @@ controllerManager:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
cloud-provider: {{ cloud_provider }}
cloud-config: {{ kube_config_dir }}/cloud_config
{% elif cloud_provider is defined and cloud_provider in ["external"] %}
cloud-config: {{ kube_config_dir }}/cloud_config
{% endif %}
{% if kube_network_plugin is defined and kube_network_plugin not in ["cloud"] %}
configure-cloud-routes: "false"
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] or controller_manager_extra_volumes %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] or controller_manager_extra_volumes %}
extraVolumes:
{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
- name: openstackcacert
hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
- name: cloud-config
hostPath: {{ kube_config_dir }}/cloud_config
mountPath: {{ kube_config_dir }}/cloud_config

7
roles/kubespray-defaults/defaults/main.yaml
View file

@ -323,6 +323,13 @@ openstack_lbaas_monitor_timeout: "30s"
openstack_lbaas_monitor_max_retries: "3"
openstack_cacert: "{{ lookup('env','OS_CACERT') }}"
# Default values for the external OpenStack Cloud Controller
external_openstack_lbaas_use_octavia: true
external_openstack_lbaas_create_monitor: false
external_openstack_lbaas_monitor_delay: "1m"
external_openstack_lbaas_monitor_timeout: "30s"
external_openstack_lbaas_monitor_max_retries: "3"
## List of authorization modes that must be configured for
## the k8s cluster. Only 'AlwaysAllow', 'AlwaysDeny', 'Node' and
## 'RBAC' modes are tested. Order is important.