From c11f981692f8c1c7d9f6c918ad7ed21a19adc528 Mon Sep 17 00:00:00 2001 From: Jan Jungnickel Date: Mon, 7 Nov 2016 12:11:16 +0100 Subject: [PATCH 1/2] Initial support for vsphere as cloud provider --- inventory/group_vars/all.yml | 2 +- .../manifests/kube-apiserver.manifest.j2 | 2 +- roles/kubernetes/node/templates/kubelet.j2 | 2 +- roles/kubernetes/preinstall/defaults/main.yml | 10 +++++++ roles/kubernetes/preinstall/tasks/main.yml | 16 ++++++---- .../tasks/vsphere-credential-check.yml | 30 +++++++++++++++++++ .../templates/vsphere-cloud-config.j2 | 9 ++++++ 7 files changed, 62 insertions(+), 9 deletions(-) create mode 100644 roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml create mode 100644 roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2 diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 56a777e05..12ca18d9b 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -47,7 +47,7 @@ ## There are some changes specific to the cloud providers ## for instance we need to encapsulate packets with some network plugins -## If set the possible values are either 'gce', 'aws', 'azure' or 'openstack' +## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', or 'vsphere' ## When openstack is used make sure to source in the openstack credentials ## like you would do when using nova-client before starting the playbook. #cloud_provider: diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index c05030697..c4882bd56 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -51,7 +51,7 @@ spec: {% endif %} - --v={{ kube_log_level }} - --allow-privileged=true -{% if cloud_provider is defined and cloud_provider in ["openstack", "azure"] %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %} - --cloud-provider={{ cloud_provider }} - --cloud-config={{ kube_config_dir }}/cloud_config {% elif cloud_provider is defined and cloud_provider == "aws" %} diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index 8ec348a05..54d3b38aa 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -42,7 +42,7 @@ KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kuben {% endif %} # Should this cluster be allowed to run privileged docker containers KUBE_ALLOW_PRIV="--allow-privileged=true" -{% if cloud_provider is defined and cloud_provider in ["openstack", "azure"] %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere" ] %} KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config" {% elif cloud_provider is defined and cloud_provider == "aws" %} KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }}" diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index 59076c204..517b91b72 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -32,3 +32,13 @@ openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')|default(lookup('env','OS_P # Container Linux by CoreOS cloud init config file to define /etc/resolv.conf content # for hostnet pods and infra needs resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf + +# For the vSphere integration kubelet will need credentials to access +# the api. Per default this values will be +# read from the environment. +vsphere_username: "{{ lookup('env', VSPHERE_USERNAME ) }}" +vsphere_password: "{{ lookup('env', VSPHERE_PASSWORD ) }}" +vsphere_server: "{{ lookup('env', VSPHERE_SERVER ) }}" +vsphere_datacenter: "{{ lookup('env', VSPHERE_DATACENTER ) }}" +vsphere_datastore: "{{ lookup('env', VSPHERE_DATASTORE ) }}" +vsphere_working_dir: "{{ lookup('env', VSPHERE_WORKING_DIR ) }}" diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 5b79c101d..8dd422353 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -64,8 +64,8 @@ - name: check cloud_provider value fail: - msg: "If set the 'cloud_provider' var must be set either to 'generic', 'gce', 'aws', 'azure' or 'openstack'" - when: cloud_provider is defined and cloud_provider not in ['generic', 'gce', 'aws', 'openstack', 'azure'] + msg: "If set the 'cloud_provider' var must be set either to 'generic', 'gce', 'aws', 'azure', 'vsphere', or 'openstack'" + when: cloud_provider is defined and cloud_provider not in ['generic', 'gce', 'aws', 'openstack', 'azure', 'vsphere'] tags: [cloud-provider, facts] - include: openstack-credential-check.yml @@ -76,6 +76,10 @@ when: cloud_provider is defined and cloud_provider == 'azure' tags: [cloud-provider, azure, facts] +- include: vsphere-credential-check.yml + when: cloud_provider is defined and cloud_provider == 'vsphere' + tags: [cloud-provider, vsphere, facts] + - name: Create cni directories file: path: "{{ item }}" @@ -179,14 +183,14 @@ state: present tags: bootstrap-os -- name: Write openstack cloud-config +- name: "Write {{ cloud_provider}} cloud-config" template: - src: openstack-cloud-config.j2 + src: "{{ cloud_provider }}-cloud-config.j2" dest: "{{ kube_config_dir }}/cloud_config" group: "{{ kube_cert_group }}" mode: 0640 - when: inventory_hostname in groups['k8s-cluster'] and cloud_provider is defined and cloud_provider == "openstack" - tags: [cloud-provider, openstack] + when: inventory_hostname in groups['k8s-cluster'] and cloud_provider is defined and cloud_provider in [ "openstack", "vsphere" ] + tags: [cloud-provider, openstack, vsphere] - name: Write azure cloud-config template: diff --git a/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml b/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml new file mode 100644 index 000000000..8f0b3ba50 --- /dev/null +++ b/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml @@ -0,0 +1,30 @@ +--- +- name: check vsphere_username value + fail: + msg: "vsphere_username is missing" + when: vsphere_username is not defined or vsphere_username == "" + +- name: check vsphere_password value + fail: + msg: "vsphere_password is missing" + when: vsphere_password is not defined or vsphere_password == "" + +- name: check vsphere_server value + fail: + msg: "vsphere_server is missing" + when: vsphere_server is not defined or vsphere_server == "" + +- name: check vsphere_datacenter value + fail: + msg: "vsphere_datacenter is missing" + when: vsphere_datacenter is not defined or vsphere_datacenter == "" + +- name: check vsphere_datastore value + fail: + msg: "vsphere_datastore is missing" + when: vsphere_datastore is not defined or vsphere_datastore == "" + +- name: check vsphere_working_dir value + fail: + msg: "vsphere_working_dir is missing" + when: vsphere_working_dir is not defined or vsphere_working_dir == "" diff --git a/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2 b/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2 new file mode 100644 index 000000000..d350f973c --- /dev/null +++ b/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2 @@ -0,0 +1,9 @@ +[Global] +user = {{ vsphere_username }} +password = {{ vsphere_password }} +server = {{ vsphere_server }} +port = 443 +insecure-flag = true +datacenter = {{ vsphere_datacenter }} +datastore = {{ vsphere_datastore }} +working-dir = {{ vsphere_working_dir }} From 6a144213c93047fc6f08456f40abb5726e206e15 Mon Sep 17 00:00:00 2001 From: Brad Beam Date: Thu, 16 Feb 2017 21:59:40 -0600 Subject: [PATCH 2/2] Updating vsphere cloud provider support --- .../kube-controller-manager.manifest.j2 | 6 +-- roles/kubernetes/node/templates/kubelet.j2 | 2 +- roles/kubernetes/preinstall/defaults/main.yml | 26 ++++++---- roles/kubernetes/preinstall/tasks/main.yml | 33 +++---------- .../tasks/vsphere-credential-check.yml | 49 ++++++++----------- .../templates/vsphere-cloud-config.j2 | 21 ++++++-- 6 files changed, 64 insertions(+), 73 deletions(-) diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 7bcd51cc4..6faf6dea5 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -32,7 +32,7 @@ spec: - --node-monitor-period={{ kube_controller_node_monitor_period }} - --pod-eviction-timeout={{ kube_controller_pod_eviction_timeout }} - --v={{ kube_log_level }} -{% if cloud_provider is defined and cloud_provider in ["openstack", "azure"] %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %} - --cloud-provider={{cloud_provider}} - --cloud-config={{ kube_config_dir }}/cloud_config {% elif cloud_provider is defined and cloud_provider == "aws" %} @@ -54,7 +54,7 @@ spec: - mountPath: {{ kube_cert_dir }} name: ssl-certs-kubernetes readOnly: true -{% if cloud_provider is defined and cloud_provider in ["openstack", "azure"] %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere" ] %} - mountPath: {{ kube_config_dir }}/cloud_config name: cloudconfig readOnly: true @@ -63,7 +63,7 @@ spec: - hostPath: path: {{ kube_cert_dir }} name: ssl-certs-kubernetes -{% if cloud_provider is defined and cloud_provider in ["openstack", "azure"] %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %} - hostPath: path: {{ kube_config_dir }}/cloud_config name: cloudconfig diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index 54d3b38aa..10135c13f 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -42,7 +42,7 @@ KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kuben {% endif %} # Should this cluster be allowed to run privileged docker containers KUBE_ALLOW_PRIV="--allow-privileged=true" -{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere" ] %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %} KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config" {% elif cloud_provider is defined and cloud_provider == "aws" %} KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }}" diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index 517b91b72..c775f748d 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -29,16 +29,22 @@ openstack_password: "{{ lookup('env','OS_PASSWORD') }}" openstack_region: "{{ lookup('env','OS_REGION_NAME') }}" openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')|default(lookup('env','OS_PROJECT_ID'),true) }}" +# For the vsphere integration, kubelet will need credentials to access +# vsphere apis +# Documentation regarting these values can be found +# https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/providers/vsphere/vsphere.go#L105 +vsphere_vcenter_ip: "{{ lookup('env', 'VSPHERE_VCENTER') }}" +vsphere_vcenter_port: "{{ lookup('env', 'VSPHERE_VCENTER_PORT') }}" +vsphere_user: "{{ lookup('env', 'VSPHERE_USER') }}" +vsphere_password: "{{ lookup('env', 'VSPHERE_PASSWORD') }}" +vsphere_datacenter: "{{ lookup('env', 'VSPHERE_DATACENTER') }}" +vsphere_datastore: "{{ lookup('env', 'VSPHERE_DATASTORE') }}" +vsphere_working_dir: "{{ lookup('env', 'VSPHERE_WORKING_DIR') }}" +vsphere_insecure: "{{ lookup('env', 'VSPHERE_INSECURE') }}" +vsphere_scsi_controller_type: pvscsi +# vsphere_public_network is name of the network the VMs are joined to +vsphere_public_network: "{{ lookup('env', 'VSPHERE_PUBLIC_NETWORK')|default('') }}" + # Container Linux by CoreOS cloud init config file to define /etc/resolv.conf content # for hostnet pods and infra needs resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf - -# For the vSphere integration kubelet will need credentials to access -# the api. Per default this values will be -# read from the environment. -vsphere_username: "{{ lookup('env', VSPHERE_USERNAME ) }}" -vsphere_password: "{{ lookup('env', VSPHERE_PASSWORD ) }}" -vsphere_server: "{{ lookup('env', VSPHERE_SERVER ) }}" -vsphere_datacenter: "{{ lookup('env', VSPHERE_DATACENTER ) }}" -vsphere_datastore: "{{ lookup('env', VSPHERE_DATASTORE ) }}" -vsphere_working_dir: "{{ lookup('env', VSPHERE_WORKING_DIR ) }}" diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 8dd422353..c995ed21e 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -64,21 +64,13 @@ - name: check cloud_provider value fail: - msg: "If set the 'cloud_provider' var must be set either to 'generic', 'gce', 'aws', 'azure', 'vsphere', or 'openstack'" - when: cloud_provider is defined and cloud_provider not in ['generic', 'gce', 'aws', 'openstack', 'azure', 'vsphere'] + msg: "If set the 'cloud_provider' var must be set either to 'generic', 'gce', 'aws', 'azure', 'openstack' or 'vsphere'" + when: cloud_provider is defined and cloud_provider not in ['generic', 'gce', 'aws', 'azure', 'openstack', 'vsphere'] tags: [cloud-provider, facts] -- include: openstack-credential-check.yml - when: cloud_provider is defined and cloud_provider == 'openstack' - tags: [cloud-provider, openstack, facts] - -- include: azure-credential-check.yml - when: cloud_provider is defined and cloud_provider == 'azure' - tags: [cloud-provider, azure, facts] - -- include: vsphere-credential-check.yml - when: cloud_provider is defined and cloud_provider == 'vsphere' - tags: [cloud-provider, vsphere, facts] +- include: "{{ cloud_provider }}-credential-check.yml" + when: cloud_provider is defined and cloud_provider in [ 'openstack', 'azure', 'vsphere' ] + tags: [cloud-provider, "{{ cloud_provider }}", facts] - name: Create cni directories file: @@ -183,23 +175,14 @@ state: present tags: bootstrap-os -- name: "Write {{ cloud_provider}} cloud-config" +- name: Write cloud-config template: src: "{{ cloud_provider }}-cloud-config.j2" dest: "{{ kube_config_dir }}/cloud_config" group: "{{ kube_cert_group }}" mode: 0640 - when: inventory_hostname in groups['k8s-cluster'] and cloud_provider is defined and cloud_provider in [ "openstack", "vsphere" ] - tags: [cloud-provider, openstack, vsphere] - -- name: Write azure cloud-config - template: - src: azure-cloud-config.j2 - dest: "{{ kube_config_dir }}/cloud_config" - group: "{{ kube_cert_group }}" - mode: 0640 - when: inventory_hostname in groups['k8s-cluster'] and cloud_provider is defined and cloud_provider == "azure" - tags: [cloud-provider, azure] + when: inventory_hostname in groups['k8s-cluster'] and cloud_provider is defined and cloud_provider in [ 'openstack', 'azure', 'vsphere' ] + tags: [cloud-provider, "{{ cloud_provider }}"] - include: etchosts.yml tags: [bootstrap-os, etchosts] diff --git a/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml b/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml index 8f0b3ba50..b91726d50 100644 --- a/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml +++ b/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml @@ -1,30 +1,21 @@ ---- -- name: check vsphere_username value +- name: check vsphere environment variables fail: - msg: "vsphere_username is missing" - when: vsphere_username is not defined or vsphere_username == "" - -- name: check vsphere_password value - fail: - msg: "vsphere_password is missing" - when: vsphere_password is not defined or vsphere_password == "" - -- name: check vsphere_server value - fail: - msg: "vsphere_server is missing" - when: vsphere_server is not defined or vsphere_server == "" - -- name: check vsphere_datacenter value - fail: - msg: "vsphere_datacenter is missing" - when: vsphere_datacenter is not defined or vsphere_datacenter == "" - -- name: check vsphere_datastore value - fail: - msg: "vsphere_datastore is missing" - when: vsphere_datastore is not defined or vsphere_datastore == "" - -- name: check vsphere_working_dir value - fail: - msg: "vsphere_working_dir is missing" - when: vsphere_working_dir is not defined or vsphere_working_dir == "" + msg: "{{ item.name }} is missing" + when: item.value is not defined or item.value == '' + with_items: + - name: vsphere_vcenter_ip + value: "{{ vsphere_vcenter_ip }}" + - name: vsphere_vcenter_port + value: "{{ vsphere_vcenter_port }}" + - name: vsphere_user + value: "{{ vsphere_user }}" + - name: vsphere_password + value: "{{ vsphere_password }}" + - name: vsphere_datacenter + value: "{{ vsphere_datacenter }}" + - name: vsphere_datastore + value: "{{ vsphere_datastore }}" + - name: vsphere_working_dir + value: "{{ vsphere_working_dir }}" + - name: vsphere_insecure + value: "{{ vsphere_insecure }}" diff --git a/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2 b/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2 index d350f973c..c68ac0f55 100644 --- a/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2 +++ b/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2 @@ -1,9 +1,20 @@ [Global] -user = {{ vsphere_username }} -password = {{ vsphere_password }} -server = {{ vsphere_server }} -port = 443 -insecure-flag = true datacenter = {{ vsphere_datacenter }} datastore = {{ vsphere_datastore }} +insecure-flag = {{ vsphere_insecure }} +password = {{ vsphere_password }} +port = {{ vsphere_vcenter_port }} +server = {{ vsphere_vcenter_ip }} +user = {{ vsphere_user }} working-dir = {{ vsphere_working_dir }} +{% if vsphere_vm_uuid is defined %} +vm-uuid = {{ vsphere_vm_uuid }} +{% endif %} + +[Disk] +scsicontrollertype = {{ vsphere_scsi_controller_type }} + +{% if vsphere_public_network is defined and vsphere_public_network != "" %} +[Network] +public-network = {{ vsphere_public_network }} +{% endif %}