Add cloud provider config to kubeadm deployments (#3766)

inventory/sample/group_vars/all/all.yml

@@ -42,9 +42,10 @@ bin_dir: /usr/local/bin
 ## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', or 'external'
 ## When openstack is used make sure to source in the openstack credentials
 ## like you would do when using nova-client before starting the playbook.
+## Note: The 'external' cloud provider is not supported. 
+## TODO(riverzhang): https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/#running-cloud-controller-manager
 #cloud_provider:
 
-
 ## kubeadm deployment mode
 kubeadm_enabled: true
 

roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2

@@ -22,8 +22,11 @@ networking:
   podSubnet: {{ kube_pods_subnet }}
   podNetworkCidr: "{{ kube_network_node_prefix }}"
 kubernetesVersion: {{ kube_version }}
-{% if cloud_provider is defined and cloud_provider not in ["gce", "oci"] %}
-cloudProvider: {{ cloud_provider }}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+cloudProvider: {{cloud_provider}}
+cloudConfig: {{ kube_config_dir }}/cloud_config
+{% elif cloud_provider is defined and cloud_provider in ["external"] %}
+cloudConfig: {{ kube_config_dir }}/cloud_config
 {% endif %}
 {% if kube_proxy_mode == 'ipvs' %}
 kubeProxy:
@@ -40,7 +43,7 @@ kubeProxy:
 {% if kube_proxy_nodeport_addresses %}
     nodePortAddresses: [{{ kube_proxy_nodeport_addresses_cidr }}]
 {% endif %}
-    resourceContainer: ""
+resourceContainer: ""
 authorizationModes:
 {% for mode in authorization_modes %}
 - {{ mode }}
@@ -111,6 +114,9 @@ apiServerExtraArgs:
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
+{% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %}
+  configure-cloud-routes: "true"
+{% endif %}
 controllerManagerExtraArgs:
   node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
   node-monitor-period: {{ kube_controller_node_monitor_period }}
@@ -123,12 +129,19 @@ controllerManagerExtraArgs:
 {% for key in kube_kubeadm_controller_extra_args %}
   {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
 {% endfor %}
-{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
 controllerManagerExtraVolumes:
+{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %}
 - name: openstackcacert
   hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
   mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
 {% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
+- name: cloud-config
+  hostPath: {{ kube_config_dir }}/cloud_config
+  mountPath: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% endif %}
 schedulerExtraArgs:
   profiling: "{{ kube_profiling }}"
 {% if kube_feature_gates %}
@@ -141,6 +154,11 @@ schedulerExtraArgs:
 {% endif %}
 {% if kube_basic_auth|default(true) or kube_token_auth|default(true) %}
 apiServerExtraVolumes:
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
+- name: cloud-config
+  hostPath: {{ kube_config_dir }}/cloud_config
+  mountPath: {{ kube_config_dir }}/cloud_config
+{% endif %}
 {% if kube_basic_auth|default(true) %}
 - name: basic-auth-config
   hostPath: {{ kube_users_dir }}

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -23,9 +23,6 @@ networking:
   podSubnet: {{ kube_pods_subnet }}
   podNetworkCidr: "{{ kube_network_node_prefix }}"
 kubernetesVersion: {{ kube_version }}
-{% if cloud_provider is defined and cloud_provider != "gce" %}
-cloudProvider: {{ cloud_provider }}
-{% endif %}
 kubeProxy:
   config:
     mode: {{ kube_proxy_mode }}
@@ -109,6 +106,15 @@ apiServerExtraArgs:
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+  cloud-provider: {{cloud_provider}}
+  cloud-config: {{ kube_config_dir }}/cloud_config
+{% elif cloud_provider is defined and cloud_provider in ["external"] %}
+  cloud-config: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %}
+  configure-cloud-routes: "true"
+{% endif %}
 controllerManagerExtraArgs:
   node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
   node-monitor-period: {{ kube_controller_node_monitor_period }}
@@ -121,12 +127,25 @@ controllerManagerExtraArgs:
 {% for key in kube_kubeadm_controller_extra_args %}
   {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
 {% endfor %}
-{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+  cloud-provider: {{cloud_provider}}
+  cloud-config: {{ kube_config_dir }}/cloud_config
+{% elif cloud_provider is defined and cloud_provider in ["external"] %}
+  cloud-config: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
 controllerManagerExtraVolumes:
+{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %}
 - name: openstackcacert
   hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
   mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
 {% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
+- name: cloud-config
+  hostPath: {{ kube_config_dir }}/cloud_config
+  mountPath: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% endif %}
 {% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) %}
 apiServerExtraVolumes:
 {% if kube_basic_auth|default(true) %}
@@ -151,11 +170,19 @@ apiServerExtraVolumes:
 {% endif %}
 {% endif %}
 {% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
+- name: cloud-config
+  hostPath: {{ kube_config_dir }}/cloud_config
+  mountPath: {{ kube_config_dir }}/cloud_config
+{% endif %}
 schedulerExtraArgs:
   profiling: "{{ kube_profiling }}"
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
+{% if volume_cross_zone_attachment %}
+  policy-config-file: {{ kube_config_dir }}/kube-scheduler-policy.yaml
+{% endif %}
 {% if kube_kubeadm_scheduler_extra_args|length > 0 %}
 {% for key in kube_kubeadm_scheduler_extra_args %}
   {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"

roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2

@@ -43,6 +43,13 @@ controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.po
 {% else %}
 controlPlaneEndpoint: {{ ip | default(ansible_default_ipv4.address) }}:{{ kube_apiserver_port }}
 {% endif %}
+apiServerCertSANs:
+{% for san in  apiserver_sans.split(' ') | unique %}
+  - {{ san }}
+{% endfor %}
+certificatesDir: {{ kube_config_dir }}/ssl
+imageRepository: {{ kube_image_repo }}
+unifiedControlPlaneImage: ""
 apiServerExtraArgs:
   authorization-mode: {{ authorization_modes | join(',') }}
   bind-address: {{ kube_apiserver_bind_address }}
@@ -109,6 +116,12 @@ apiServerExtraArgs:
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+  cloud-provider: {{cloud_provider}}
+  cloud-config: {{ kube_config_dir }}/cloud_config
+{% elif cloud_provider is defined and cloud_provider in ["external"] %}
+  cloud-config: {{ kube_config_dir }}/cloud_config
+{% endif %}
 controllerManagerExtraArgs:
   node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
   node-monitor-period: {{ kube_controller_node_monitor_period }}
@@ -116,14 +129,28 @@ controllerManagerExtraArgs:
 {% if kube_feature_gates %}
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
-{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
-controllerManagerExtraVolumes:
-- name: openstackcacert
-  hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
-  mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+  cloud-provider: {{cloud_provider}}
+  cloud-config: {{ kube_config_dir }}/cloud_config
+{% elif cloud_provider is defined and cloud_provider in ["external"] %}
+  cloud-config: {{ kube_config_dir }}/cloud_config
 {% endif %}
-{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) %}
+schedulerExtraArgs:
+{% if kube_feature_gates %}
+  feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+{% if kube_kubeadm_scheduler_extra_args|length > 0 %}
+{% for key in kube_kubeadm_scheduler_extra_args %}
+  {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
+{% endfor %}
+{% endif %}
+{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) %}
 apiServerExtraVolumes:
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
+- name: cloud-config
+  hostPath: {{ kube_config_dir }}/cloud_config
+  mountPath: {{ kube_config_dir }}/cloud_config
+{% endif %}
 {% if kube_basic_auth|default(true) %}
 - name: basic-auth-config
   hostPath: {{ kube_users_dir }}
@@ -149,22 +176,19 @@ apiServerExtraVolumes:
 {% for key in kube_kubeadm_controller_extra_args %}
   {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
 {% endfor %}
-schedulerExtraArgs:
-{% if kube_feature_gates %}
-  feature-gates: {{ kube_feature_gates|join(',') }}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
+controllerManagerExtraVolumes:
+{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
+- name: openstackcacert
+  hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+  mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
+- name: cloud-config
+  hostPath: {{ kube_config_dir }}/cloud_config
+  mountPath: {{ kube_config_dir }}/cloud_config
 {% endif %}
-{% if kube_kubeadm_scheduler_extra_args|length > 0 %}
-{% for key in kube_kubeadm_scheduler_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
-{% endfor %}
 {% endif %}
-apiServerCertSANs:
-{% for san in  apiserver_sans.split(' ') | unique %}
-  - {{ san }}
-{% endfor %}
-certificatesDir: {{ kube_config_dir }}/ssl
-imageRepository: {{ kube_image_repo }}
-unifiedControlPlaneImage: ""
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -106,8 +106,8 @@ KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kuben
 KUBE_ALLOW_PRIV="--allow-privileged=true"
 {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
 KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"
-{% elif cloud_provider is defined and cloud_provider in ["oci", "external"] %}
-KUBELET_CLOUDPROVIDER="--cloud-provider=external"
+{% elif cloud_provider is defined and cloud_provider in ["external"] %}
+KUBELET_CLOUDPROVIDER="--cloud-provider=external --cloud-config={{ kube_config_dir }}/cloud_config"
 {% else %}
 KUBELET_CLOUDPROVIDER=""
 {% endif %}