Upgrade kubernetes to v1.13.0 (#3810)

* Upgrade kubernetes to v1.13.0

* Remove all precense of scheduler.alpha.kubernetes.io/critical-pod in templates

* Fix cert dir

* Use kubespray v2.8 as baseline for gitlab

.gitlab-ci.yml

@@ -41,7 +41,7 @@ before_script:
   tags:
     - kubernetes
     - docker
-  image: quay.io/kubespray/kubespray:v2.7
+  image: quay.io/kubespray/kubespray:v2.8
 
 .docker_service: &docker_service
   services:
@@ -88,11 +88,11 @@ before_script:
     - echo ${PWD}
     - echo "${STARTUP_SCRIPT}"
     - cd tests && make create-${CI_PLATFORM} -s ; cd -
-    #- git fetch --all && git checkout v2.7.0
 
     # Check out latest tag if testing upgrade
     # Uncomment when gitlab kubespray repo has tags
-    - test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
+    #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
+    - test "${UPGRADE_TEST}" != "false" && git checkout 9051aa5296ef76fcff69a2e3827cef28752aa475
     # Checkout the CI vars file so it is available
     - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
     # Workaround https://github.com/kubernetes-sigs/kubespray/issues/2021

README.md

@@ -111,7 +111,7 @@ Supported Components
 --------------------
 
 -   Core
-    -   [kubernetes](https://github.com/kubernetes/kubernetes) v1.12.3
+    -   [kubernetes](https://github.com/kubernetes/kubernetes) v1.13.0
     -   [etcd](https://github.com/coreos/etcd) v3.2.24
     -   [docker](https://www.docker.com/) v18.06 (see note)
     -   [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2)

inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml

@@ -19,7 +19,7 @@ kube_users_dir: "{{ kube_config_dir }}/users"
 kube_api_anonymous_auth: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.12.3
+kube_version: v1.13.0
 
 # kubernetes image repo define
 kube_image_repo: "gcr.io/google-containers"

roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2

@@ -28,7 +28,6 @@ spec:
       labels:
         k8s-app: dnsmasq-autoscaler
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}

roles/download/defaults/main.yml

@@ -35,7 +35,7 @@ download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube
 image_arch: "{{host_architecture | default('amd64')}}"
 
 # Versions
-kube_version: v1.12.3
+kube_version: v1.13.0
 kubeadm_version: "{{ kube_version }}"
 etcd_version: v3.2.24
 
@@ -70,6 +70,7 @@ cni_download_url: "https://github.com/containernetworking/plugins/releases/downl
 
 # Checksums
 hyperkube_checksums:
+  v1.13.0: 754f1baae5dc2ba29afc66e1f5d3b676ee59cd5c40ccce813092408d53bde3d9
   v1.12.3: 600aad3f0d016716abd85931239806193ffbe95f2edfdcea11532d518ae5cdb1
   v1.12.2: 566dfed398c20c9944f8999d6370cb584cb8c228b3c5881137b6b3d9306e4b06
   v1.12.1: 4aa23cfb2fc2e2e4d0cbe0d83a648c38e4baabd6c66f5cdbbb40cbc7582fdc74
@@ -88,6 +89,7 @@ hyperkube_checksums:
   v1.10.1: 6e0642ad6bae68dc81b8d1c9efa18e265e17e23da1895862823cafac08c0344c
   v1.10.0: b5575b2fb4266754c1675b8cd5d9b6cac70f3fee7a05c4e80da3a9e83e58c57e
 kubeadm_checksums:
+  v1.13.0: f5366206416dc4cfc840a7add2289957b56ccc479cc1b74f7397a4df995d6b06
   v1.12.3: c675aa3be82754b3f8dfdde2a1526a72986713312d46d898e65cb564c6aa8ad4
   v1.12.2: 51bc4bfd1d934a27245111c0ad1f793d5147ed15389415a1509502f23fcfa642
   v1.12.1: 5d95efd65aad398d85a9802799f36410ae7a95f9cbe73c8b10d2213c10a6d7be

roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2

@@ -31,7 +31,6 @@ spec:
       labels:
         k8s-app: dns-autoscaler{{ coredns_ordinal_suffix | default('') }}
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}

roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2

@@ -25,7 +25,6 @@ spec:
       labels:
         k8s-app: kube-dns
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}

roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2

@@ -14,8 +14,6 @@ spec:
     metadata:
       labels:
         k8s-app: nvidia-gpu-device-plugin
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-node-critical
       affinity:

roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2

@@ -22,8 +22,6 @@ spec:
     metadata:
       labels:
         name: nvidia-driver-installer
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-node-critical
       affinity:

roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2

@@ -21,7 +21,6 @@ spec:
         app.kubernetes.io/name: metrics-server
         version: {{ metrics_server_version }}
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}

roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2

@@ -6,8 +6,6 @@ metadata:
   labels:
     k8s-app: calico-kube-controllers
     kubernetes.io/cluster-service: "true"
-  annotations:
-    scheduler.alpha.kubernetes.io/critical-pod: ''
 spec:
   replicas: 1
   strategy:

roles/kubernetes/kubeadm/tasks/main.yml

@@ -46,7 +46,14 @@
 - name: sets kubeadm api version to v1alpha3
   set_fact:
     kubeadmConfig_api_version: v1alpha3
-  when: kubeadm_output.stdout is version('v1.12.0', '>=')
+  when:
+    - kubeadm_output.stdout is version('v1.12.0', '>=')
+    - kubeadm_output.stdout is version('v1.13.0', '<')
+
+- name: sets kubeadm api version to v1beta1
+  set_fact:
+    kubeadmConfig_api_version: v1beta1
+  when: kubeadm_output.stdout is version('v1.13.0', '>=')
 
 - name: Create kubeadm client config
   template:

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha1.j2

@@ -1,6 +1,6 @@
 apiVersion: kubeadm.k8s.io/v1alpha1
 kind: NodeConfiguration
-caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+caCertPath: {{ kube_cert_dir }}/ca.crt
 token: {{ kubeadm_token }}
 discoveryTokenAPIServers:
 {% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %}

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2

@@ -2,7 +2,7 @@ apiVersion: kubeadm.k8s.io/v1alpha2
 kind: NodeConfiguration
 clusterName: {{ cluster_name }}
 discoveryFile: ""
-caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+caCertPath: {{ kube_cert_dir }}/ca.crt
 discoveryTimeout: {{ discovery_timeout }}
 discoveryToken: {{ kubeadm_token }}
 tlsBootstrapToken: {{ kubeadm_token }}

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2

@@ -2,7 +2,7 @@ apiVersion: kubeadm.k8s.io/v1alpha3
 kind: JoinConfiguration
 clusterName: {{ cluster_name }}
 discoveryFile: ""
-caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+caCertPath: {{ kube_cert_dir }}/ca.crt
 discoveryTimeout: {{ discovery_timeout }}
 discoveryToken: {{ kubeadm_token }}
 tlsBootstrapToken: {{ kubeadm_token }}

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2

@@ -0,0 +1,27 @@
+apiVersion: kubeadm.k8s.io/v1beta1
+kind: JoinConfiguration
+discovery:
+  bootstrapToken:
+{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %}
+    apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% else %}
+    apiServerEndpoint: {{ kubeadm_discovery_address | replace("https://", "")}}
+{% endif %}
+    token: {{ kubeadm_token }}
+    unsafeSkipCAVerification: true
+  timeout: {{ discovery_timeout }}
+  tlsBootstrapToken: {{ kubeadm_token }}
+{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %}
+controlPlane:
+  localAPIEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% endif %}
+caCertPath: {{ kube_cert_dir }}/ca.crt
+nodeRegistration:
+  name: {{ inventory_hostname  }}
+{% if container_manager == 'crio' %}
+  criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+  criSocket: /var/run/rkt.sock
+{% else %}
+  criSocket: /var/run/dockershim.sock
+{% endif %}

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -103,7 +103,14 @@
 - name: sets kubeadm api version to v1alpha3
   set_fact:
     kubeadmConfig_api_version: v1alpha3
-  when: kubeadm_output.stdout is version('v1.12.0', '>=')
+  when:
+    - kubeadm_output.stdout is version('v1.12.0', '>=')
+    - kubeadm_output.stdout is version('v1.13.0', '<')
+
+- name: sets kubeadm api version to v1beta1
+  set_fact:
+    kubeadmConfig_api_version: v1beta1
+  when: kubeadm_output.stdout is version('v1.13.0', '>=')
 
 # Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint.
 - name: set kubeadm_config_api_fqdn define
@@ -144,15 +151,6 @@
   failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
   notify: Master | restart kubelet
 
-# FIXME(mattymo): remove when https://github.com/kubernetes/kubeadm/issues/433 is fixed
-- name: kubeadm | Enable kube-proxy
-  command: "{{ bin_dir }}/kubeadm alpha phase addon kube-proxy --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml"
-  register: kubeadm_kube_proxy_enable
-  retries: 10
-  until: kubeadm_kube_proxy_enable is succeeded
-  when: inventory_hostname == groups['kube-master']|first
-  changed_when: false
-
 - name: slurp kubeadm certs
   slurp:
     src: "{{ item }}"

roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2

@@ -13,9 +13,9 @@ etcd:
 {% for endpoint in etcd_access_addresses.split(',') %}
   - {{ endpoint }}
 {% endfor %}
-  caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
+  caFile: {{ etcd_cert_dir }}/ca.pem
-  certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
+  certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
-  keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+  keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
 networking:
   dnsDomain: {{ dns_domain }}
   serviceSubnet: {{ kube_service_addresses }}
@@ -69,6 +69,7 @@ apiServerExtraArgs:
 {% if kube_version is version('v1.9', '>=') %}
   endpoint-reconciler-type: lease
 {% endif %}
+  storage-backend: etcd3
 {% if etcd_events_cluster_enabled %}
   etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
 {% endif %}

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -14,9 +14,9 @@ etcd:
 {% for endpoint in etcd_access_addresses.split(',') %}
       - {{ endpoint }}
 {% endfor %}
-      caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
+      caFile: {{ etcd_cert_dir }}/ca.pem
-      certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
+      certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
-      keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+      keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
 networking:
   dnsDomain: {{ dns_domain }}
   serviceSubnet: {{ kube_service_addresses }}
@@ -54,6 +54,7 @@ apiServerExtraArgs:
 {% if kube_version is version('v1.9', '>=') %}
   endpoint-reconciler-type: lease
 {% endif %}
+  storage-backend: etcd3
 {% if etcd_events_cluster_enabled %}
   etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
 {% endif %}

roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2

@@ -29,9 +29,9 @@ etcd:
 {% for endpoint in etcd_access_addresses.split(',') %}
       - {{ endpoint }}
 {% endfor %}
-      caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
+      caFile: {{ etcd_cert_dir }}/ca.pem
-      certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
+      certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
-      keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+      keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
 networking:
   dnsDomain: {{ dns_domain }}
   serviceSubnet: {{ kube_service_addresses }}
@@ -71,6 +71,7 @@ apiServerExtraArgs:
 {% if kube_version is version('v1.9', '>=') %}
   endpoint-reconciler-type: lease
 {% endif %}
+  storage-backend: etcd3
 {% if etcd_events_cluster_enabled %}
   etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
 {% endif %}

roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2

@@ -0,0 +1,258 @@
+apiVersion: kubeadm.k8s.io/v1beta1
+kind: InitConfiguration
+localAPIEndpoint:
+  advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+  bindPort: {{ kube_apiserver_port }}
+nodeRegistration:
+{% if kube_override_hostname|default('') %}
+  name: {{ kube_override_hostname }}
+{% endif %}
+{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %}
+  taints:
+  - key: "kubeadmNode"
+    value: "master"
+    effect: "NoSchedule"
+{% endif %}
+{% if container_manager == 'crio' %}
+  criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+  criSocket: /var/run/rkt.sock
+{% else %}
+  criSocket: /var/run/dockershim.sock
+{% endif %}
+---
+apiVersion: kubeadm.k8s.io/v1beta1
+kind: ClusterConfiguration
+clusterName: {{ cluster_name }}
+etcd:
+  external:
+      endpoints:
+{% for endpoint in etcd_access_addresses.split(',') %}
+      - {{ endpoint }}
+{% endfor %}
+      caFile: {{ etcd_cert_dir }}/ca.pem
+      certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
+      keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
+networking:
+  dnsDomain: {{ dns_domain }}
+  serviceSubnet: {{ kube_service_addresses }}
+  podSubnet: {{ kube_pods_subnet }}
+  podNetworkCidr: "{{ kube_network_node_prefix }}"
+kubernetesVersion: {{ kube_version }}
+{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %}
+controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% else %}
+controlPlaneEndpoint: {{ ip | default(ansible_default_ipv4.address) }}:{{ kube_apiserver_port }}
+{% endif %}
+certificatesDir: {{ kube_cert_dir }}
+imageRepository: {{ kube_image_repo }}
+UseHyperKubeImage: false
+apiServer:
+  extraArgs:
+    authorization-mode: {{ authorization_modes | join(',') }}
+    bind-address: {{ kube_apiserver_bind_address }}
+{% if kube_apiserver_insecure_port|string != "0" %}
+    insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
+{% endif %}
+    insecure-port: "{{ kube_apiserver_insecure_port }}"
+{% if kube_version is version('v1.10', '<') %}
+    admission-control: {{ kube_apiserver_admission_control | join(',') }}
+{% else %}
+{% if kube_apiserver_enable_admission_plugins|length > 0 %}
+    enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }}
+{% endif %}
+{% if kube_apiserver_disable_admission_plugins|length > 0 %}
+    disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }}
+{% endif %}
+{% endif %}
+    apiserver-count: "{{ kube_apiserver_count }}"
+{% if kube_version is version('v1.9', '>=') %}
+    endpoint-reconciler-type: lease
+{% endif %}
+    storage-backend: etcd3
+{% if etcd_events_cluster_enabled %}
+    etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
+{% endif %}
+    service-node-port-range: {{ kube_apiserver_node_port_range }}
+    kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
+{% if kube_basic_auth|default(true) %}
+    basic-auth-file: {{ kube_users_dir }}/known_users.csv
+{% endif %}
+{% if kube_token_auth|default(true) %}
+    token-auth-file: {{ kube_token_dir }}/known_tokens.csv
+{% endif %}
+{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
+    oidc-issuer-url: {{ kube_oidc_url }}
+    oidc-client-id: {{ kube_oidc_client_id }}
+{%   if kube_oidc_ca_file is defined %}
+    oidc-ca-file: {{ kube_oidc_ca_file }}
+{%   endif %}
+{%   if kube_oidc_username_claim is defined %}
+    oidc-username-claim: {{ kube_oidc_username_claim }}
+{%   endif %}
+{%   if kube_oidc_groups_claim is defined %}
+    oidc-groups-claim: {{ kube_oidc_groups_claim }}
+{%   endif %}
+{% endif %}
+{% if kube_encrypt_secret_data %}
+    encryption-provider-config: {{ kube_config_dir }}/ssl/secrets_encryption.yaml
+{% endif %}
+    storage-backend: {{ kube_apiserver_storage_backend }}
+{% if kube_api_runtime_config is defined %}
+    runtime-config: {{ kube_api_runtime_config | join(',') }}
+{% endif %}
+    allow-privileged: "true"
+{% if kubernetes_audit %}
+    audit-log-path: "{{ audit_log_path }}"
+    audit-log-maxage: "{{ audit_log_maxage }}"
+    audit-log-maxbackup: "{{ audit_log_maxbackups }}"
+    audit-log-maxsize: "{{ audit_log_maxsize }}"
+    audit-policy-file: {{ audit_policy_file }}
+{% endif %}
+{% for key in kube_kubeadm_apiserver_extra_args %}
+    {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
+{% endfor %}
+{% if kube_feature_gates %}
+    feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+    cloud-provider: {{cloud_provider}}
+    cloud-config: {{ kube_config_dir }}/cloud_config
+{% elif cloud_provider is defined and cloud_provider in ["external"] %}
+    cloud-config: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) or apiserver_extra_volumes %}
+  extraVolumes:
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
+  - name: cloud-config
+    hostPath: {{ kube_config_dir }}/cloud_config
+    mountPath: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% if kube_basic_auth|default(true) %}
+  - name: basic-auth-config
+    hostPath: {{ kube_users_dir }}
+    mountPath: {{ kube_users_dir }}
+{% endif %}
+{% if kube_token_auth|default(true) %}
+  - name: token-auth-config
+    hostPath: {{ kube_token_dir }}
+    mountPath: {{ kube_token_dir }}
+{% endif %}
+{% if kubernetes_audit %}
+  - name: {{ audit_policy_name }}
+    hostPath: {{ audit_policy_hostpath }}
+    mountPath: {{ audit_policy_mountpath }}
+{% if audit_log_path != "-" %}
+  - name: {{ audit_log_name }}
+    hostPath: {{ audit_log_hostpath }}
+    mountPath: {{ audit_log_mountpath }}
+    writable: true
+{% endif %}
+{% endif %}
+{% for volume in apiserver_extra_volumes %}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    writable: {{ volume.writable | default(false)}}
+{% endfor %}
+{% endif %}
+  certSANs:
+{% for san in  apiserver_sans.split(' ') | unique %}
+  - {{ san }}
+{% endfor %}
+  timeoutForControlPlane: 5m0s
+controllerManager:
+  extraArgs:
+    node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
+    node-monitor-period: {{ kube_controller_node_monitor_period }}
+    pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+{% if kube_feature_gates %}
+    feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+{% for key in kube_kubeadm_controller_extra_args %}
+    {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
+{% endfor %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+    cloud-provider: {{cloud_provider}}
+    cloud-config: {{ kube_config_dir }}/cloud_config
+{% elif cloud_provider is defined and cloud_provider in ["external"] %}
+    cloud-config: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] or controller_manager_extra_volumes %}
+  extraVolumes:
+{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
+  - name: openstackcacert
+    hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+    mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
+  - name: cloud-config
+    hostPath: {{ kube_config_dir }}/cloud_config
+    mountPath: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% for volume in controller_manager_extra_volumes %}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    writable: {{ volume.writable | default(false)}}
+{% endfor %}
+{% endif %}
+scheduler:
+  extraArgs:
+{% if kube_feature_gates %}
+    feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+{% if kube_kubeadm_scheduler_extra_args|length > 0 %}
+{% for key in kube_kubeadm_scheduler_extra_args %}
+    {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
+{% endfor %}
+{% endif %}
+  extraVolumes:
+{% if scheduler_extra_volumes %}
+  extraVolumes:
+{% for volume in scheduler_extra_volumes %}
+  - name: {{ volume.name }}
+    hostPath: {{ volume.hostPath }}
+    mountPath: {{ volume.mountPath }}
+    writable: {{ volume.writable | default(false)}}
+{% endfor %}
+{% endif %}
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+bindAddress: 0.0.0.0
+clientConnection:
+ acceptContentTypes: ""
+ burst: 10
+ contentType: application/vnd.kubernetes.protobuf
+ kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
+ qps: 5
+clusterCIDR: ""
+configSyncPeriod: 15m0s
+conntrack:
+ max: null
+ maxPerCore: 32768
+ min: 131072
+ tcpCloseWaitTimeout: 1h0m0s
+ tcpEstablishedTimeout: 24h0m0s
+enableProfiling: false
+healthzBindAddress: 0.0.0.0:10256
+iptables:
+ masqueradeAll: false
+ masqueradeBit: 14
+ minSyncPeriod: 0s
+ syncPeriod: 30s
+ipvs:
+ excludeCIDRs: null
+ minSyncPeriod: 0s
+ scheduler: ""
+ syncPeriod: 30s
+metricsBindAddress: 127.0.0.1:10249
+mode: {{ kube_proxy_mode }}
+{% if kube_proxy_nodeport_addresses %}
+nodePortAddresses: [{{ kube_proxy_nodeport_addresses_cidr }}]
+{% endif %}
+oomScoreAdj: -999
+portRange: ""
+resourceContainer: ""
+udpIdleTimeout: 250ms

roles/kubespray-defaults/defaults/main.yaml

@@ -12,7 +12,7 @@ is_atomic: false
 disable_swap: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.12.3
+kube_version: v1.13.0
 
 ## Kube Proxy mode One of ['iptables','ipvs']
 kube_proxy_mode: ipvs

roles/network_plugin/calico/templates/calico-node.yml.j2

@@ -19,7 +19,6 @@ spec:
         k8s-app: calico-node
       annotations:
         # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
 {% if calico_felix_prometheusmetricsenabled %}
         prometheus.io/scrape: 'true'

roles/network_plugin/canal/templates/canal-node.yaml.j2

@@ -12,9 +12,6 @@ spec:
       k8s-app: canal-node
   template:
     metadata:
-      annotations:
-        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
-        scheduler.alpha.kubernetes.io/critical-pod: ''
       labels:
         k8s-app: canal-node
     spec:

roles/network_plugin/cilium/templates/cilium-ds.yml.j2

@@ -21,12 +21,6 @@ spec:
       labels:
         k8s-app: cilium
         kubernetes.io/cluster-service: "true"
-      annotations:
-        # This annotation plus the CriticalAddonsOnly toleration makes
-        # cilium to be a critical pod in the cluster, which ensures cilium
-        # gets priority scheduling.
-        # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
-        scheduler.alpha.kubernetes.io/critical-pod: ''
 {% if cilium_enable_prometheus %}
         prometheus.io/scrape: "true"
         prometheus.io/port: "9090"

roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2

@@ -15,9 +15,6 @@ spec:
       namespace: kube-system
       labels:
         k8s-app: contiv-api-proxy
-      annotations:
-        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}
       priorityClassName: system-node-critical

roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2

@@ -14,9 +14,6 @@ spec:
     metadata:
       labels:
         k8s-app: contiv-cleanup
-      annotations:
-        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}
       priorityClassName: system-node-critical

roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2

@@ -14,8 +14,6 @@ spec:
     metadata:
       labels:
         k8s-app: contiv-etcd-proxy
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}
       priorityClassName: system-node-critical

roles/network_plugin/contiv/templates/contiv-etcd.yml.j2

@@ -14,8 +14,6 @@ spec:
     metadata:
       labels:
         k8s-app: contiv-etcd
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}
       priorityClassName: system-node-critical

roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2

@@ -15,9 +15,6 @@ spec:
       namespace: kube-system
       labels:
         k8s-app: contiv-netmaster
-      annotations:
-        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}
       priorityClassName: system-node-critical

roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2

@@ -19,9 +19,6 @@ spec:
     metadata:
       labels:
         k8s-app: contiv-netplugin
-      annotations:
-        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}
       priorityClassName: system-node-critical

roles/network_plugin/contiv/templates/contiv-ovs.yml.j2

@@ -16,9 +16,6 @@ spec:
     metadata:
       labels:
         k8s-app: contiv-ovs
-      annotations:
-        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}
       priorityClassName: system-node-critical

roles/network_plugin/flannel/templates/cni-flannel.yml.j2

@@ -51,9 +51,6 @@ spec:
       labels:
         tier: node
         k8s-app: flannel
-      annotations:
-        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}
       priorityClassName: system-node-critical

roles/network_plugin/kube-router/templates/kube-router.yml.j2

@@ -60,8 +60,6 @@ spec:
       labels:
         k8s-app: kube-router
         tier: node
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}
       priorityClassName: system-cluster-critical

roles/network_plugin/weave/templates/weave-net.yml.j2

@@ -114,9 +114,6 @@ items:
         metadata:
           labels:
             name: weave-net
-          annotations:
-            # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
-            scheduler.alpha.kubernetes.io/critical-pod: ''
         spec:
 {% if kube_version is version('v1.11.1', '>=') %}
           priorityClassName: system-node-critical