Upgrade kubernetes to V1.11.x (#3078)

Upgrade Kubernetes to V1.11.2
The kubeadm configuration file version has been upgraded from v1alpha1 to v1alpha2
Add bootstrap kubeadm-config.yaml with external etcd

.gitlab-ci.yml

@@ -93,7 +93,7 @@ before_script:
     # Check out latest tag if testing upgrade
     # Uncomment when gitlab kubespray repo has tags
     #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
-    - test "${UPGRADE_TEST}" != "false" && git checkout 02cd5418c22d51e40261775908d55bc562206023
+    - test "${UPGRADE_TEST}" != "false" && git checkout 8b3ce6e418ccf48171eb5b3888ee1af84f8d71ba
     # Checkout the CI vars file so it is available
     - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
     # Workaround https://github.com/kubernetes-incubator/kubespray/issues/2021

README.md

@@ -90,7 +90,7 @@ Supported Components
 --------------------
 
 -   Core
-    -   [kubernetes](https://github.com/kubernetes/kubernetes) v1.10.4
+    -   [kubernetes](https://github.com/kubernetes/kubernetes) v1.11.2
     -   [etcd](https://github.com/coreos/etcd) v3.2.18
     -   [docker](https://www.docker.com/) v17.03 (see note)
     -   [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2)

inventory/sample/group_vars/k8s-cluster.yml

@@ -19,7 +19,7 @@ kube_users_dir: "{{ kube_config_dir }}/users"
 kube_api_anonymous_auth: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.10.4
+kube_version: v1.11.2
 
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)

roles/download/defaults/main.yml

@@ -24,7 +24,7 @@ download_always_pull: False
 download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}"
 
 # Versions
-kube_version: v1.10.4
+kube_version: v1.11.2
 kubeadm_version: "{{ kube_version }}"
 etcd_version: v3.2.18
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
@@ -47,7 +47,7 @@ kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release
 vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip"
 
 # Checksums
-kubeadm_checksum: 7e1169bbbeed973ab402941672dec957638dea5952a1e8bc89a37d5e709cc4b4
+kubeadm_checksum: 6b17720a65b8ff46efe92a5544f149c39a221910d89939838d75581d4e6924c0
 vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188
 
 # Containers

roles/kubernetes/kubeadm/tasks/main.yml

@@ -27,23 +27,36 @@
   register: temp_token
   delegate_to: "{{ groups['kube-master'][0] }}"
 
+- name: gets the kubeadm version
+  command: "{{ bin_dir }}/kubeadm version -o short"
+  register: kubeadm_output
+
+- name: sets kubeadm api version to v1alpha1
+  set_fact:
+    kubeadmConfig_api_version: v1alpha1
+  when: kubeadm_output.stdout|version_compare('v1.11.0', '<')
+
+- name: defaults kubeadm api version to v1alpha2
+  set_fact:
+    kubeadmConfig_api_version: v1alpha2
+  when: kubeadm_output.stdout|version_compare('v1.11.0', '>=')
+
 - name: Create kubeadm client config
   template:
-    src: kubeadm-client.conf.j2
-    dest: "{{ kube_config_dir }}/kubeadm-client.conf"
+    src: "kubeadm-client.conf.{{ kubeadmConfig_api_version }}.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-client.{{ kubeadmConfig_api_version }}.conf"
     backup: yes
   when: not is_kube_master
   vars:
     kubeadm_token: "{{ temp_token.stdout }}"
-  register: kubeadm_client_conf
 
 - name: Join to cluster if needed
   command: >-
     {{ bin_dir }}/kubeadm join
-    --config {{ kube_config_dir}}/kubeadm-client.conf
+    --config {{ kube_config_dir}}/kubeadm-client.{{ kubeadmConfig_api_version }}.conf
     --ignore-preflight-errors=all
   register: kubeadm_join
-  when: not is_kube_master and (kubeadm_client_conf.changed or not kubelet_conf.stat.exists)
+  when: not is_kube_master and (not kubelet_conf.stat.exists)
 
 - name: Wait for kubelet bootstrap to create config
   wait_for:

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2

@@ -0,0 +1,13 @@
+apiVersion: kubeadm.k8s.io/v1alpha2
+kind: NodeConfiguration
+clusterName: {{ cluster_name }}
+discoveryFile: ""
+caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+discoveryToken: {{ kubeadm_token }}
+tlsBootstrapToken: {{ kubeadm_token }}
+token: {{ kubeadm_token }}
+discoveryTokenAPIServers:
+- {{ kubeadm_discovery_address | replace("https://", "")}}
+discoveryTokenUnsafeSkipCAVerification: true
+nodeRegistration:
+  name: {{ inventory_hostname  }}

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -65,14 +65,28 @@
   command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd"
   changed_when: false
 
+- name: gets the kubeadm version
+  command: "{{ bin_dir }}/kubeadm version -o short"
+  register: kubeadm_output
+
+- name: sets kubeadm api version to v1alpha1
+  set_fact:
+    kubeadmConfig_api_version: v1alpha1
+  when: kubeadm_output.stdout|version_compare('v1.11.0', '<')
+
+- name: defaults kubeadm api version to v1alpha2
+  set_fact:
+    kubeadmConfig_api_version: v1alpha2
+  when: kubeadm_output.stdout|version_compare('v1.11.0', '>=')
+
 - name: kubeadm | Create kubeadm config
   template:
-    src: kubeadm-config.yaml.j2
-    dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
+    src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml"
   register: kubeadm_config
 
 - name: kubeadm | Initialize first master
-  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all
+  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all
   register: kubeadm_init
   # Retry is because upload config sometimes fails
   retries: 3
@@ -85,7 +99,7 @@
     timeout -k 240s 240s
     {{ bin_dir }}/kubeadm
     upgrade apply -y {{ kube_version }}
-    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml
     --ignore-preflight-errors=all
     --allow-experimental-upgrades
     --allow-release-candidate-upgrades
@@ -98,7 +112,7 @@
 
 # FIXME(mattymo): remove when https://github.com/kubernetes/kubeadm/issues/433 is fixed
 - name: kubeadm | Enable kube-proxy
-  command: "{{ bin_dir }}/kubeadm alpha phase addon kube-proxy --config={{ kube_config_dir }}/kubeadm-config.yaml"
+  command: "{{ bin_dir }}/kubeadm alpha phase addon kube-proxy --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml"
   when: inventory_hostname == groups['kube-master']|first
   changed_when: false
 
@@ -135,7 +149,7 @@
   when: inventory_hostname != groups['kube-master']|first
 
 - name: kubeadm | Init other uninitialized masters
-  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all
+  command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all
   register: kubeadm_init
   when: inventory_hostname != groups['kube-master']|first and not kubeadm_ca.stat.exists
   failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
@@ -146,7 +160,7 @@
     timeout -k 240s 240s
     {{ bin_dir }}/kubeadm
     upgrade apply -y {{ kube_version }}
-    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml
     --ignore-preflight-errors=all
     --allow-experimental-upgrades
     --allow-release-candidate-upgrades

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -0,0 +1,115 @@
+apiVersion: kubeadm.k8s.io/v1alpha2
+kind: MasterConfiguration
+api:
+  advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+  bindPort: {{ kube_apiserver_port }}
+etcd:
+  external:
+      endpoints:
+{% for endpoint in etcd_access_addresses.split(',') %}
+      - {{ endpoint }}
+{% endfor %}
+      caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
+      certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
+      keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+networking:
+  dnsDomain: {{ dns_domain }}
+  serviceSubnet: {{ kube_service_addresses }}
+  podSubnet: {{ kube_pods_subnet }}
+kubernetesVersion: {{ kube_version }}
+{% if cloud_provider is defined and cloud_provider != "gce" %}
+cloudProvider: {{ cloud_provider }}
+{% endif %}
+kubeProxy:
+  config:
+    mode: {{ kube_proxy_mode }}
+    hostnameOverride: {{ inventory_hostname }}
+authorizationModes:
+{% for mode in authorization_modes %}
+- {{ mode }}
+{% endfor %}
+apiServerExtraArgs:
+  bind-address: {{ kube_apiserver_bind_address }}
+  insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
+  insecure-port: "{{ kube_apiserver_insecure_port }}"
+{% if kube_version | version_compare('v1.10', '<') %}
+  admission-control: {{ kube_apiserver_admission_control | join(',') }}
+{% else %}
+{% if kube_apiserver_enable_admission_plugins|length > 0 %}
+  enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }} 
+{% endif %}
+{% if kube_apiserver_disable_admission_plugins|length > 0 %}
+  disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }} 
+{% endif %}
+{% endif %}
+  apiserver-count: "{{ kube_apiserver_count }}"
+{% if kube_version | version_compare('v1.9', '>=') %}
+  endpoint-reconciler-type: lease
+{% endif %}
+{% if etcd_events_cluster_enabled %}
+  etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
+{% endif %}
+  service-node-port-range: {{ kube_apiserver_node_port_range }}
+  kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
+{% if kube_basic_auth|default(true) %}
+  basic-auth-file: {{ kube_users_dir }}/known_users.csv
+{% endif %}
+{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
+  oidc-issuer-url: {{ kube_oidc_url }}
+  oidc-client-id: {{ kube_oidc_client_id }}
+{%   if kube_oidc_ca_file is defined %}
+  oidc-ca-file: {{ kube_oidc_ca_file }}
+{%   endif %}
+{%   if kube_oidc_username_claim is defined %}
+  oidc-username-claim: {{ kube_oidc_username_claim }}
+{%   endif %}
+{%   if kube_oidc_groups_claim is defined %}
+  oidc-groups-claim: {{ kube_oidc_groups_claim }}
+{%   endif %}
+{% endif %}
+{% if kube_encrypt_secret_data %}
+  experimental-encryption-provider-config: {{ kube_config_dir }}/ssl/secrets_encryption.yaml
+{% endif %}
+  storage-backend: {{ kube_apiserver_storage_backend }}
+{% if kube_api_runtime_config is defined %}
+  runtime-config: {{ kube_api_runtime_config | join(',') }}
+{% endif %}
+  allow-privileged: "true"
+{% for key in kube_kubeadm_apiserver_extra_args %}
+  {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
+{% endfor %}
+controllerManagerExtraArgs:
+  node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
+  node-monitor-period: {{ kube_controller_node_monitor_period }}
+  pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
+controllerManagerExtraVolumes:
+- name: openstackcacert
+  hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+  mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+{% endif %}
+{% if kube_feature_gates %}
+  feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+{% for key in kube_kubeadm_controller_extra_args %}
+  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
+{% endfor %}
+{% if kube_kubeadm_scheduler_extra_args|length > 0 %}
+schedulerExtraArgs:
+{% for key in kube_kubeadm_scheduler_extra_args %}
+  {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
+{% endfor %}
+{% endif %}
+apiServerCertSANs:
+{% for san in  apiserver_sans.split(' ') | unique %}
+  - {{ san }}
+{% endfor %}
+certificatesDir: {{ kube_config_dir }}/ssl
+unifiedControlPlaneImage: "{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}"
+nodeRegistration:
+{% if kube_override_hostname|default('') %}
+  name: {{ kube_override_hostname }}
+{% endif %}
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/master

roles/kubernetes/node/tasks/main.yml

@@ -110,13 +110,13 @@
   modprobe:
     name: "{{ item }}"
     state: present
-  when: kube_proxy_mode == 'ipvs'
   with_items:
     - ip_vs
     - ip_vs_rr
     - ip_vs_wrr
     - ip_vs_sh
     - nf_conntrack_ipv4
+  when: kube_proxy_mode == 'ipvs'
   tags:
     - kube-proxy
 

roles/kubespray-defaults/defaults/main.yaml

@@ -16,7 +16,7 @@ is_atomic: false
 disable_swap: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.10.4
+kube_version: v1.11.2
 
 ## Kube Proxy mode One of ['iptables','ipvs']
 kube_proxy_mode: iptables

tests/files/gce_opensuse-canal.yml

@@ -6,7 +6,6 @@ mode: default
 # Deployment settings
 bootstrap_os: opensuse
 kube_network_plugin: canal
-kubeadm_enabled: true
 deploy_netchecker: true
 kubedns_min_replicas: 1
 cloud_provider: gce