From a11e1eba9e071f3042a39c888c2b7f638451f031 Mon Sep 17 00:00:00 2001 From: Rong Zhang Date: Tue, 14 Aug 2018 20:13:44 +0800 Subject: [PATCH] Upgrade kubernetes to V1.11.x (#3078) Upgrade Kubernetes to V1.11.2 The kubeadm configuration file version has been upgraded from v1alpha1 to v1alpha2 Add bootstrap kubeadm-config.yaml with external etcd --- .gitlab-ci.yml | 2 +- README.md | 2 +- inventory/sample/group_vars/k8s-cluster.yml | 2 +- roles/download/defaults/main.yml | 4 +- roles/kubernetes/kubeadm/tasks/main.yml | 23 +++- ...onf.j2 => kubeadm-client.conf.v1alpha1.j2} | 0 .../templates/kubeadm-client.conf.v1alpha2.j2 | 13 ++ .../kubernetes/master/tasks/kubeadm-setup.yml | 28 +++-- ...aml.j2 => kubeadm-config.v1alpha1.yaml.j2} | 0 .../templates/kubeadm-config.v1alpha2.yaml.j2 | 115 ++++++++++++++++++ roles/kubernetes/node/tasks/main.yml | 2 +- roles/kubespray-defaults/defaults/main.yaml | 2 +- tests/files/gce_opensuse-canal.yml | 1 - 13 files changed, 174 insertions(+), 20 deletions(-) rename roles/kubernetes/kubeadm/templates/{kubeadm-client.conf.j2 => kubeadm-client.conf.v1alpha1.j2} (100%) create mode 100644 roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 rename roles/kubernetes/master/templates/{kubeadm-config.yaml.j2 => kubeadm-config.v1alpha1.yaml.j2} (100%) create mode 100644 roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 42f7ec169..5ba68ab05 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,7 +93,7 @@ before_script: # Check out latest tag if testing upgrade # Uncomment when gitlab kubespray repo has tags #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1)) - - test "${UPGRADE_TEST}" != "false" && git checkout 02cd5418c22d51e40261775908d55bc562206023 + - test "${UPGRADE_TEST}" != "false" && git checkout 8b3ce6e418ccf48171eb5b3888ee1af84f8d71ba # Checkout the CI vars file so it is available - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml # Workaround https://github.com/kubernetes-incubator/kubespray/issues/2021 diff --git a/README.md b/README.md index e1213f4a0..708c99fb3 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Supported Components -------------------- - Core - - [kubernetes](https://github.com/kubernetes/kubernetes) v1.10.4 + - [kubernetes](https://github.com/kubernetes/kubernetes) v1.11.2 - [etcd](https://github.com/coreos/etcd) v3.2.18 - [docker](https://www.docker.com/) v17.03 (see note) - [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2) diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index 2407ceb2b..795d76eb3 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -19,7 +19,7 @@ kube_users_dir: "{{ kube_config_dir }}/users" kube_api_anonymous_auth: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.10.4 +kube_version: v1.11.2 # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 0b0584b39..beb590a63 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -24,7 +24,7 @@ download_always_pull: False download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}" # Versions -kube_version: v1.10.4 +kube_version: v1.11.2 kubeadm_version: "{{ kube_version }}" etcd_version: v3.2.18 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults @@ -47,7 +47,7 @@ kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip" # Checksums -kubeadm_checksum: 7e1169bbbeed973ab402941672dec957638dea5952a1e8bc89a37d5e709cc4b4 +kubeadm_checksum: 6b17720a65b8ff46efe92a5544f149c39a221910d89939838d75581d4e6924c0 vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188 # Containers diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index f6addc5a8..26fa22d8a 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -27,23 +27,36 @@ register: temp_token delegate_to: "{{ groups['kube-master'][0] }}" +- name: gets the kubeadm version + command: "{{ bin_dir }}/kubeadm version -o short" + register: kubeadm_output + +- name: sets kubeadm api version to v1alpha1 + set_fact: + kubeadmConfig_api_version: v1alpha1 + when: kubeadm_output.stdout|version_compare('v1.11.0', '<') + +- name: defaults kubeadm api version to v1alpha2 + set_fact: + kubeadmConfig_api_version: v1alpha2 + when: kubeadm_output.stdout|version_compare('v1.11.0', '>=') + - name: Create kubeadm client config template: - src: kubeadm-client.conf.j2 - dest: "{{ kube_config_dir }}/kubeadm-client.conf" + src: "kubeadm-client.conf.{{ kubeadmConfig_api_version }}.j2" + dest: "{{ kube_config_dir }}/kubeadm-client.{{ kubeadmConfig_api_version }}.conf" backup: yes when: not is_kube_master vars: kubeadm_token: "{{ temp_token.stdout }}" - register: kubeadm_client_conf - name: Join to cluster if needed command: >- {{ bin_dir }}/kubeadm join - --config {{ kube_config_dir}}/kubeadm-client.conf + --config {{ kube_config_dir}}/kubeadm-client.{{ kubeadmConfig_api_version }}.conf --ignore-preflight-errors=all register: kubeadm_join - when: not is_kube_master and (kubeadm_client_conf.changed or not kubelet_conf.stat.exists) + when: not is_kube_master and (not kubelet_conf.stat.exists) - name: Wait for kubelet bootstrap to create config wait_for: diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha1.j2 similarity index 100% rename from roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2 rename to roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha1.j2 diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 new file mode 100644 index 000000000..e2cd04a86 --- /dev/null +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 @@ -0,0 +1,13 @@ +apiVersion: kubeadm.k8s.io/v1alpha2 +kind: NodeConfiguration +clusterName: {{ cluster_name }} +discoveryFile: "" +caCertPath: {{ kube_config_dir }}/ssl/ca.crt +discoveryToken: {{ kubeadm_token }} +tlsBootstrapToken: {{ kubeadm_token }} +token: {{ kubeadm_token }} +discoveryTokenAPIServers: +- {{ kubeadm_discovery_address | replace("https://", "")}} +discoveryTokenUnsafeSkipCAVerification: true +nodeRegistration: + name: {{ inventory_hostname }} diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index b841d8357..80e89acc9 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -65,14 +65,28 @@ command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd" changed_when: false +- name: gets the kubeadm version + command: "{{ bin_dir }}/kubeadm version -o short" + register: kubeadm_output + +- name: sets kubeadm api version to v1alpha1 + set_fact: + kubeadmConfig_api_version: v1alpha1 + when: kubeadm_output.stdout|version_compare('v1.11.0', '<') + +- name: defaults kubeadm api version to v1alpha2 + set_fact: + kubeadmConfig_api_version: v1alpha2 + when: kubeadm_output.stdout|version_compare('v1.11.0', '>=') + - name: kubeadm | Create kubeadm config template: - src: kubeadm-config.yaml.j2 - dest: "{{ kube_config_dir }}/kubeadm-config.yaml" + src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2" + dest: "{{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml" register: kubeadm_config - name: kubeadm | Initialize first master - command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all + command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all register: kubeadm_init # Retry is because upload config sometimes fails retries: 3 @@ -85,7 +99,7 @@ timeout -k 240s 240s {{ bin_dir }}/kubeadm upgrade apply -y {{ kube_version }} - --config={{ kube_config_dir }}/kubeadm-config.yaml + --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all --allow-experimental-upgrades --allow-release-candidate-upgrades @@ -98,7 +112,7 @@ # FIXME(mattymo): remove when https://github.com/kubernetes/kubeadm/issues/433 is fixed - name: kubeadm | Enable kube-proxy - command: "{{ bin_dir }}/kubeadm alpha phase addon kube-proxy --config={{ kube_config_dir }}/kubeadm-config.yaml" + command: "{{ bin_dir }}/kubeadm alpha phase addon kube-proxy --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml" when: inventory_hostname == groups['kube-master']|first changed_when: false @@ -135,7 +149,7 @@ when: inventory_hostname != groups['kube-master']|first - name: kubeadm | Init other uninitialized masters - command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all + command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all register: kubeadm_init when: inventory_hostname != groups['kube-master']|first and not kubeadm_ca.stat.exists failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr @@ -146,7 +160,7 @@ timeout -k 240s 240s {{ bin_dir }}/kubeadm upgrade apply -y {{ kube_version }} - --config={{ kube_config_dir }}/kubeadm-config.yaml + --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all --allow-experimental-upgrades --allow-release-candidate-upgrades diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 similarity index 100% rename from roles/kubernetes/master/templates/kubeadm-config.yaml.j2 rename to roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 new file mode 100644 index 000000000..907ea55f2 --- /dev/null +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -0,0 +1,115 @@ +apiVersion: kubeadm.k8s.io/v1alpha2 +kind: MasterConfiguration +api: + advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }} + bindPort: {{ kube_apiserver_port }} +etcd: + external: + endpoints: +{% for endpoint in etcd_access_addresses.split(',') %} + - {{ endpoint }} +{% endfor %} + caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem + certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem + keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem +networking: + dnsDomain: {{ dns_domain }} + serviceSubnet: {{ kube_service_addresses }} + podSubnet: {{ kube_pods_subnet }} +kubernetesVersion: {{ kube_version }} +{% if cloud_provider is defined and cloud_provider != "gce" %} +cloudProvider: {{ cloud_provider }} +{% endif %} +kubeProxy: + config: + mode: {{ kube_proxy_mode }} + hostnameOverride: {{ inventory_hostname }} +authorizationModes: +{% for mode in authorization_modes %} +- {{ mode }} +{% endfor %} +apiServerExtraArgs: + bind-address: {{ kube_apiserver_bind_address }} + insecure-bind-address: {{ kube_apiserver_insecure_bind_address }} + insecure-port: "{{ kube_apiserver_insecure_port }}" +{% if kube_version | version_compare('v1.10', '<') %} + admission-control: {{ kube_apiserver_admission_control | join(',') }} +{% else %} +{% if kube_apiserver_enable_admission_plugins|length > 0 %} + enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }} +{% endif %} +{% if kube_apiserver_disable_admission_plugins|length > 0 %} + disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }} +{% endif %} +{% endif %} + apiserver-count: "{{ kube_apiserver_count }}" +{% if kube_version | version_compare('v1.9', '>=') %} + endpoint-reconciler-type: lease +{% endif %} +{% if etcd_events_cluster_enabled %} + etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}" +{% endif %} + service-node-port-range: {{ kube_apiserver_node_port_range }} + kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" +{% if kube_basic_auth|default(true) %} + basic-auth-file: {{ kube_users_dir }}/known_users.csv +{% endif %} +{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %} + oidc-issuer-url: {{ kube_oidc_url }} + oidc-client-id: {{ kube_oidc_client_id }} +{% if kube_oidc_ca_file is defined %} + oidc-ca-file: {{ kube_oidc_ca_file }} +{% endif %} +{% if kube_oidc_username_claim is defined %} + oidc-username-claim: {{ kube_oidc_username_claim }} +{% endif %} +{% if kube_oidc_groups_claim is defined %} + oidc-groups-claim: {{ kube_oidc_groups_claim }} +{% endif %} +{% endif %} +{% if kube_encrypt_secret_data %} + experimental-encryption-provider-config: {{ kube_config_dir }}/ssl/secrets_encryption.yaml +{% endif %} + storage-backend: {{ kube_apiserver_storage_backend }} +{% if kube_api_runtime_config is defined %} + runtime-config: {{ kube_api_runtime_config | join(',') }} +{% endif %} + allow-privileged: "true" +{% for key in kube_kubeadm_apiserver_extra_args %} + {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}" +{% endfor %} +controllerManagerExtraArgs: + node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} + node-monitor-period: {{ kube_controller_node_monitor_period }} + pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }} +{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %} +controllerManagerExtraVolumes: +- name: openstackcacert + hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" + mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" +{% endif %} +{% if kube_feature_gates %} + feature-gates: {{ kube_feature_gates|join(',') }} +{% endif %} +{% for key in kube_kubeadm_controller_extra_args %} + {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" +{% endfor %} +{% if kube_kubeadm_scheduler_extra_args|length > 0 %} +schedulerExtraArgs: +{% for key in kube_kubeadm_scheduler_extra_args %} + {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}" +{% endfor %} +{% endif %} +apiServerCertSANs: +{% for san in apiserver_sans.split(' ') | unique %} + - {{ san }} +{% endfor %} +certificatesDir: {{ kube_config_dir }}/ssl +unifiedControlPlaneImage: "{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}" +nodeRegistration: +{% if kube_override_hostname|default('') %} + name: {{ kube_override_hostname }} +{% endif %} + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/master diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index f7520caf8..7f807ceeb 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -110,13 +110,13 @@ modprobe: name: "{{ item }}" state: present - when: kube_proxy_mode == 'ipvs' with_items: - ip_vs - ip_vs_rr - ip_vs_wrr - ip_vs_sh - nf_conntrack_ipv4 + when: kube_proxy_mode == 'ipvs' tags: - kube-proxy diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index bfddbd3a4..72f807ff5 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -16,7 +16,7 @@ is_atomic: false disable_swap: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.10.4 +kube_version: v1.11.2 ## Kube Proxy mode One of ['iptables','ipvs'] kube_proxy_mode: iptables diff --git a/tests/files/gce_opensuse-canal.yml b/tests/files/gce_opensuse-canal.yml index 9eae57e2e..e5bea621c 100644 --- a/tests/files/gce_opensuse-canal.yml +++ b/tests/files/gce_opensuse-canal.yml @@ -6,7 +6,6 @@ mode: default # Deployment settings bootstrap_os: opensuse kube_network_plugin: canal -kubeadm_enabled: true deploy_netchecker: true kubedns_min_replicas: 1 cloud_provider: gce