From af97febb042446f920dad404738cea547a139389 Mon Sep 17 00:00:00 2001 From: Rong Zhang Date: Thu, 4 Oct 2018 17:05:55 +0800 Subject: [PATCH] Upgrade kubernetes to v1.12.0 (#3410) * Upgrade kubernetes to v1.12.0 Use kubeadm v1alpha3 config * Upgrade coredns and etcd * Upgrage docker to 18.06 --- .gitlab-ci.yml | 2 +- README.md | 6 +- .../group_vars/k8s-cluster/k8s-cluster.yml | 2 +- roles/docker/defaults/main.yml | 2 +- roles/docker/vars/debian.yml | 10 +- roles/docker/vars/fedora.yml | 2 +- roles/docker/vars/redhat.yml | 26 ++- roles/docker/vars/ubuntu-amd64.yml | 2 +- roles/docker/vars/ubuntu-arm64.yml | 8 +- roles/download/defaults/main.yml | 10 +- roles/kubernetes/kubeadm/tasks/main.yml | 11 +- .../templates/kubeadm-client.conf.v1alpha3.j2 | 23 +++ .../kubernetes/master/tasks/kubeadm-setup.yml | 11 +- .../templates/kubeadm-config.v1alpha3.yaml.j2 | 185 ++++++++++++++++++ .../node/templates/kubelet.kubeadm.env.j2 | 4 +- .../node/templates/kubelet.standard.env.j2 | 4 +- roles/kubespray-defaults/defaults/main.yaml | 19 +- tests/files/gce_centos-weave-kubeadm.yml | 1 - tests/files/gce_ubuntu-canal-kubeadm.yml | 1 + 19 files changed, 294 insertions(+), 35 deletions(-) create mode 100644 roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 create mode 100644 roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d70a479f8..98bc57c20 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,7 +93,7 @@ before_script: # Check out latest tag if testing upgrade # Uncomment when gitlab kubespray repo has tags #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1)) - - test "${UPGRADE_TEST}" != "false" && git checkout 8b3ce6e418ccf48171eb5b3888ee1af84f8d71ba + - test "${UPGRADE_TEST}" != "false" && git checkout 53d87e53c5899d4ea2904ab7e3883708dd6363d3 # Checkout the CI vars file so it is available - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml # Workaround https://github.com/kubernetes-incubator/kubespray/issues/2021 diff --git a/README.md b/README.md index d4e361325..f3097eb5a 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ Supported Linux Distributions ----------------------------- - **Container Linux by CoreOS** -- **Debian** Jessie, Stretch, Wheezy +- **Debian** Buster, Jessie, Stretch, Wheezy - **Ubuntu** 16.04, 18.04 - **CentOS/RHEL** 7 - **Fedora** 28 @@ -102,9 +102,9 @@ Supported Components -------------------- - Core - - [kubernetes](https://github.com/kubernetes/kubernetes) v1.11.3 + - [kubernetes](https://github.com/kubernetes/kubernetes) v1.12.0 - [etcd](https://github.com/coreos/etcd) v3.2.18 - - [docker](https://www.docker.com/) v17.03 (see note) + - [docker](https://www.docker.com/) v18.06 (see note) - [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2) - [cri-o](http://cri-o.io/) v1.11.5 (experimental: see [CRI-O Note](docs/cri-o.md). Only on centos based OS) - Network Plugin diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index b8dced281..0c79a6f7e 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -19,7 +19,7 @@ kube_users_dir: "{{ kube_config_dir }}/users" kube_api_anonymous_auth: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.11.3 +kube_version: v1.12.0 # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index ae8eefbd3..bd8d293c1 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -1,5 +1,5 @@ --- -docker_version: '17.03' +docker_version: '18.06' docker_selinux_version: '17.03' docker_package_info: diff --git a/roles/docker/vars/debian.yml b/roles/docker/vars/debian.yml index 8138996c1..fe53f1393 100644 --- a/roles/docker/vars/debian.yml +++ b/roles/docker/vars/debian.yml @@ -9,9 +9,13 @@ docker_versioned_pkg: '1.12': docker-engine=1.12.6-0~debian-{{ ansible_distribution_release|lower }} '1.13': docker-engine=1.13.1-0~debian-{{ ansible_distribution_release|lower }} '17.03': docker-ce=17.03.2~ce-0~debian-{{ ansible_distribution_release|lower }} - '17.09': docker-ce=17.09.0~ce-0~debian-{{ ansible_distribution_release|lower }} - 'stable': docker-ce=17.03.2~ce-0~debian-{{ ansible_distribution_release|lower }} - 'edge': docker-ce=17.12.1~ce-0~debian-{{ ansible_distribution_release|lower }} + '17.06': docker-ce=17.06.2~ce-0~debian + '17.09': docker-ce=17.09.0~ce-0~debian + '17.12': docker-ce=17.12.1~ce-0~debian + '18.03': docker-ce=18.03.1~ce-0~debian + '18.06': docker-ce=18.06.1~ce~3-0~debian + 'stable': docker-ce=18.06.1~ce~3-0~debian + 'edge': docker-ce=17.12.1~ce-0~debian docker_package_info: pkg_mgr: apt diff --git a/roles/docker/vars/fedora.yml b/roles/docker/vars/fedora.yml index 2388f1ffc..87bc8578c 100644 --- a/roles/docker/vars/fedora.yml +++ b/roles/docker/vars/fedora.yml @@ -17,4 +17,4 @@ docker_version: "latest" docker_package_info: pkg_mgr: dnf pkgs: - - name: "{{ docker_versioned_pkg[docker_version | string] }}" \ No newline at end of file + - name: "{{ docker_versioned_pkg[docker_version | string] }}" diff --git a/roles/docker/vars/redhat.yml b/roles/docker/vars/redhat.yml index 57970eb50..d42bc8522 100644 --- a/roles/docker/vars/redhat.yml +++ b/roles/docker/vars/redhat.yml @@ -12,7 +12,10 @@ docker_versioned_pkg: '1.13': docker-engine-1.13.1-1.el7.centos '17.03': docker-ce-17.03.2.ce-1.el7.centos '17.09': docker-ce-17.09.0.ce-1.el7.centos - 'stable': docker-ce-17.03.2.ce-1.el7.centos + '17.12': docker-ce-17.12.1.ce-1.el7.centos + '18.03': docker-ce-18.03.1.ce-1.el7.centos + '18.06': docker-ce-18.06.1.ce-3.el7 + 'stable': docker-ce-18.06.1.ce-3.el7 'edge': docker-ce-17.12.1.ce-1.el7.centos docker_selinux_versioned_pkg: @@ -25,13 +28,24 @@ docker_selinux_versioned_pkg: 'edge': docker-ce-selinux-17.03.2.ce-1.el7.centos +docker_pkgs_use_docker_ce: + - name: "{{ docker_selinux_versioned_pkg[docker_selinux_version | string] }}" + yum_conf: "{{ docker_yum_conf }}" + - name: "{{ docker_versioned_pkg[docker_version | string] }}" + yum_conf: "{{ docker_yum_conf }}" + +docker_pkgs: + - name: "{{ docker_versioned_pkg[docker_version | string] }}" + yum_conf: "{{ docker_yum_conf }}" + docker_package_info: pkg_mgr: yum - pkgs: - - name: "{{ docker_selinux_versioned_pkg[docker_selinux_version | string] }}" - yum_conf: "{{ docker_yum_conf }}" - - name: "{{ docker_versioned_pkg[docker_version | string] }}" - yum_conf: "{{ docker_yum_conf }}" + pkgs: |- + {%- if docker_version | version_compare('17.04', '<') -%} + {{ docker_pkgs_use_docker_ce }} + {%- else -%} + {{ docker_pkgs }} + {%- endif %} docker_repo_key_info: pkg_key: '' diff --git a/roles/docker/vars/ubuntu-amd64.yml b/roles/docker/vars/ubuntu-amd64.yml index b88800b4c..6b91267ba 100644 --- a/roles/docker/vars/ubuntu-amd64.yml +++ b/roles/docker/vars/ubuntu-amd64.yml @@ -11,7 +11,7 @@ docker_versioned_pkg: '17.09': docker-ce=17.09.0~ce-0~ubuntu-{{ ansible_distribution_release|lower }} '17.12': docker-ce=17.12.1~ce-0~ubuntu-{{ ansible_distribution_release|lower }} '18.06': docker-ce=18.06.1~ce~3-0~ubuntu - 'stable': docker-ce=17.03.2~ce-0~ubuntu-{{ ansible_distribution_release|lower }} + 'stable': docker-ce=18.06.1~ce~3-0~ubuntu 'edge': docker-ce=18.06.1~ce~3-0~ubuntu docker_package_info: diff --git a/roles/docker/vars/ubuntu-arm64.yml b/roles/docker/vars/ubuntu-arm64.yml index 3dbb1a0e4..1033b5ff5 100644 --- a/roles/docker/vars/ubuntu-arm64.yml +++ b/roles/docker/vars/ubuntu-arm64.yml @@ -1,12 +1,14 @@ --- docker_kernel_min_version: '3.10' -docker_version: 17.09 +docker_version: 18.06 # https://download.docker.com/linux/ubuntu/ docker_versioned_pkg: 'latest': docker-ce '17.09': docker-ce=17.09.1~ce-0~ubuntu - 'stable': docker-ce=17.09.1~ce-0~ubuntu - 'edge': docker-ce=17.12.1~ce-0~ubuntu + '17.12': docker-ce=17.12.1~ce-0~ubuntu-{{ ansible_distribution_release|lower }} + '18.06': docker-ce=18.06.1~ce~3-0~ubuntu + 'stable': docker-ce=18.06.1~ce~3-0~ubuntu + 'edge': docker-ce=18.06.1~ce~3-0~ubuntu docker_package_info: pkg_mgr: apt diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index bddb8cb69..ebfae9971 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -35,9 +35,9 @@ download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube image_arch: "{{host_architecture | default('amd64')}}" # Versions -kube_version: v1.11.3 +kube_version: v1.12.0 kubeadm_version: "{{ kube_version }}" -etcd_version: v3.2.18 +etcd_version: v3.2.24 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download @@ -64,6 +64,7 @@ hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/relea # Checksums hyperkube_checksums: + v1.12.0: f80336201f3152a5307c01f8a7206847398dde15c69b3d20c76a7d9520b60daf v1.11.3: dac8da16dd6688e52b5dc510f5dd0a20b54350d52fb27ceba2f018ba2c8be692 v1.11.2: d727f8cae3fc26b1add9b4ff0d4d9b99605544ff7fb3baeecdca394362adbfb8 v1.11.1: 019ce1ecf4c6a70c06a7f4ef107443351458b4d9e6b9ce4a436bfbfbef93feea @@ -78,6 +79,7 @@ hyperkube_checksums: v1.10.1: 6e0642ad6bae68dc81b8d1c9efa18e265e17e23da1895862823cafac08c0344c v1.10.0: b5575b2fb4266754c1675b8cd5d9b6cac70f3fee7a05c4e80da3a9e83e58c57e kubeadm_checksums: + v1.12.0: 463fb058b7fa2591fb01f29f2451b054f6cbaa0f8a20394b4a4eb5d68473176f v1.11.3: 422a7a32ed9a7b1eaa2a4f9d121674dfbe80eb41e206092c13017d097f75aaec v1.11.2: 6b17720a65b8ff46efe92a5544f149c39a221910d89939838d75581d4e6924c0 v1.11.1: 425ec24b95f7217ee06d1588aba22f206a5829f8c6a5352c2862368552361fe6 @@ -92,7 +94,7 @@ kubeadm_checksums: v1.10.1: 012e48fb92b1c22543b12ab2db7d780777972043287404c98cca4d2c6ec964ec v1.10.0: ebbac985834289037b544523c3e2f39bb44bea938aca9d9e88ef7e880fb8472f -etcd_binary_checksum: b729db0732448064271ea6fdcb901773c4fe917763ca07776f22d0e5e0bd4097 +etcd_binary_checksum: 947849dbcfa13927c81236fb76a7c01d587bbab42ab1e807184cd91b026ebed7 vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188 hyperkube_binary_checksum: "{{ hyperkube_checksums[kube_version] }}" kubeadm_binary_checksum: "{{ kubeadm_checksums[kubeadm_version] }}" @@ -157,7 +159,7 @@ kubedns_version: 1.14.13 kubedns_image_repo: "gcr.io/google_containers/k8s-dns-kube-dns-{{ image_arch }}" kubedns_image_tag: "{{ kubedns_version }}" -coredns_version: "{%- if image_arch != 'amd64' -%}1.1.3{%- else -%}1.2.2{%- endif -%}" +coredns_version: "1.2.2" coredns_image_repo: "gcr.io/google-containers/coredns" coredns_image_tag: "{{ coredns_version }}{%- if image_arch != 'amd64' -%}__{{ image_arch}}_linux{%- endif -%}" diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 6c4dbe5ca..55dbf29a9 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -36,10 +36,17 @@ kubeadmConfig_api_version: v1alpha1 when: kubeadm_output.stdout|version_compare('v1.11.0', '<') -- name: defaults kubeadm api version to v1alpha2 +- name: sets kubeadm api version to v1alpha2 set_fact: kubeadmConfig_api_version: v1alpha2 - when: kubeadm_output.stdout|version_compare('v1.11.0', '>=') + when: + - kubeadm_output.stdout|version_compare('v1.11.0', '>=') + - kubeadm_output.stdout|version_compare('v1.12.0', '<') + +- name: sets kubeadm api version to v1alpha3 + set_fact: + kubeadmConfig_api_version: v1alpha3 + when: kubeadm_output.stdout|version_compare('v1.12.0', '>=') - name: Create kubeadm client config template: diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 new file mode 100644 index 000000000..61707ea85 --- /dev/null +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 @@ -0,0 +1,23 @@ +apiVersion: kubeadm.k8s.io/v1alpha3 +kind: JoinConfiguration +clusterName: {{ cluster_name }} +discoveryFile: "" +caCertPath: {{ kube_config_dir }}/ssl/ca.crt +discoveryTimeout: {{ discovery_timeout }} +discoveryToken: {{ kubeadm_token }} +tlsBootstrapToken: {{ kubeadm_token }} +token: {{ kubeadm_token }} +discoveryTokenAPIServers: +{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %} +- {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} +{% else %} +- {{ kubeadm_discovery_address | replace("https://", "")}} +{% endif %} +discoveryTokenUnsafeSkipCAVerification: true +nodeRegistration: + name: {{ inventory_hostname }} +{% if container_manager == 'crio' %} + criSocket: /var/run/crio/crio.sock +{% else %} + criSocket: /var/run/dockershim.sock +{% endif %} diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index 827154612..39a45d31d 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -86,10 +86,17 @@ kubeadmConfig_api_version: v1alpha1 when: kubeadm_output.stdout|version_compare('v1.11.0', '<') -- name: defaults kubeadm api version to v1alpha2 +- name: sets kubeadm api version to v1alpha2 set_fact: kubeadmConfig_api_version: v1alpha2 - when: kubeadm_output.stdout|version_compare('v1.11.0', '>=') + when: + - kubeadm_output.stdout|version_compare('v1.11.0', '>=') + - kubeadm_output.stdout|version_compare('v1.12.0', '<') + +- name: sets kubeadm api version to v1alpha3 + set_fact: + kubeadmConfig_api_version: v1alpha3 + when: kubeadm_output.stdout|version_compare('v1.12.0', '>=') # Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint. - name: set kubeadm_config_api_fqdn define diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 new file mode 100644 index 000000000..b2f5aee9a --- /dev/null +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 @@ -0,0 +1,185 @@ +apiVersion: kubeadm.k8s.io/v1alpha3 +kind: InitConfiguration +apiEndpoint: + advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }} + bindPort: {{ kube_apiserver_port }} +nodeRegistration: +{% if kube_override_hostname|default('') %} + name: {{ kube_override_hostname }} +{% endif %} + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/master +{% if container_manager == 'crio' %} + criSocket: /var/run/crio/crio.sock +{% else %} + criSocket: /var/run/dockershim.sock +{% endif %} +--- +apiVersion: kubeadm.k8s.io/v1alpha3 +kind: ClusterConfiguration +etcd: + external: + endpoints: +{% for endpoint in etcd_access_addresses.split(',') %} + - {{ endpoint }} +{% endfor %} + caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem + certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem + keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem +networking: + dnsDomain: {{ dns_domain }} + serviceSubnet: {{ kube_service_addresses }} + podSubnet: {{ kube_pods_subnet }} +kubernetesVersion: {{ kube_version }} +{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %} +controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} +{% else %} +controlPlaneEndpoint: {{ ip | default(ansible_default_ipv4.address) }}:{{ kube_apiserver_port }} +{% endif %} +apiServerExtraArgs: + authorization-mode: {{ authorization_modes | join(',') }} + bind-address: {{ kube_apiserver_bind_address }} +{% if kube_apiserver_insecure_port|string != "0" %} + insecure-bind-address: {{ kube_apiserver_insecure_bind_address }} +{% endif %} + insecure-port: "{{ kube_apiserver_insecure_port }}" +{% if kube_version | version_compare('v1.10', '<') %} + admission-control: {{ kube_apiserver_admission_control | join(',') }} +{% else %} +{% if kube_apiserver_enable_admission_plugins|length > 0 %} + enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }} +{% endif %} +{% if kube_apiserver_disable_admission_plugins|length > 0 %} + disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }} +{% endif %} +{% endif %} + apiserver-count: "{{ kube_apiserver_count }}" +{% if kube_version | version_compare('v1.9', '>=') %} + endpoint-reconciler-type: lease +{% endif %} +{% if etcd_events_cluster_enabled %} + etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}" +{% endif %} + service-node-port-range: {{ kube_apiserver_node_port_range }} + kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" +{% if kube_basic_auth|default(true) %} + basic-auth-file: {{ kube_users_dir }}/known_users.csv +{% endif %} +{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %} + oidc-issuer-url: {{ kube_oidc_url }} + oidc-client-id: {{ kube_oidc_client_id }} +{% if kube_oidc_ca_file is defined %} + oidc-ca-file: {{ kube_oidc_ca_file }} +{% endif %} +{% if kube_oidc_username_claim is defined %} + oidc-username-claim: {{ kube_oidc_username_claim }} +{% endif %} +{% if kube_oidc_groups_claim is defined %} + oidc-groups-claim: {{ kube_oidc_groups_claim }} +{% endif %} +{% endif %} +{% if kube_encrypt_secret_data %} + experimental-encryption-provider-config: {{ kube_config_dir }}/ssl/secrets_encryption.yaml +{% endif %} + storage-backend: {{ kube_apiserver_storage_backend }} +{% if kube_api_runtime_config is defined %} + runtime-config: {{ kube_api_runtime_config | join(',') }} +{% endif %} + allow-privileged: "true" +{% if kubernetes_audit %} + audit-log-path: "{{ audit_log_path }}" + audit-log-maxage: "{{ audit_log_maxage }}" + audit-log-maxbackup: "{{ audit_log_maxbackups }}" + audit-log-maxsize: "{{ audit_log_maxsize }}" + audit-policy-file: {{ audit_policy_file }} +{% endif %} +{% for key in kube_kubeadm_apiserver_extra_args %} + {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}" +{% endfor %} +{% if kube_feature_gates %} + feature-gates: {{ kube_feature_gates|join(',') }} +{% endif %} +controllerManagerExtraArgs: + node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} + node-monitor-period: {{ kube_controller_node_monitor_period }} + pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }} +{% if kube_feature_gates %} + feature-gates: {{ kube_feature_gates|join(',') }} +{% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %} +controllerManagerExtraVolumes: +- name: openstackcacert + hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" + mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" +{% endif %} +{% if kubernetes_audit %} +apiServerExtraVolumes: +- name: {{ audit_policy_name }} + hostPath: {{ audit_policy_hostpath }} + mountPath: {{ audit_policy_mountpath }} +{% if audit_log_path != "-" %} +- name: {{ audit_log_name }} + hostPath: {{ audit_log_hostpath }} + mountPath: {{ audit_log_mountpath }} + writable: true +{% endif %} +{% endif %} +{% for key in kube_kubeadm_controller_extra_args %} + {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" +{% endfor %} +schedulerExtraArgs: +{% if kube_feature_gates %} + feature-gates: {{ kube_feature_gates|join(',') }} +{% endif %} +{% if kube_kubeadm_scheduler_extra_args|length > 0 %} +{% for key in kube_kubeadm_scheduler_extra_args %} + {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}" +{% endfor %} +{% endif %} +apiServerCertSANs: +{% for san in apiserver_sans.split(' ') | unique %} + - {{ san }} +{% endfor %} +certificatesDir: {{ kube_config_dir }}/ssl +unifiedControlPlaneImage: "{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}" +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +bindAddress: 0.0.0.0 +clientConnection: + acceptContentTypes: "" + burst: 10 + contentType: application/vnd.kubernetes.protobuf + kubeconfig: /var/lib/kube-proxy/kubeconfig.conf + qps: 5 +clusterCIDR: "" +configSyncPeriod: 15m0s +conntrack: + max: null + maxPerCore: 32768 + min: 131072 + tcpCloseWaitTimeout: 1h0m0s + tcpEstablishedTimeout: 24h0m0s +enableProfiling: false +healthzBindAddress: 0.0.0.0:10256 +hostnameOverride: {{ inventory_hostname }} +iptables: + masqueradeAll: false + masqueradeBit: 14 + minSyncPeriod: 0s + syncPeriod: 30s +ipvs: + excludeCIDRs: null + minSyncPeriod: 0s + scheduler: "" + syncPeriod: 30s +metricsBindAddress: 127.0.0.1:10249 +mode: {{ kube_proxy_mode }} +{% if kube_proxy_nodeport_addresses %} +nodePortAddresses: [{{ kube_proxy_nodeport_addresses_cidr }}] +{% endif %} +oomScoreAdj: -999 +portRange: "" +resourceContainer: /kube-proxy +udpIdleTimeout: 250ms diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index 8dc19d2cf..903544b60 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -29,13 +29,15 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" --enforce-node-allocatable={{ kubelet_enforce_node_allocatable }} \ --client-ca-file={{ kube_cert_dir }}/ca.crt \ --pod-manifest-path={{ kube_manifest_dir }} \ +{% if kube_version | version_compare('v1.12.0', '<') %} --cadvisor-port={{ kube_cadvisor_port }} \ +{% endif %} {# end kubeadm specific settings #} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \ --node-status-update-frequency={{ kubelet_status_update_frequency }} \ --cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \ --max-pods={{ kubelet_max_pods }} \ -{% if container_manager == 'docker' %} +{% if container_manager == 'docker' and kube_version | version_compare('v1.12.0', '<') %} --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ {% endif %} {% if container_manager == 'crio' %} diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2 index d978b4ad8..f649859fe 100644 --- a/roles/kubernetes/node/templates/kubelet.standard.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2 @@ -12,10 +12,12 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {# Base kubelet args #} {% set kubelet_args_base %} --pod-manifest-path={{ kube_manifest_dir }} \ +{% if kube_version | version_compare('v1.12.0', '<') %} --cadvisor-port={{ kube_cadvisor_port }} \ +{% endif %} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \ --node-status-update-frequency={{ kubelet_status_update_frequency }} \ -{% if container_manager == 'docker' %} +{% if container_manager == 'docker' and kube_version | version_compare('v1.12.0', '<') %} --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ {% endif %} --client-ca-file={{ kube_cert_dir }}/ca.pem \ diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 359bad349..ef691f10d 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -16,7 +16,7 @@ is_atomic: false disable_swap: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.11.3 +kube_version: v1.12.0 ## Kube Proxy mode One of ['iptables','ipvs'] kube_proxy_mode: iptables @@ -276,13 +276,24 @@ kubelet_authentication_token_webhook: true # When enabled, access to the kubelet API requires authorization by delegation to the API server kubelet_authorization_mode_webhook: false -## List of key=value pairs that describe feature gates for -## the k8s cluster. -kube_feature_gates: +## v1.11 feature +feature_gate_v1_11: - "PersistentLocalVolumes={{ local_volume_provisioner_enabled | string }}" - "VolumeScheduling={{ local_volume_provisioner_enabled | string }}" - "MountPropagation={{ local_volume_provisioner_enabled | string }}" +## v1.12 feature +feature_gate_v1_12: [] + +## List of key=value pairs that describe feature gates for +## the k8s cluster. +kube_feature_gates: |- + {%- if kube_version | version_compare('v1.12.0', '<') -%} + {{ feature_gate_v1_11 }} + {%- else -%} + {{ feature_gate_v1_12 }} + {%- endif %} + # Vault data dirs. vault_base_dir: /etc/vault vault_cert_dir: "{{ vault_base_dir }}/ssl" diff --git a/tests/files/gce_centos-weave-kubeadm.yml b/tests/files/gce_centos-weave-kubeadm.yml index 24183eb6e..199fa437c 100644 --- a/tests/files/gce_centos-weave-kubeadm.yml +++ b/tests/files/gce_centos-weave-kubeadm.yml @@ -10,6 +10,5 @@ kube_network_plugin: weave kubeadm_enabled: true deploy_netchecker: true kubernetes_audit: true -dynamic_kubelet_configuration: true kubedns_min_replicas: 1 cloud_provider: gce diff --git a/tests/files/gce_ubuntu-canal-kubeadm.yml b/tests/files/gce_ubuntu-canal-kubeadm.yml index 1f8fd2d76..affb852fe 100644 --- a/tests/files/gce_ubuntu-canal-kubeadm.yml +++ b/tests/files/gce_ubuntu-canal-kubeadm.yml @@ -8,6 +8,7 @@ mode: ha bootstrap_os: ubuntu kube_network_plugin: canal kubeadm_enabled: true +dynamic_kubelet_configuration: true deploy_netchecker: true kubedns_min_replicas: 1 cloud_provider: gce