From f2b8a3614d57df09b88d5e2cc2e12b55be7d85ea Mon Sep 17 00:00:00 2001 From: okamototk Date: Tue, 2 Jul 2019 16:51:08 +0800 Subject: [PATCH] Use K8s 1.15 (#4905) * Use K8s 1.15 * Use Kubernetes 1.15 and use kubeadm.k8s.io/v1beta2 for InitConfiguration. * bump to v1.15.0 * Remove k8s 1.13 checksums. * Update README kubernetes version 1.15.0. * Update metrics server 0.3.3 for k8s 1.15 * Remove less than k8s 1.14 related code * Use kubeadm with --upload-certs instead of --experimental-upload-certs due to depricate * Update dnsautoscaler 1.6.0 * Skip certificateKey if it's not defined * Add kubeadm-conftolplane.v2beta2 for k8s 1.15 or later * Support kubeadm control plane for k8s 1.15 * Update sonobuoy version 0.15.0 for k8s 1.15 --- README.md | 2 +- .../group_vars/k8s-cluster/k8s-cluster.yml | 2 +- roles/download/defaults/main.yml | 60 +--- .../download/templates/kubeadm-images.yaml.j2 | 22 -- .../cluster_roles/tasks/main.yml | 47 --- roles/kubernetes/kubeadm/tasks/main.yml | 19 -- .../kubernetes/master/defaults/main/main.yml | 8 +- .../tasks/kubeadm-secondary-experimental.yml | 11 +- .../kubernetes/master/tasks/kubeadm-setup.yml | 4 + .../master/tasks/kubeadm-version.yml | 19 +- .../templates/kubeadm-config.v1alpha2.yaml.j2 | 235 --------------- .../templates/kubeadm-config.v1beta1.yaml.j2 | 12 - ...yaml.j2 => kubeadm-config.v1beta2.yaml.j2} | 280 +++++++++--------- .../kubeadm-controlplane.v1beta2.yaml.j2 | 25 ++ roles/kubernetes/node/defaults/main.yml | 3 - roles/kubernetes/node/tasks/kubelet.yml | 11 - .../kubernetes/node/templates/kubelet.env.j2 | 133 --------- .../node/templates/kubelet.env.v1beta1.j2 | 12 - roles/kubespray-defaults/defaults/main.yaml | 8 +- tests/testcases/100_check-k8s-conformance.yml | 2 +- 20 files changed, 199 insertions(+), 716 deletions(-) delete mode 100644 roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 rename roles/kubernetes/master/templates/{kubeadm-config.v1alpha3.yaml.j2 => kubeadm-config.v1beta2.yaml.j2} (52%) create mode 100644 roles/kubernetes/master/templates/kubeadm-controlplane.v1beta2.yaml.j2 delete mode 100644 roles/kubernetes/node/templates/kubelet.env.j2 diff --git a/README.md b/README.md index dc6da09dc..c3714ed9a 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ Supported Components -------------------- - Core - - [kubernetes](https://github.com/kubernetes/kubernetes) v1.14.3 + - [kubernetes](https://github.com/kubernetes/kubernetes) v1.15.0 - [etcd](https://github.com/coreos/etcd) v3.3.10 - [docker](https://www.docker.com/) v18.06 (see note) - [cri-o](http://cri-o.io/) v1.11.5 (experimental: see [CRI-O Note](docs/cri-o.md). Only on centos based OS) diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index c07d943e3..46ab62966 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -20,7 +20,7 @@ kube_users_dir: "{{ kube_config_dir }}/users" kube_api_anonymous_auth: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.14.3 +kube_version: v1.15.0 # kubernetes image repo define kube_image_repo: "gcr.io/google-containers" diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 939a3bde6..6fc922563 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -48,7 +48,7 @@ download_delegate: "{% if download_localhost %}localhost{% else %}{{ groups['kub image_arch: "{{host_architecture | default('amd64')}}" # Versions -kube_version: v1.14.3 +kube_version: v1.15.0 kubeadm_version: "{{ kube_version }}" etcd_version: v3.3.10 @@ -100,84 +100,42 @@ crictl_checksums: # Checksums hyperkube_checksums: arm: + v1.15.0: d923c781031bfd97d0fbe50311e4d7c3616aa5b6d466b99049931f09d73d07b9 v1.14.3: 3fac785261bcf79f7a80b12c4a1dda893ce8c0879caf57b36d4701730671b574 v1.14.2: 6929a59850c8702c04d62cd343d1143b17456da040f32317e09f8c25a08d2346 v1.14.1: 839a4abfeafbd5f5ab057ad0e8a0b0b488b3cde14a646eba040a7f579875f565 v1.14.0: d090b1da23564a7e9bb8f1f4264f2116536c52611ae203fe2ca13eaad0a8003e - v1.13.7: 48e0b381b8a01580dc0627dc52f67617c54c7b78451b46b4cc86d5fa0d5ee1f2 - v1.13.6: ec8dcfeb11e5ff9cb30873a0c2f0c75274eff5c916623ccdbb64533a1f0d3d67 - v1.13.5: 0bc1ecec81f94212a44427a8d9e717a523ea09d45886e641796fb20f41028b2f - v1.13.4: 2530212d807b00c94109b84be42a7baaea97ba91e6bb6c8bca03ab3d5c343c4c - v1.13.3: 4051e88174fedc0ea643466081ca461d9d175f714594dbe5208559fed0c4ae49 - v1.13.2: a981aa0950e86a4380526a3a53f465ce013b95f6d9d8139a9df4a6406b67316f - v1.13.1: 1880ba36aae85474bcea42be0bf37dfa70eb23dd71eb8e956c474e004343f5a4 - v1.13.0: 41c05bf9b0272322fc947760030c21907c21dd8a88576b20cdb110003e818b8f arm64: + v1.15.0: 824af7d925b87a5ade63575b98b59ee81005fc76eac1dc399602308d7a60bc3c v1.14.3: f29211d668cbcf1aa415dfa64aad95ffc53b5410482a23cddb680caec4e907a3 v1.14.2: 959fb7d9c17fc8f7cb1a69920aaf08aefd62c0fbf6b5bdc46250f147ea6a5cd4 v1.14.1: d5236efc2547fd07c7cc2ed9345dfbcd1204385847ca686cf1c62d15056de399 v1.14.0: 708e00a41f6516d525dee00c91ebe3c3bf2feaf9b7f0af7689487e3e17e356c2 - v1.13.7: 5fc44231eb96d3a30c9001c6c64b28efb8db9015dcec1baf5032272cc6b674ca - v1.13.6: 71cef67197517e22ee0cba1ca047905b9578e2cb1f6b7e43cefbf15d14ac3099 - v1.13.5: 8ffd84ba0cb6382a0ff96000458db8a83c92cac09458defe8496f0f0e155a6a8 - v1.13.4: b9e909e388634d103fe5376aafa313bed5e69293383b0c740de4fe8e18d42d12 - v1.13.3: 588037923b7f4090f5f7a3de23ea49a10345295f0b39bd0c1ebdaa24eaa76731 - v1.13.2: 7f2c2b0c6dcc81102a89fa41957db214416fc8a0cfae664fc0e150a7d3ad337b - v1.13.1: 66205d99ec93090c6d814ab1de7c38cd84257d3dcf3a957618fad5878caea13d - v1.13.0: 4391ea0d8d472c1737f1ce945756bf2a11395c708824c780d1a44fbddf031e59 amd64: + v1.15.0: 3cc72cc58517b97c608c7a59a20255675bc70f07217c9e11e58cac7746139283 v1.14.3: 6c6cb5c118b2129ba4e56697f42567be3587eb636a477cd342b69f87b3b049d1 v1.14.2: 05546057f2053e085fa8387ab82581c95fe4195cd783408ccbb4fc3487c50176 v1.14.1: fb34b98da9325feca8daa09bb934dbe6a533aad69c2a5599bbed81b99bb9c267 v1.14.0: af8b04504365dbe4ce6a1772f42eb390d4221a21149b522fc8a0c4b1cd3d97aa - v1.13.7: 972cb9424d7d83660ea96e572520ecb76baecca0dacf61ed8896bcf46a9f63c9 - v1.13.6: 66ea574972d8b7dbe637e2f435f6b881895bc300fb532302587c0da30e47f2ae - v1.13.5: 1a8a357ebfeab8ec62d0c6f11b59df1a93d6711c3a16e1501da32b55c144c73a - v1.13.4: 6f2d755a350efec8b3b29e0ddf8362f60475cc10d42dea37f8f2159f7776867b - v1.13.3: b238c772b5e4b9deed0cdc695fe86324660d037b38c6d6d7eeae7d7a657840c7 - v1.13.2: f159b587ec80ad03bf3b9bb09de5d64b773d01b0e34f2a4f1c816879c56aae6d - v1.13.1: f64c4328d3853f3e5680e7d296b0f3ed25e67ff98321867309edea100ebb4fd7 - v1.13.0: 754f1baae5dc2ba29afc66e1f5d3b676ee59cd5c40ccce813092408d53bde3d9 kubeadm_checksums: arm: + v1.15.0: 9464030a1d4e101de5f47348f3514d5a9eb95cbce2e5e31f53ada1ca485cf75e v1.14.3: 270b8c346aeaa309d11d65695c4a90f6bff5b1ea14bdec3c417ca2dfb3de0db3 v1.14.2: d2a59269aa68a4bace2a80b247b6f9a82f0542ec3004185fb0ba86e181fdfb29 v1.14.1: 4bd111411208f1270ed3af8780b87d24a3c17c9fdbe4b0f8c7a9a21cd765543e v1.14.0: 11f2cfa8bf7ee177dbac8073ab0f039dc265536baaa8dc0c4dea699f981f6fd1 - v1.13.7: 13f6cac67616c2a0d54cb8ae46df67e98d0beab0633ff1f6a9188cb07eaf1b7a - v1.13.6: 0e56c8b804263b0fca1ad2de06e6f0da3471e43f9839702564fa39c415badf74 - v1.13.5: 3eb413c6e7f3fc84ca81de2f725bae8618c65d92a50c6e1e89ce157828ca588c - v1.13.4: 9281b57f0e62330b3905774e38dfad7430d0d54c50cd2a0f87e6c993bb784b17 - v1.13.3: 77afb511c895bc6fb0d2ee3198a0c15d89c0f19bf91fb1fb6274634e3e147d4a - v1.13.2: 5bf5d766050245abde802fdea77a85586ce1477e538bcc4fa618bba854c18980 - v1.13.1: c92bc8672a31158e33489ec9285d0a5546cb5be5bdfdb8cd424fff08439fff9c - v1.13.0: a35e9248fccddb3f2381fd3695c889a576e9ecc63f2b3c9bb0e8daf0308427ef arm64: + v1.15.0: fe3c79070814fe847a23209b1027672fe5c5e7e5c9611e329225058926836f96 v1.14.3: 8edcc07c65f81eea3fc47cd237dd6560c6907c5e0ca52d71eab53ca1164e7d01 v1.14.2: bff0712b87796509129aa802ad3ac25b8cc83af01762b22b4dcca8dbdb26b520 v1.14.1: 5cf05464168e45ee4719264a267c65f9319fae1ceb9923fedab97a9d6a629e0b v1.14.0: 7ed9d706e50cd6d3fc618a7af3d19b691b8a5343ddedaeccb4ea09af3ecfae2c - v1.13.7: 372b33754f4dea201b9db559b2178a833dccd3393cd1a2beefd56ee761a5548d - v1.13.6: 62ae7f2ad28026d4bd006b0307820db08b99e28787ec46641361be281d3f381f - v1.13.5: 59a1995c171e5c1e74f5d02657eb2c155706f2d159ec1847b64dc866228c40d2 - v1.13.4: 4de71d4cfa4dc64127148d48f3a1a1fa7ea24cf0c4fa42957459d0e7f9c03799 - v1.13.3: bef1cbc2d199d32a1a31e70b864dc539b24e3c1cb87b50a1295cf03bec4832b0 - v1.13.2: 08279a3bfeff8c4f6768d6fd92ceff8276a555f9e81bf9d541112fc8eb29963e - v1.13.1: 0f5c2c8a1ffe235785c0a38c9a6530d3d9e67b00e9a07c9d5dca4c36ede2e078 - v1.13.0: efc2669952b05161e181f0805bb0647308891259528a4868e69f4b1b68c70489 amd64: + v1.15.0: fc4aa44b96dc143d7c3062124e25fed671cab884ebb8b2446edd10abb45e88c2 v1.14.3: 026700dfff3c78be1295417e96d882136e5e1f095eb843e6575e57ef9930b5d3 v1.14.2: 77510f61352bb6e537e70730b670627963f2c314fbd36a644b0c435b97e9705a v1.14.1: c4fc478572b5623857f5d820e1c107ae02049ca02cf2993e512a091a0196957b v1.14.0: 03678f49ee4737f8b8c4f59ace0d140a36ffbc4f6035c59561f59f45b57d0c93 - v1.13.7: f591b4d9aade0ed9c54d097caad5c9a936b78fef7180d6436d3595fe6a984a7b - v1.13.6: 347a84461040ea9898ef1e12813abc22c4259b78ac27a87f64908bceca50dbb4 - v1.13.5: 274bf887039a9993e30f96047a4a474c39e8471c4094acb75aea6beed793f079 - v1.13.4: c4300d1f3ebccad48c8e267e45a736c7d227b0e45ef36582fa8dcfe2ef7b1b10 - v1.13.3: ab767ea53e45aceba628977ef6c8c62eace72d6d232efeaf35ac50cbea5f3739 - v1.13.2: 7cb0ce57c1e6e2d85e05de3780a2f35a191fe93f89cfc5816b424efcf39834b9 - v1.13.1: 438173bfa0b7014ecae994c5b9e1f27e1328ab971a3fdb06a393a8095a176ba0 - v1.13.0: f5366206416dc4cfc840a7add2289957b56ccc479cc1b74f7397a4df995d6b06 crictl_binary_checksums: amd64: v1.14.0: 483c90a9fe679590df4332ba807991c49232e8cd326c307c575ecef7fe22327b @@ -282,7 +240,7 @@ nodelocaldns_version: "1.15.1" nodelocaldns_image_repo: "k8s.gcr.io/k8s-dns-node-cache" nodelocaldns_image_tag: "{{ nodelocaldns_version }}" -dnsautoscaler_version: 1.4.0 +dnsautoscaler_version: 1.6.0 dnsautoscaler_image_repo: "k8s.gcr.io/cluster-proportional-autoscaler-{{ image_arch }}" dnsautoscaler_image_tag: "{{ dnsautoscaler_version }}" test_image_repo: docker.io/busybox @@ -299,7 +257,7 @@ registry_image_repo: "docker.io/registry" registry_image_tag: "2.6" registry_proxy_image_repo: "gcr.io/google_containers/kube-registry-proxy" registry_proxy_image_tag: "0.4" -metrics_server_version: "v0.3.2" +metrics_server_version: "v0.3.3" metrics_server_image_repo: "gcr.io/google_containers/metrics-server-amd64" metrics_server_image_tag: "{{ metrics_server_version }}" local_volume_provisioner_image_repo: "quay.io/external_storage/local-volume-provisioner" diff --git a/roles/download/templates/kubeadm-images.yaml.j2 b/roles/download/templates/kubeadm-images.yaml.j2 index eb80e15db..f49a4ca82 100644 --- a/roles/download/templates/kubeadm-images.yaml.j2 +++ b/roles/download/templates/kubeadm-images.yaml.j2 @@ -1,28 +1,10 @@ -{% if kube_version is version('v1.12.0', '>=') %} -{% if kube_version is version('v1.12.0', '>=') and kube_version is version('v1.13.0', '<') %} -apiVersion: kubeadm.k8s.io/v1alpha3 -{% else %} apiVersion: kubeadm.k8s.io/v1beta1 -{% endif %} kind: InitConfiguration nodeRegistration: criSocket: {{ cri_socket }} --- -{% endif %} -{% if kube_version is version('v1.11.0', '<') %} -apiVersion: kubeadm.k8s.io/v1alpha1 -{% elif kube_version is version('v1.11.0', '>=') and kube_version is version('v1.12.0', '<') %} -apiVersion: kubeadm.k8s.io/v1alpha2 -{% elif kube_version is version('v1.12.0', '>=') and kube_version is version('v1.13.0', '<') %} -apiVersion: kubeadm.k8s.io/v1alpha3 -{% else %} apiVersion: kubeadm.k8s.io/v1beta1 -{% endif %} -{% if kube_version is version('v1.12.0', '<') %} -kind: MasterConfiguration -{% else %} kind: ClusterConfiguration -{% endif %} imageRepository: {{ kube_image_repo }} kubernetesVersion: {{ kube_version }} etcd: @@ -31,7 +13,3 @@ etcd: {% for endpoint in etcd_access_addresses.split(',') %} - {{ endpoint }} {% endfor %} -{% if kube_version is version('v1.12.0', '<') %} -nodeRegistration: - criSocket: {{ cri_socket }} -{% endif %} diff --git a/roles/kubernetes-apps/cluster_roles/tasks/main.yml b/roles/kubernetes-apps/cluster_roles/tasks/main.yml index 675417492..3a4b0fcc1 100644 --- a/roles/kubernetes-apps/cluster_roles/tasks/main.yml +++ b/roles/kubernetes-apps/cluster_roles/tasks/main.yml @@ -132,53 +132,6 @@ - inventory_hostname == groups['kube-master'][0] tags: node-webhook -- name: Check if vsphere-cloud-provider ClusterRole exists - command: "{{ bin_dir }}/kubectl get clusterroles system:vsphere-cloud-provider" - register: vsphere_cloud_provider - ignore_errors: true - when: - - rbac_enabled - - cloud_provider is defined - - cloud_provider == 'vsphere' - - kube_version is version('v1.9.0', '>=') - - kube_version is version('v1.9.3', '<=') - - inventory_hostname == groups['kube-master'][0] - tags: vsphere - -- name: Write vsphere-cloud-provider ClusterRole manifest - template: - src: "vsphere-rbac.yml.j2" - dest: "{{ kube_config_dir }}/vsphere-rbac.yml" - register: vsphere_rbac_manifest - when: - - rbac_enabled - - cloud_provider is defined - - cloud_provider == 'vsphere' - - vsphere_cloud_provider.rc is defined - - vsphere_cloud_provider.rc != 0 - - kube_version is version('v1.9.0', '>=') - - kube_version is version('v1.9.3', '<=') - - inventory_hostname == groups['kube-master'][0] - tags: vsphere - -- name: Apply vsphere-cloud-provider ClusterRole - kube: - name: "system:vsphere-cloud-provider" - kubectl: "{{ bin_dir }}/kubectl" - resource: "clusterrolebinding" - filename: "{{ kube_config_dir }}/vsphere-rbac.yml" - state: latest - when: - - rbac_enabled - - cloud_provider is defined - - cloud_provider == 'vsphere' - - vsphere_cloud_provider.rc is defined - - vsphere_cloud_provider.rc != 0 - - kube_version is version('v1.9.0', '>=') - - kube_version is version('v1.9.3', '<=') - - inventory_hostname == groups['kube-master'][0] - tags: vsphere - - include_tasks: oci.yml tags: oci when: diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 171bf8710..110de5f55 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -38,25 +38,6 @@ command: "{{ bin_dir }}/kubeadm version -o short" register: kubeadm_output -- name: sets kubeadm api version to v1alpha1 - set_fact: - kubeadmConfig_api_version: v1alpha1 - when: kubeadm_output.stdout is version('v1.11.0', '<') - -- name: sets kubeadm api version to v1alpha2 - set_fact: - kubeadmConfig_api_version: v1alpha2 - when: - - kubeadm_output.stdout is version('v1.11.0', '>=') - - kubeadm_output.stdout is version('v1.12.0', '<') - -- name: sets kubeadm api version to v1alpha3 - set_fact: - kubeadmConfig_api_version: v1alpha3 - when: - - kubeadm_output.stdout is version('v1.12.0', '>=') - - kubeadm_output.stdout is version('v1.13.0', '<') - - name: sets kubeadm api version to v1beta1 set_fact: kubeadmConfig_api_version: v1beta1 diff --git a/roles/kubernetes/master/defaults/main/main.yml b/roles/kubernetes/master/defaults/main/main.yml index 195bbf82a..28b4a0980 100644 --- a/roles/kubernetes/master/defaults/main/main.yml +++ b/roles/kubernetes/master/defaults/main/main.yml @@ -96,12 +96,8 @@ kube_apiserver_admission_control: - ServiceAccount - DefaultStorageClass - PersistentVolumeClaimResize - - >- - {%- if kube_version is version('v1.9', '<') -%} - GenericAdmissionWebhook - {%- else -%} - MutatingAdmissionWebhook,ValidatingAdmissionWebhook - {%- endif -%} + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook - ResourceQuota # 1.10+ admission plugins diff --git a/roles/kubernetes/master/tasks/kubeadm-secondary-experimental.yml b/roles/kubernetes/master/tasks/kubeadm-secondary-experimental.yml index fd52389b3..00df73cd3 100644 --- a/roles/kubernetes/master/tasks/kubeadm-secondary-experimental.yml +++ b/roles/kubernetes/master/tasks/kubeadm-secondary-experimental.yml @@ -30,8 +30,13 @@ command: >- {{ bin_dir }}/kubeadm init phase --config {{ kube_config_dir }}/kubeadm-config.yaml - upload-certs --experimental-upload-certs - {% if kubeadm_certificate_key is defined %} + upload-certs + {% if kubeadm_version is version('v1.15.0', '<') %} + --experimental-upload-certs + {% else %} + --upload-certs + {% endif %} + {% if kubeadm_certificate_key is defined and kubeadm_version is version('v1.15.0', '<') %} --certificate-key={{ kubeadm_certificate_key }} {% endif %} register: kubeadm_upload_cert @@ -52,7 +57,7 @@ {{ bin_dir }}/kubeadm join --config {{ kube_config_dir }}/kubeadm-controlplane.yaml --ignore-preflight-errors=all - {% if kubeadm_certificate_key is defined %} + {% if kubeadm_certificate_key is defined and kubeadm_version is version('v1.15.0', '<') %} --certificate-key={{ kubeadm_certificate_key }} {% endif %} register: kubeadm_join_control_plane diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index b9a2cc956..8a00efa49 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -109,10 +109,14 @@ --ignore-preflight-errors=all --skip-phases=addon/coredns {% if kubeadm_version is version('v1.14.0', '>=') %} + {% if kubeadm_version is version('v1.15.0', '<') %} --experimental-upload-certs {% if kubeadm_certificate_key is defined %} --certificate-key={{ kubeadm_certificate_key }} {% endif %} + {% else %} + --upload-certs + {% endif %} {% endif %} register: kubeadm_init # Retry is because upload config sometimes fails diff --git a/roles/kubernetes/master/tasks/kubeadm-version.yml b/roles/kubernetes/master/tasks/kubeadm-version.yml index f61657f38..4947a85d8 100644 --- a/roles/kubernetes/master/tasks/kubeadm-version.yml +++ b/roles/kubernetes/master/tasks/kubeadm-version.yml @@ -3,25 +3,16 @@ command: "{{ bin_dir }}/kubeadm version -o short" register: kubeadm_output -- name: sets kubeadm api version to v1alpha2 - set_fact: - kubeadmConfig_api_version: v1alpha2 - when: - - kubeadm_output.stdout is version('v1.11.0', '>=') - - kubeadm_output.stdout is version('v1.12.0', '<') - -- name: sets kubeadm api version to v1alpha3 - set_fact: - kubeadmConfig_api_version: v1alpha3 - when: - - kubeadm_output.stdout is version('v1.12.0', '>=') - - kubeadm_output.stdout is version('v1.13.0', '<') - - name: sets kubeadm api version to v1beta1 set_fact: kubeadmConfig_api_version: v1beta1 when: kubeadm_output.stdout is version('v1.13.0', '>=') +- name: sets kubeadm api version to v1beta2 + set_fact: + kubeadmConfig_api_version: v1beta2 + when: kubeadm_output.stdout is version('v1.15.0', '>=') + - name: kubeadm | Create kubeadm config template: src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2" diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 deleted file mode 100644 index c46f75c20..000000000 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ /dev/null @@ -1,235 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1alpha2 -kind: MasterConfiguration -api: -{% if kubeadm_config_api_fqdn is defined %} - controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }} - bindPort: {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} -{% else %} - advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }} - bindPort: {{ kube_apiserver_port }} -{% endif %} -etcd: - external: - endpoints: -{% for endpoint in etcd_access_addresses.split(',') %} - - {{ endpoint }} -{% endfor %} - caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }} - certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }} - keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }} -networking: - dnsDomain: {{ dns_domain }} - serviceSubnet: {{ kube_service_addresses }} - podSubnet: {{ kube_pods_subnet }} -kubernetesVersion: {{ kube_version }} -kubeProxy: - config: - mode: {{ kube_proxy_mode }} -{% if kube_proxy_nodeport_addresses %} - nodePortAddresses: {{ kube_proxy_nodeport_addresses }} -{% endif %} - resourceContainer: "" -authorizationModes: -{% for mode in authorization_modes %} -- {{ mode }} -{% endfor %} -apiServerExtraArgs: - bind-address: {{ kube_apiserver_bind_address }} -{% if kube_apiserver_insecure_port|string != "0" %} - insecure-bind-address: {{ kube_apiserver_insecure_bind_address }} -{% endif %} - insecure-port: "{{ kube_apiserver_insecure_port }}" -{% if kube_version is version('v1.10', '<') %} - admission-control: {{ kube_apiserver_admission_control | join(',') }} -{% else %} -{% if kube_apiserver_enable_admission_plugins|length > 0 %} - enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }} -{% endif %} -{% if kube_apiserver_disable_admission_plugins|length > 0 %} - disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }} -{% endif %} -{% endif %} - apiserver-count: "{{ kube_apiserver_count }}" -{% if kube_version is version('v1.9', '>=') %} - endpoint-reconciler-type: lease -{% endif %} -{% if etcd_events_cluster_enabled %} - etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}" -{% endif %} - service-node-port-range: {{ kube_apiserver_node_port_range }} - kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" - profiling: "{{ kube_profiling }}" - request-timeout: "{{ kube_apiserver_request_timeout }}" - repair-malformed-updates: "false" - enable-aggregator-routing: "{{ kube_api_aggregator_routing }}" -{% if kube_api_anonymous_auth is defined and kube_version is version('v1.5', '>=') %} - anonymous-auth: "{{ kube_api_anonymous_auth }}" -{% endif %} -{% if kube_basic_auth|default(true) %} - basic-auth-file: {{ kube_users_dir }}/known_users.csv -{% endif %} -{% if kube_token_auth|default(true) %} - token-auth-file: {{ kube_token_dir }}/known_tokens.csv -{% endif %} -{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %} - oidc-issuer-url: {{ kube_oidc_url }} - oidc-client-id: {{ kube_oidc_client_id }} -{% if kube_oidc_ca_file is defined %} - oidc-ca-file: {{ kube_oidc_ca_file }} -{% endif %} -{% if kube_oidc_username_claim is defined %} - oidc-username-claim: {{ kube_oidc_username_claim }} -{% endif %} -{% if kube_oidc_groups_claim is defined %} - oidc-groups-claim: {{ kube_oidc_groups_claim }} -{% endif %} -{% if kube_oidc_username_prefix is defined %} - oidc-username-prefix: "{{ kube_oidc_username_prefix }}" -{% endif %} -{% if kube_oidc_groups_prefix is defined %} - oidc-groups-prefix: "{{ kube_oidc_groups_prefix }}" -{% endif %} -{% endif %} -{% if kube_webhook_token_auth|default(false) %} - authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml -{% endif %} -{% if kube_encrypt_secret_data %} - experimental-encryption-provider-config: {{ kube_cert_dir }}/secrets_encryption.yaml -{% endif %} - storage-backend: {{ kube_apiserver_storage_backend }} -{% if kube_api_runtime_config is defined %} - runtime-config: {{ kube_api_runtime_config | join(',') }} -{% endif %} - allow-privileged: "true" -{% if kubernetes_audit %} - audit-log-path: "{{ audit_log_path }}" - audit-log-maxage: "{{ audit_log_maxage }}" - audit-log-maxbackup: "{{ audit_log_maxbackups }}" - audit-log-maxsize: "{{ audit_log_maxsize }}" - audit-policy-file: {{ audit_policy_file }} -{% endif %} -{% for key in kube_kubeadm_apiserver_extra_args %} - {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}" -{% endfor %} -{% if kube_feature_gates %} - feature-gates: {{ kube_feature_gates|join(',') }} -{% endif %} -{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} - cloud-provider: {{cloud_provider}} - cloud-config: {{ kube_config_dir }}/cloud_config -{% elif cloud_provider is defined and cloud_provider in ["external"] %} - cloud-config: {{ kube_config_dir }}/cloud_config -{% endif %} -{% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %} - configure-cloud-routes: "true" -{% endif %} -controllerManagerExtraArgs: - node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} - node-monitor-period: {{ kube_controller_node_monitor_period }} - pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }} - node-cidr-mask-size: "{{ kube_network_node_prefix }}" - profiling: "{{ kube_profiling }}" - terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}" -{% if kube_feature_gates %} - feature-gates: {{ kube_feature_gates|join(',') }} -{% endif %} -{% for key in kube_kubeadm_controller_extra_args %} - {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" -{% endfor %} -{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} - cloud-provider: {{cloud_provider}} - cloud-config: {{ kube_config_dir }}/cloud_config -{% elif cloud_provider is defined and cloud_provider in ["external"] %} - cloud-config: {{ kube_config_dir }}/cloud_config -{% endif %} -{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} -controllerManagerExtraVolumes: -{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %} -- name: openstackcacert - hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" - mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" -{% endif %} -{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} -- name: cloud-config - hostPath: {{ kube_config_dir }}/cloud_config - mountPath: {{ kube_config_dir }}/cloud_config -{% endif %} -{% endif %} -{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or kube_webhook_token_auth|default(false) or ssl_ca_dirs|length %} -apiServerExtraVolumes: -{% if kube_basic_auth|default(true) %} -- name: basic-auth-config - hostPath: {{ kube_users_dir }} - mountPath: {{ kube_users_dir }} -{% endif %} -{% if kube_token_auth|default(true) %} -- name: token-auth-config - hostPath: {{ kube_token_dir }} - mountPath: {{ kube_token_dir }} -{% endif %} -{% if kube_webhook_token_auth|default(false) %} -- name: webhook-token-auth-config - hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml - mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml -{% endif %} -{% if kubernetes_audit %} -- name: {{ audit_policy_name }} - hostPath: {{ audit_policy_hostpath }} - mountPath: {{ audit_policy_mountpath }} -{% if audit_log_path != "-" %} -- name: {{ audit_log_name }} - hostPath: {{ audit_log_hostpath }} - mountPath: {{ audit_log_mountpath }} - writable: true -{% endif %} -{% endif %} -{% if ssl_ca_dirs|length %} -{% for dir in ssl_ca_dirs %} -- name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} - hostPath: {{ dir }} - mountPath: {{ dir }} - writable: false -{% endfor %} -{% endif %} -{% endif %} -{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} -- name: cloud-config - hostPath: {{ kube_config_dir }}/cloud_config - mountPath: {{ kube_config_dir }}/cloud_config -{% endif %} -schedulerExtraArgs: - profiling: "{{ kube_profiling }}" -{% if kube_feature_gates %} - feature-gates: {{ kube_feature_gates|join(',') }} -{% endif %} -{% if volume_cross_zone_attachment %} - policy-config-file: {{ kube_config_dir }}/kube-scheduler-policy.yaml -{% endif %} -{% if kube_kubeadm_scheduler_extra_args|length > 0 %} -{% for key in kube_kubeadm_scheduler_extra_args %} - {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}" -{% endfor %} -{% endif %} -apiServerCertSANs: -{% for san in apiserver_sans %} - - {{ san }} -{% endfor %} -certificatesDir: {{ kube_cert_dir }} -imageRepository: {{ kube_image_repo }} -unifiedControlPlaneImage: "" -nodeRegistration: -{% if kube_override_hostname|default('') %} - name: {{ kube_override_hostname }} -{% endif %} -{% if inventory_hostname not in groups['kube-node'] %} - taints: - - effect: NoSchedule - key: node-role.kubernetes.io/master -{% else %} - taints: {} -{% endif %} - criSocket: {{ cri_socket }} -{% if dynamic_kubelet_configuration %} -featureGates: - DynamicKubeletConfig: true -{% endif %} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 index b5ee0dd0d..4f5a4bcba 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 @@ -93,15 +93,11 @@ apiServer: insecure-bind-address: {{ kube_apiserver_insecure_bind_address }} {% endif %} insecure-port: "{{ kube_apiserver_insecure_port }}" -{% if kube_version is version('v1.10', '<') %} - admission-control: {{ kube_apiserver_admission_control | join(',') }} -{% else %} {% if kube_apiserver_enable_admission_plugins|length > 0 %} enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }} {% endif %} {% if kube_apiserver_disable_admission_plugins|length > 0 %} disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }} -{% endif %} {% endif %} apiserver-count: "{{ kube_apiserver_count }}" {% if kube_version is version('v1.9', '>=') %} @@ -231,11 +227,7 @@ controllerManager: node-cidr-mask-size: "{{ kube_network_node_prefix }}" profiling: "{{ kube_profiling }}" terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}" -{% if kube_version is version('v1.14', '<') %} - address: {{ kube_controller_manager_bind_address }} -{% else %} bind-address: {{ kube_controller_manager_bind_address }} -{% endif %} {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} @@ -272,11 +264,7 @@ controllerManager: {% endif %} scheduler: extraArgs: -{% if kube_version is version('v1.14', '<') %} - address: {{ kube_scheduler_bind_address }} -{% else %} bind-address: {{ kube_scheduler_bind_address }} -{% endif %} {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2 similarity index 52% rename from roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 rename to roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2 index b83ffea0d..b0a44270b 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2 @@ -1,22 +1,29 @@ -apiVersion: kubeadm.k8s.io/v1alpha3 +apiVersion: kubeadm.k8s.io/v1beta2 kind: InitConfiguration -apiEndpoint: +localAPIEndpoint: advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }} bindPort: {{ kube_apiserver_port }} +{% if kubeadm_certificate_key is defined %} +certificateKey: {{ kubeadm_certificate_key }} +{% endif %} nodeRegistration: {% if kube_override_hostname|default('') %} name: {{ kube_override_hostname }} {% endif %} -{% if inventory_hostname not in groups['kube-node'] %} +{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %} taints: - effect: NoSchedule key: node-role.kubernetes.io/master {% else %} - taints: {} + taints: [] +{% endif %} +{% if container_manager == 'crio' %} + criSocket: /var/run/crio/crio.sock +{% else %} + criSocket: /var/run/dockershim.sock {% endif %} - criSocket: {{ cri_socket }} --- -apiVersion: kubeadm.k8s.io/v1alpha3 +apiVersion: kubeadm.k8s.io/v1beta2 kind: ClusterConfiguration clusterName: {{ cluster_name }} etcd: @@ -38,212 +45,208 @@ controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.po {% else %} controlPlaneEndpoint: {{ ip | default(fallback_ips[inventory_hostname]) }}:{{ kube_apiserver_port }} {% endif %} -apiServerCertSANs: -{% for san in apiserver_sans %} - - {{ san }} -{% endfor %} certificatesDir: {{ kube_cert_dir }} imageRepository: {{ kube_image_repo }} -unifiedControlPlaneImage: "" -apiServerExtraArgs: +useHyperKubeImage: false +apiServer: + extraArgs: {% if kube_api_anonymous_auth is defined and kube_version is version('v1.5', '>=') %} - anonymous-auth: "{{ kube_api_anonymous_auth }}" + anonymous-auth: "{{ kube_api_anonymous_auth }}" {% endif %} - authorization-mode: {{ authorization_modes | join(',') }} - bind-address: {{ kube_apiserver_bind_address }} + authorization-mode: {{ authorization_modes | join(',') }} + bind-address: {{ kube_apiserver_bind_address }} {% if kube_apiserver_insecure_port|string != "0" %} - insecure-bind-address: {{ kube_apiserver_insecure_bind_address }} + insecure-bind-address: {{ kube_apiserver_insecure_bind_address }} {% endif %} - insecure-port: "{{ kube_apiserver_insecure_port }}" -{% if kube_version is version('v1.10', '<') %} - admission-control: {{ kube_apiserver_admission_control | join(',') }} -{% else %} + insecure-port: "{{ kube_apiserver_insecure_port }}" {% if kube_apiserver_enable_admission_plugins|length > 0 %} - enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }} + enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }} {% endif %} {% if kube_apiserver_disable_admission_plugins|length > 0 %} - disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }} + disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }} {% endif %} -{% endif %} - apiserver-count: "{{ kube_apiserver_count }}" + apiserver-count: "{{ kube_apiserver_count }}" {% if kube_version is version('v1.9', '>=') %} - endpoint-reconciler-type: lease + endpoint-reconciler-type: lease {% endif %} {% if etcd_events_cluster_enabled %} - etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}" + etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}" {% endif %} - service-node-port-range: {{ kube_apiserver_node_port_range }} - kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" - profiling: "{{ kube_profiling }}" - request-timeout: "{{ kube_apiserver_request_timeout }}" + service-node-port-range: {{ kube_apiserver_node_port_range }} + kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" + profiling: "{{ kube_profiling }}" + request-timeout: "{{ kube_apiserver_request_timeout }}" + enable-aggregator-routing: "{{ kube_api_aggregator_routing }}" {% if kube_basic_auth|default(true) %} - basic-auth-file: {{ kube_users_dir }}/known_users.csv + basic-auth-file: {{ kube_users_dir }}/known_users.csv {% endif %} {% if kube_token_auth|default(true) %} - token-auth-file: {{ kube_token_dir }}/known_tokens.csv + token-auth-file: {{ kube_token_dir }}/known_tokens.csv {% endif %} {% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %} - oidc-issuer-url: {{ kube_oidc_url }} - oidc-client-id: {{ kube_oidc_client_id }} + oidc-issuer-url: {{ kube_oidc_url }} + oidc-client-id: {{ kube_oidc_client_id }} {% if kube_oidc_ca_file is defined %} - oidc-ca-file: {{ kube_oidc_ca_file }} + oidc-ca-file: {{ kube_oidc_ca_file }} {% endif %} {% if kube_oidc_username_claim is defined %} - oidc-username-claim: {{ kube_oidc_username_claim }} + oidc-username-claim: {{ kube_oidc_username_claim }} {% endif %} {% if kube_oidc_groups_claim is defined %} - oidc-groups-claim: {{ kube_oidc_groups_claim }} + oidc-groups-claim: {{ kube_oidc_groups_claim }} {% endif %} {% if kube_oidc_username_prefix is defined %} - oidc-username-prefix: "{{ kube_oidc_username_prefix }}" + oidc-username-prefix: "{{ kube_oidc_username_prefix }}" {% endif %} {% if kube_oidc_groups_prefix is defined %} - oidc-groups-prefix: "{{ kube_oidc_groups_prefix }}" + oidc-groups-prefix: "{{ kube_oidc_groups_prefix }}" {% endif %} {% endif %} {% if kube_webhook_token_auth|default(false) %} - authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml + authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml {% endif %} {% if kube_encrypt_secret_data %} - experimental-encryption-provider-config: {{ kube_cert_dir }}/secrets_encryption.yaml + encryption-provider-config: {{ kube_cert_dir }}/secrets_encryption.yaml {% endif %} - storage-backend: {{ kube_apiserver_storage_backend }} + storage-backend: {{ kube_apiserver_storage_backend }} {% if kube_api_runtime_config is defined %} - runtime-config: {{ kube_api_runtime_config | join(',') }} + runtime-config: {{ kube_api_runtime_config | join(',') }} {% endif %} - allow-privileged: "true" + allow-privileged: "true" {% if kubernetes_audit %} - audit-log-path: "{{ audit_log_path }}" - audit-log-maxage: "{{ audit_log_maxage }}" - audit-log-maxbackup: "{{ audit_log_maxbackups }}" - audit-log-maxsize: "{{ audit_log_maxsize }}" - audit-policy-file: {{ audit_policy_file }} + audit-log-path: "{{ audit_log_path }}" + audit-log-maxage: "{{ audit_log_maxage }}" + audit-log-maxbackup: "{{ audit_log_maxbackups }}" + audit-log-maxsize: "{{ audit_log_maxsize }}" + audit-policy-file: {{ audit_policy_file }} {% endif %} {% for key in kube_kubeadm_apiserver_extra_args %} - {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}" + {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}" {% endfor %} {% if kube_feature_gates %} - feature-gates: {{ kube_feature_gates|join(',') }} + feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} - cloud-provider: {{cloud_provider}} - cloud-config: {{ kube_config_dir }}/cloud_config + cloud-provider: {{cloud_provider}} + cloud-config: {{ kube_config_dir }}/cloud_config {% elif cloud_provider is defined and cloud_provider in ["external"] %} - cloud-config: {{ kube_config_dir }}/cloud_config -{% endif %} -controllerManagerExtraArgs: - node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} - node-monitor-period: {{ kube_controller_node_monitor_period }} - pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }} - node-cidr-mask-size: "{{ kube_network_node_prefix }}" - profiling: "{{ kube_profiling }}" - terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}" - enable-aggregator-routing: "{{ kube_api_aggregator_routing }}" -{% if kube_version is version('v1.14', '<') %} - address: {{ kube_controller_manager_bind_address }} -{% else %} - bind-address: {{ kube_controller_manager_bind_address }} -{% endif %} -{% if kube_feature_gates %} - feature-gates: {{ kube_feature_gates|join(',') }} -{% endif %} -{% for key in kube_kubeadm_controller_extra_args %} - {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" -{% endfor %} -{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} - cloud-provider: {{cloud_provider}} - cloud-config: {{ kube_config_dir }}/cloud_config -{% elif cloud_provider is defined and cloud_provider in ["external"] %} - cloud-config: {{ kube_config_dir }}/cloud_config -{% endif %} -schedulerExtraArgs: -{% if kube_version is version('v1.14', '<') %} - address: {{ kube_scheduler_bind_address }} -{% else %} - bind-address: {{ kube_scheduler_bind_address }} -{% endif %} -{% if kube_feature_gates %} - feature-gates: {{ kube_feature_gates|join(',') }} -{% endif %} -{% if kube_kubeadm_scheduler_extra_args|length > 0 %} -{% for key in kube_kubeadm_scheduler_extra_args %} - {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}" -{% endfor %} + cloud-config: {{ kube_config_dir }}/cloud_config {% endif %} {% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or kube_webhook_token_auth|default(false) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) or apiserver_extra_volumes or ssl_ca_dirs|length %} -apiServerExtraVolumes: + extraVolumes: {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} -- name: cloud-config - hostPath: {{ kube_config_dir }}/cloud_config - mountPath: {{ kube_config_dir }}/cloud_config + - name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config {% endif %} {% if kube_basic_auth|default(true) %} -- name: basic-auth-config - hostPath: {{ kube_users_dir }} - mountPath: {{ kube_users_dir }} + - name: basic-auth-config + hostPath: {{ kube_users_dir }} + mountPath: {{ kube_users_dir }} {% endif %} {% if kube_token_auth|default(true) %} -- name: token-auth-config - hostPath: {{ kube_token_dir }} - mountPath: {{ kube_token_dir }} + - name: token-auth-config + hostPath: {{ kube_token_dir }} + mountPath: {{ kube_token_dir }} {% endif %} {% if kube_webhook_token_auth|default(false) %} -- name: webhook-token-auth-config - hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml - mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml + - name: webhook-token-auth-config + hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml + mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml {% endif %} {% if kubernetes_audit %} -- name: {{ audit_policy_name }} - hostPath: {{ audit_policy_hostpath }} - mountPath: {{ audit_policy_mountpath }} + - name: {{ audit_policy_name }} + hostPath: {{ audit_policy_hostpath }} + mountPath: {{ audit_policy_mountpath }} {% if audit_log_path != "-" %} -- name: {{ audit_log_name }} - hostPath: {{ audit_log_hostpath }} - mountPath: {{ audit_log_mountpath }} - writable: true + - name: {{ audit_log_name }} + hostPath: {{ audit_log_hostpath }} + mountPath: {{ audit_log_mountpath }} + readOnly: false {% endif %} {% endif %} {% for volume in apiserver_extra_volumes %} -- name: {{ volume.name }} - hostPath: {{ volume.hostPath }} - mountPath: {{ volume.mountPath }} - writable: {{ volume.writable | default(false)}} + - name: {{ volume.name }} + hostPath: {{ volume.hostPath }} + mountPath: {{ volume.mountPath }} + readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }} {% endfor %} {% if ssl_ca_dirs|length %} {% for dir in ssl_ca_dirs %} -- name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} - hostPath: {{ dir }} - mountPath: {{ dir }} - writable: false + - name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} + hostPath: {{ dir }} + mountPath: {{ dir }} + readOnly: true {% endfor %} {% endif %} +{% endif %} + certSANs: +{% for san in apiserver_sans %} + - {{ san }} +{% endfor %} + timeoutForControlPlane: 5m0s +controllerManager: + extraArgs: + node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} + node-monitor-period: {{ kube_controller_node_monitor_period }} + pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }} + node-cidr-mask-size: "{{ kube_network_node_prefix }}" + profiling: "{{ kube_profiling }}" + terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}" + bind-address: {{ kube_controller_manager_bind_address }} +{% if kube_feature_gates %} + feature-gates: {{ kube_feature_gates|join(',') }} +{% endif %} +{% for key in kube_kubeadm_controller_extra_args %} + {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" +{% endfor %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} + cloud-provider: {{cloud_provider}} + cloud-config: {{ kube_config_dir }}/cloud_config +{% elif cloud_provider is defined and cloud_provider in ["external"] %} + cloud-config: {{ kube_config_dir }}/cloud_config +{% endif %} +{% if kube_network_plugin is defined and kube_network_plugin not in ["cloud"] %} + configure-cloud-routes: "false" {% endif %} {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] or controller_manager_extra_volumes %} -controllerManagerExtraVolumes: + extraVolumes: {% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %} -- name: openstackcacert - hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" - mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" + - name: openstackcacert + hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" + mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" {% endif %} {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} -- name: cloud-config - hostPath: {{ kube_config_dir }}/cloud_config - mountPath: {{ kube_config_dir }}/cloud_config + - name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config {% endif %} {% for volume in controller_manager_extra_volumes %} -- name: {{ volume.name }} - hostPath: {{ volume.hostPath }} - mountPath: {{ volume.mountPath }} - writable: {{ volume.writable | default(false)}} + - name: {{ volume.name }} + hostPath: {{ volume.hostPath }} + mountPath: {{ volume.mountPath }} + readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }} {% endfor %} {% endif %} +scheduler: + extraArgs: + bind-address: {{ kube_scheduler_bind_address }} +{% if kube_feature_gates %} + feature-gates: {{ kube_feature_gates|join(',') }} +{% endif %} +{% if kube_kubeadm_scheduler_extra_args|length > 0 %} +{% for key in kube_kubeadm_scheduler_extra_args %} + {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}" +{% endfor %} +{% endif %} + extraVolumes: {% if scheduler_extra_volumes %} -schedulerExtraVolumes: + extraVolumes: {% for volume in scheduler_extra_volumes %} -- name: {{ volume.name }} - hostPath: {{ volume.hostPath }} - mountPath: {{ volume.mountPath }} - writable: {{ volume.writable | default(false)}} + - name: {{ volume.name }} + hostPath: {{ volume.hostPath }} + mountPath: {{ volume.mountPath }} + readOnly: {{ volume.readOnly | d(not (volume.writable | d(false))) }} {% endfor %} {% endif %} --- @@ -259,7 +262,6 @@ clientConnection: clusterCIDR: {{ kube_pods_subnet }} configSyncPeriod: {{ kube_proxy_config_sync_period }} conntrack: - max: {{ kube_proxy_conntrack_max }} maxPerCore: {{ kube_proxy_conntrack_max_per_core }} min: {{ kube_proxy_conntrack_min }} tcpCloseWaitTimeout: {{ kube_proxy_conntrack_tcp_close_wait_timeout }} @@ -273,7 +275,7 @@ iptables: minSyncPeriod: {{ kube_proxy_min_sync_period }} syncPeriod: {{ kube_proxy_sync_period }} ipvs: - excludeCIDRs: {{ "[]" if kube_proxy_exclude_cidrs is not defined or kube_proxy_exclude_cidrs == "null" or kube_proxy_exclude_cidrs | length == 0 else (kube_proxy_exclude_cidrs if kube_proxy_exclude_cidrs[0] == '[' else ("[" + kube_proxy_exclude_cidrs + "]" if (kube_proxy_exclude_cidrs[0] | length) == 1 else "[" + kube_proxy_exclude_cidrs | join(",") + "]")) }} + excludeCIDRs: {{ kube_proxy_exclude_cidrs }} minSyncPeriod: {{ kube_proxy_min_sync_period }} scheduler: {{ kube_proxy_scheduler }} syncPeriod: {{ kube_proxy_sync_period }} diff --git a/roles/kubernetes/master/templates/kubeadm-controlplane.v1beta2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-controlplane.v1beta2.yaml.j2 new file mode 100644 index 000000000..3c9cf9280 --- /dev/null +++ b/roles/kubernetes/master/templates/kubeadm-controlplane.v1beta2.yaml.j2 @@ -0,0 +1,25 @@ +apiVersion: kubeadm.k8s.io/v1beta2 +kind: JoinConfiguration +discovery: + bootstrapToken: +{% if kubeadm_config_api_fqdn is defined %} + apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} +{% else %} + apiServerEndpoint: {{ kubeadm_discovery_address | replace("https://", "")}} +{% endif %} + token: {{ kubeadm_token }} + unsafeSkipCAVerification: true + timeout: {{ discovery_timeout }} + tlsBootstrapToken: {{ kubeadm_token }} +controlPlane: + localAPIEndpoint: + advertiseAddress: {{ kube_apiserver_address }} + bindPort: {{ kube_apiserver_port }} + certificateKey: {{ kubeadm_certificate_key }} +nodeRegistration: + name: {{ kube_override_hostname|default(inventory_hostname) }} +{% if container_manager == 'crio' %} + criSocket: /var/run/crio/crio.sock +{% else %} + criSocket: /var/run/dockershim.sock +{% endif %} diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index a262131ad..64502af65 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -75,9 +75,6 @@ kube_override_hostname: >- {{ inventory_hostname }} {%- endif -%} -# cAdvisor port -kube_cadvisor_port: 0 - # The read-only port for the Kubelet to serve on with no authentication/authorization. kube_read_only_port: 0 diff --git a/roles/kubernetes/node/tasks/kubelet.yml b/roles/kubernetes/node/tasks/kubelet.yml index 6963c965d..fe785a6d6 100644 --- a/roles/kubernetes/node/tasks/kubelet.yml +++ b/roles/kubernetes/node/tasks/kubelet.yml @@ -29,17 +29,6 @@ - kubelet - kubeadm -- name: Write kubelet environment config file (kubeadm) - template: - src: "kubelet.env.j2" - dest: "{{ kube_config_dir }}/kubelet.env" - backup: yes - notify: restart kubelet - when: kubeadm_output.stdout is version('v1.13.0', '<') - tags: - - kubelet - - kubeadm - - name: Write kubelet config file template: src: "kubelet-config.{{ kubeadmConfig_api_version }}.yaml.j2" diff --git a/roles/kubernetes/node/templates/kubelet.env.j2 b/roles/kubernetes/node/templates/kubelet.env.j2 deleted file mode 100644 index eb335e45a..000000000 --- a/roles/kubernetes/node/templates/kubelet.env.j2 +++ /dev/null @@ -1,133 +0,0 @@ -### Upstream source https://github.com/kubernetes/release/blob/master/debian/xenial/kubeadm/channel/stable/etc/systemd/system/kubelet.service.d/ -### All upstream values should be present in this file - -# logging to stderr means we get it in the systemd journal -KUBE_LOGTOSTDERR="--logtostderr=true" -KUBE_LOG_LEVEL="--v={{ kube_log_level }}" -# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces) -KUBELET_ADDRESS="--address={{ kubelet_bind_address }} --node-ip={{ kubelet_address }}" -# The port for the info server to serve on -# KUBELET_PORT="--port=10250" -{% if kube_override_hostname|default('') %} -# You may leave this blank to use the actual hostname -KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" -{% endif %} -{# Base kubelet args #} -{% set kubelet_args_base -%} -{# start kubeadm specific settings #} ---bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ ---kubeconfig={{ kube_config_dir }}/kubelet.conf \ -{% if kube_version is version('v1.8', '<') %} ---require-kubeconfig \ -{% endif %} -{% if kubelet_authentication_token_webhook %} ---authentication-token-webhook \ -{% endif %} -{% if kubelet_authorization_mode_webhook %} ---authorization-mode=Webhook \ -{% endif %} ---enforce-node-allocatable={{ kubelet_enforce_node_allocatable }} \ ---client-ca-file={{ kube_cert_dir }}/ca.crt \ -{% if kubelet_rotate_certificates %} ---rotate-certificates \ -{% endif %} ---pod-manifest-path={{ kube_manifest_dir }} \ -{% if kube_version is version('v1.12.0', '<') %} ---cadvisor-port={{ kube_cadvisor_port }} \ -{% endif %} -{# end kubeadm specific settings #} ---pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \ ---node-status-update-frequency={{ kubelet_status_update_frequency }} \ ---cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \ ---max-pods={{ kubelet_max_pods }} \ -{% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %} ---docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ -{% endif %} -{% if container_manager != 'docker' %} ---container-runtime=remote \ ---container-runtime-endpoint={{ cri_socket }} \ -{% endif %} ---anonymous-auth=false \ ---read-only-port={{ kube_read_only_port }} \ -{% if kube_version is version('v1.8', '<') %} ---experimental-fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \ -{% else %} ---fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \ -{% endif %} -{% if dynamic_kubelet_configuration %} ---dynamic-config-dir={{ dynamic_kubelet_configuration_dir }} \ -{% endif %} ---runtime-cgroups={{ kubelet_runtime_cgroups }} --kubelet-cgroups={{ kubelet_kubelet_cgroups }} \ -{% endset %} - -{# Node reserved CPU/memory #} -{% if is_kube_master|bool %} -{% set kube_reserved %}--kube-reserved cpu={{ kube_master_cpu_reserved }},memory={{ kube_master_memory_reserved|regex_replace('Mi', 'M') }}{% endset %} -{% else %} -{% set kube_reserved %}--kube-reserved cpu={{ kube_cpu_reserved }},memory={{ kube_memory_reserved|regex_replace('Mi', 'M') }}{% endset %} -{% endif %} - -{# DNS settings for kubelet #} -{% if dns_mode == 'coredns' %} -{% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }}{% endset %} -{% elif dns_mode == 'coredns_dual' %} -{% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }},{{ skydns_server_secondary }}{% endset %} -{% elif dns_mode == 'manual' %} -{% set kubelet_args_cluster_dns %}--cluster-dns={{ manual_dns_server }}{% endset %} -{% else %} -{% set kubelet_args_cluster_dns %}{% endset %} -{% endif %} -{% if enable_nodelocaldns %} -{% set kubelet_args_cluster_dns %}--cluster-dns={{ nodelocaldns_ip }}{% endset %} -{% endif %} -{% set kubelet_args_dns %}{{ kubelet_args_cluster_dns }} --cluster-domain={{ dns_domain }} --resolv-conf={{ kube_resolv_conf }}{% endset %} - -{# Kubelet node labels #} -{% set role_node_labels = [] %} -{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %} -{% if inventory_hostname in nvidia_gpu_nodes %} -{% set dummy = role_node_labels.append('nvidia.com/gpu=true') %} -{% endif %} -{% endif %} - -{% set inventory_node_labels = [] %} -{% if node_labels is defined %} -{% if node_labels is mapping %} -{% for labelname, labelvalue in node_labels.items() %} -{% set dummy = inventory_node_labels.append('%s=%s'|format(labelname, labelvalue)) %} -{% endfor %} -{% else %} -{% for label in node_labels.split(",") %} -{% set dummy = inventory_node_labels.append(label) %} -{% endfor %} -{% endif %} -{% set all_node_labels = role_node_labels + inventory_node_labels %} - -{# Kubelet node taints for gpu #} -{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %} -{% if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %} -{% set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %} -{% elif inventory_hostname in nvidia_gpu_nodes and node_taints is not defined %} -{% set node_taints = [] %} -{% set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %} -{% endif %} -{% endif %} - -KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" -{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router", "macvlan"] %} -KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" -{% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %} -KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet" -{% endif %} -KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}" -# Should this cluster be allowed to run privileged docker containers -KUBE_ALLOW_PRIV="--allow-privileged=true" -{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} -KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config" -{% elif cloud_provider is defined and cloud_provider in ["external"] %} -KUBELET_CLOUDPROVIDER="--cloud-provider=external --cloud-config={{ kube_config_dir }}/cloud_config" -{% else %} -KUBELET_CLOUDPROVIDER="" -{% endif %} - -PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin diff --git a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 index b186065a7..543197fad 100644 --- a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 +++ b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 @@ -11,24 +11,12 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --config={{ kube_config_dir }}/kubelet-config.yaml \ --kubeconfig={{ kube_config_dir }}/kubelet.conf \ -{% if kube_version is version('v1.8', '<') %} ---require-kubeconfig \ -{% endif %} -{% if kube_version is version('v1.12.0', '<') %} ---cadvisor-port={{ kube_cadvisor_port }} \ -{% endif %} {# end kubeadm specific settings #} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \ -{% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %} ---docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ -{% endif %} {% if container_manager != 'docker' %} --container-runtime=remote \ --container-runtime-endpoint={{ cri_socket }} \ {% endif %} -{% if kube_version is version('v1.8', '<') %} ---experimental-fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \ -{% endif %} {% if dynamic_kubelet_configuration %} --dynamic-config-dir={{ dynamic_kubelet_configuration_dir }} \ {% endif %} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 6865703bb..b5da646d5 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -12,10 +12,10 @@ is_atomic: false disable_swap: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.14.3 +kube_version: v1.15.0 ## The minimum version working -kube_version_min_required: v1.13.0 +kube_version_min_required: v1.14.0 ## Kube Proxy mode One of ['iptables','ipvs'] kube_proxy_mode: ipvs @@ -344,11 +344,7 @@ feature_gate_v1_12: [] ## List of key=value pairs that describe feature gates for ## the k8s cluster. kube_feature_gates: |- - {%- if kube_version is version('v1.12.0', '<') -%} - {{ feature_gate_v1_11 }} - {%- else -%} {{ feature_gate_v1_12 }} - {%- endif %} # Enable kubeadm experimental control plane kubeadm_control_plane: false diff --git a/tests/testcases/100_check-k8s-conformance.yml b/tests/testcases/100_check-k8s-conformance.yml index 18c2a8d2b..72bcfb2f0 100644 --- a/tests/testcases/100_check-k8s-conformance.yml +++ b/tests/testcases/100_check-k8s-conformance.yml @@ -1,7 +1,7 @@ --- - hosts: kube-master[0] vars: - sonobuoy_version: 0.14.1 + sonobuoy_version: 0.15.0 sonobuoy_arch: amd64 sonobuoy_parallel: 30 sonobuoy_path: /usr/local/bin/sonobuoy