From f14f04c5ea251df8968f4faefaa1b390e8d354e5 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Thu, 5 Oct 2017 10:51:21 +0100 Subject: [PATCH] Upgrade to kubernetes v1.8.0 (#1730) * Upgrade to kubernetes v1.8.0 hyperkube no longer contains rsync, so now use cp * Enable node authorization mode * change kube-proxy cert group name --- .gitlab-ci.yml | 31 ++++++------------- docs/vars.md | 8 +++-- inventory/group_vars/k8s-cluster.yml | 2 +- roles/download/defaults/main.yml | 7 ++--- .../master/templates/kubeadm-config.yaml.j2 | 1 - roles/kubernetes/secrets/files/make-ssl.sh | 4 +-- roles/kubespray-defaults/defaults/main.yaml | 4 +-- roles/network_plugin/calico/tasks/main.yml | 2 +- roles/network_plugin/canal/tasks/main.yml | 2 +- 9 files changed, 25 insertions(+), 36 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 008c36f80..8b02e8607 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,7 +60,6 @@ before_script: KUBELET_DEPLOYMENT: "host" VAULT_DEPLOYMENT: "docker" WEAVE_CPU_LIMIT: "100m" - AUTHORIZATION_MODES: "{ 'authorization_modes': [] }" MAGIC: "ci check this" .gce: &gce @@ -131,7 +130,6 @@ before_script: -e weave_cpu_requests=${WEAVE_CPU_LIMIT} -e weave_cpu_limit=${WEAVE_CPU_LIMIT} -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}" - -e "${AUTHORIZATION_MODES}" --limit "all:!fake_hosts" cluster.yml @@ -161,7 +159,6 @@ before_script: -e weave_cpu_requests=${WEAVE_CPU_LIMIT} -e weave_cpu_limit=${WEAVE_CPU_LIMIT} -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}" - -e "${AUTHORIZATION_MODES}" --limit "all:!fake_hosts" $PLAYBOOK; fi @@ -199,7 +196,6 @@ before_script: -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}" -e weave_cpu_requests=${WEAVE_CPU_LIMIT} -e weave_cpu_limit=${WEAVE_CPU_LIMIT} - -e "${AUTHORIZATION_MODES}" --limit "all:!fake_hosts" cluster.yml; fi @@ -248,7 +244,6 @@ before_script: -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}" -e weave_cpu_requests=${WEAVE_CPU_LIMIT} -e weave_cpu_limit=${WEAVE_CPU_LIMIT} - -e "${AUTHORIZATION_MODES}" --limit "all:!fake_hosts" cluster.yml; fi @@ -278,7 +273,6 @@ before_script: # Test matrix. Leave the comments for markup scripts. .coreos_calico_aio_variables: &coreos_calico_aio_variables # stage: deploy-gce-part1 - AUTHORIZATION_MODES: "{ 'authorization_modes': [ 'RBAC' ] }" KUBE_NETWORK_PLUGIN: calico CLOUD_IMAGE: coreos-stable-1465-6-0-v20170817 CLOUD_REGION: us-west1-b @@ -289,10 +283,9 @@ before_script: ##User-data to simply turn off coreos upgrades STARTUP_SCRIPT: 'systemctl disable locksmithd && systemctl stop locksmithd' -.ubuntu_canal_ha_rbac_variables: &ubuntu_canal_ha_rbac_variables +.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables # stage: deploy-gce-part1 KUBE_NETWORK_PLUGIN: canal - AUTHORIZATION_MODES: "{ 'authorization_modes': [ 'RBAC' ] }" CLOUD_IMAGE: ubuntu-1604-xenial CLOUD_REGION: europe-west1-b CLUSTER_MODE: ha @@ -302,7 +295,6 @@ before_script: .centos_weave_kubeadm_variables: ¢os_weave_kubeadm_variables # stage: deploy-gce-part1 KUBE_NETWORK_PLUGIN: weave - AUTHORIZATION_MODES: "{ 'authorization_modes': [ 'RBAC' ] }" CLOUD_IMAGE: centos-7 CLOUD_MACHINE_TYPE: "n1-standard-1" CLOUD_REGION: us-central1-b @@ -314,7 +306,6 @@ before_script: .ubuntu_canal_kubeadm_variables: &ubuntu_canal_kubeadm_variables # stage: deploy-gce-part1 KUBE_NETWORK_PLUGIN: canal - AUTHORIZATION_MODES: "{ 'authorization_modes': [ 'RBAC' ] }" CLOUD_IMAGE: ubuntu-1604-xenial CLOUD_MACHINE_TYPE: "n1-standard-1" CLOUD_REGION: europe-west1-b @@ -409,7 +400,6 @@ before_script: .ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables # stage: deploy-gce-part1 - AUTHORIZATION_MODES: "{ 'authorization_modes': [ 'RBAC' ] }" CLOUD_MACHINE_TYPE: "n1-standard-2" KUBE_NETWORK_PLUGIN: canal CERT_MGMT: vault @@ -418,9 +408,8 @@ before_script: CLUSTER_MODE: separate STARTUP_SCRIPT: "" -.ubuntu_flannel_rbac_variables: &ubuntu_flannel_rbac_variables +.ubuntu_flannel_variables: &ubuntu_flannel_variables # stage: deploy-gce-special - AUTHORIZATION_MODES: "{ 'authorization_modes': [ 'RBAC' ] }" KUBE_NETWORK_PLUGIN: flannel CLOUD_IMAGE: ubuntu-1604-xenial CLOUD_REGION: europe-west1-b @@ -492,28 +481,28 @@ ubuntu-weave-sep-triggers: only: ['triggers'] # More builds for PRs/merges (manual) and triggers (auto) -ubuntu-canal-ha-rbac: +ubuntu-canal-ha: stage: deploy-gce-part1 <<: *job <<: *gce variables: <<: *gce_variables - <<: *ubuntu_canal_ha_rbac_variables + <<: *ubuntu_canal_ha_variables when: manual except: ['triggers'] only: ['master', /^pr-.*$/] -ubuntu-canal-ha-rbac-triggers: +ubuntu-canal-ha-triggers: stage: deploy-gce-part1 <<: *job <<: *gce variables: <<: *gce_variables - <<: *ubuntu_canal_ha_rbac_variables + <<: *ubuntu_canal_ha_variables when: on_success only: ['triggers'] -ubuntu-canal-kubeadm-rbac: +ubuntu-canal-kubeadm: stage: deploy-gce-part1 <<: *job <<: *gce @@ -534,7 +523,7 @@ ubuntu-canal-kubeadm-triggers: when: on_success only: ['triggers'] -centos-weave-kubeadm-rbac: +centos-weave-kubeadm: stage: deploy-gce-part1 <<: *job <<: *gce @@ -694,13 +683,13 @@ ubuntu-vault-sep: except: ['triggers'] only: ['master', /^pr-.*$/] -ubuntu-flannel-rbac-sep: +ubuntu-flannel-sep: stage: deploy-gce-special <<: *job <<: *gce variables: <<: *gce_variables - <<: *ubuntu_flannel_rbac_variables + <<: *ubuntu_flannel_variables when: manual except: ['triggers'] only: ['master', /^pr-.*$/] diff --git a/docs/vars.md b/docs/vars.md index 87402e381..702f3ac6a 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -71,9 +71,11 @@ following default cluster paramters: alpha/experimental Kubernetes features. (defaults is `[]`) * *authorization_modes* - A list of [authorization mode]( https://kubernetes.io/docs/admin/authorization/#using-flags-for-your-authorization-module) - that the cluster should be configured for. Defaults to `[]` (i.e. no authorization). - Note: `RBAC` is currently in experimental phase, and do not support either calico or - vault. Upgrade from non-RBAC to RBAC is not tested. + that the cluster should be configured for. Defaults to `['RBAC', 'Node']` (RBAC and Node authorizers). + Note: `RBAC` is enabled by default. Previously deployed clusters can be + converted to RBAC mode. However, your apps which rely on Kubernetes API will + require a service account and cluster role bindings. You can override this + setting by setting authorization_modes to `[]`. Note, if cloud providers have any use of the ``10.233.0.0/16``, like instances' private addresses, make sure to pick another values for ``kube_service_addresses`` diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index c185fe46c..dd6142bd3 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -23,7 +23,7 @@ kube_users_dir: "{{ kube_config_dir }}/users" kube_api_anonymous_auth: false ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.7.5 +kube_version: v1.8.0 # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 99c7427b4..ec6d473d9 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -18,9 +18,8 @@ download_localhost: False download_always_pull: False # Versions -kube_version: v1.7.5 -# Change to kube_version after v1.8.0 release -kubeadm_version: "v1.8.0-rc.1" +kube_version: v1.8.0 +kubeadm_version: "{{ kube_version }}" etcd_version: v3.2.4 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download @@ -37,7 +36,7 @@ pod_infra_version: 3.0 kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/amd64/kubeadm" # Checksums -kubeadm_checksum: "8f6ceb26b8503bfc36a99574cf6f853be1c55405aa31669561608ad8099bf5bf" +kubeadm_checksum: "9f4b9cf255d5ef45481d5a1b20bfe84c1d633d67cd50eeaa5c8712fb8fc1bd5b" # Containers etcd_image_repo: "quay.io/coreos/etcd" diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index 78d94d31e..f5571a87d 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -18,7 +18,6 @@ networking: kubernetesVersion: {{ kube_version }} cloudProvider: {{ cloud_provider|default('') }} authorizationModes: -- Node {% for mode in authorization_modes %} - {{ mode }} {% endfor %} diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh index 09342625d..e5277c768 100755 --- a/roles/kubernetes/secrets/files/make-ssl.sh +++ b/roles/kubernetes/secrets/files/make-ssl.sh @@ -109,12 +109,12 @@ if [ -n "$HOSTS" ]; then done fi -# system:kube-proxy +# system:node-proxier if [ -n "$HOSTS" ]; then for host in $HOSTS; do cn="${host%%.*}" # kube-proxy - gen_key_and_cert "kube-proxy-${host}" "/CN=system:kube-proxy" + gen_key_and_cert "kube-proxy-${host}" "/CN=system:kube-proxy/O=system:node-proxier" done fi diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 61f820c62..edcc224ae 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -146,9 +146,9 @@ openstack_lbaas_enabled: false # openstack_lbaas_monitor_max_retries: false ## List of authorization modes that must be configured for -## the k8s cluster. Only 'AlwaysAllow','AlwaysDeny', and +## the k8s cluster. Only 'AlwaysAllow', 'AlwaysDeny', 'Node' and ## 'RBAC' modes are tested. -authorization_modes: [] +authorization_modes: ['RBAC', 'Node'] rbac_enabled: "{{ 'RBAC' in authorization_modes or kubeadm_enabled }}" ## List of key=value pairs that describe feature gates for diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index c4cb60a7a..3889e801c 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -48,7 +48,7 @@ changed_when: false - name: Calico | Copy cni plugins from hyperkube - command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -ac /opt/cni/bin/ /cnibindir/" + command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/" register: cni_task_result until: cni_task_result.rc == 0 retries: 4 diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml index b9d7cdfe9..6d062cc15 100644 --- a/roles/network_plugin/canal/tasks/main.yml +++ b/roles/network_plugin/canal/tasks/main.yml @@ -50,7 +50,7 @@ - rbac_enabled or item.type not in rbac_resources - name: Canal | Copy cni plugins from hyperkube - command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -ac /opt/cni/bin/ /cnibindir/" + command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/" register: cni_task_result until: cni_task_result.rc == 0 retries: 4