From 9ecbf75cb485b860268fec79c4956ea99dbb33df Mon Sep 17 00:00:00 2001 From: Etienne Champetier Date: Wed, 3 Mar 2021 17:16:23 -0500 Subject: [PATCH] Remove rotate_tokens logic kubeadm never rotates sa.key/sa.pub, so there is no need to delete tokens/restart pods Signed-off-by: Etienne Champetier (cherry picked from commit 8800b5c01d7dbdc99f81464ba5f2f96799e7eed1) --- cluster.yml | 1 - docs/upgrades.md | 14 ------ .../rotate_tokens/tasks/main.yml | 48 ------------------- roles/kubernetes/master/handlers/main.yml | 18 ------- .../master/tasks/kubeadm-secondary.yml | 4 -- .../kubernetes/master/tasks/kubeadm-setup.yml | 23 --------- upgrade-cluster.yml | 1 - 7 files changed, 109 deletions(-) delete mode 100644 roles/kubernetes-apps/rotate_tokens/tasks/main.yml diff --git a/cluster.yml b/cluster.yml index 009a5f61f..a05e20a8d 100644 --- a/cluster.yml +++ b/cluster.yml @@ -100,7 +100,6 @@ environment: "{{ proxy_disable_env }}" roles: - { role: kubespray-defaults } - - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" } - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] } - hosts: kube-master diff --git a/docs/upgrades.md b/docs/upgrades.md index b42dd69d0..4cac24c32 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -284,20 +284,6 @@ follows: * kube-apiserver, kube-scheduler, and kube-controller-manager * Add-ons (such as KubeDNS) -## Upgrade considerations - -Kubespray supports rotating certificates used for etcd and Kubernetes -components, but some manual steps may be required. If you have a pod that -requires use of a service token and is deployed in a namespace other than -`kube-system`, you will need to manually delete the affected pods after -rotating certificates. This is because all service account tokens are dependent -on the apiserver token that is used to generate them. When the certificate -rotates, all service account tokens must be rotated as well. During the -kubernetes-apps/rotate_tokens role, only pods in kube-system are destroyed and -recreated. All other invalidated service account tokens are cleaned up -automatically, but other pods are not deleted out of an abundance of caution -for impact to user deployed pods. - ### Component-based upgrades A deployer may want to upgrade specific components in order to minimize risk diff --git a/roles/kubernetes-apps/rotate_tokens/tasks/main.yml b/roles/kubernetes-apps/rotate_tokens/tasks/main.yml deleted file mode 100644 index e9de24b52..000000000 --- a/roles/kubernetes-apps/rotate_tokens/tasks/main.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -- name: Rotate Tokens | Get default token name # noqa 306 - shell: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf get secrets -o custom-columns=name:{.metadata.name} --no-headers | grep -m1 default-token" - register: default_token - changed_when: false - until: default_token.rc == 0 - delay: 4 - retries: 10 - -- name: Rotate Tokens | Get default token data - command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf get secrets {{ default_token.stdout }} -ojson" - register: default_token_data - changed_when: false - -- name: Rotate Tokens | Test if default certificate is expired - uri: - url: https://{{ kube_apiserver_ip }}/api/v1/nodes - method: GET - return_content: no - validate_certs: no - headers: - Authorization: "Bearer {{ (default_token_data.stdout|from_json)['data']['token']|b64decode }}" - register: check_secret - failed_when: false - -- name: Rotate Tokens | Determine if certificate is expired - set_fact: - needs_rotation: '{{ check_secret.status not in [200, 403] }}' - -# FIXME(mattymo): Exclude built in secrets that were automatically rotated, -# instead of filtering manually -- name: Rotate Tokens | Get all serviceaccount tokens to expire # noqa 306 - shell: >- - {{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf get secrets --all-namespaces - -o 'jsonpath={range .items[*]}{"\n"}{.metadata.namespace}{" "}{.metadata.name}{" "}{.type}{end}' - | grep kubernetes.io/service-account-token - | egrep 'default-token|kube-proxy|coredns|netchecker|weave|calico|canal|flannel|dashboard|cluster-proportional-autoscaler|tiller|local-volume-provisioner' - register: tokens_to_delete - when: needs_rotation - -- name: Rotate Tokens | Delete expired tokens - command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf delete secrets -n {{ item.split(' ')[0] }} {{ item.split(' ')[1] }}" - with_items: "{{ tokens_to_delete.stdout_lines }}" - when: needs_rotation - -- name: Rotate Tokens | Delete pods in system namespace - command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf delete pods -n kube-system --all --grace-period=0 --force" - when: needs_rotation diff --git a/roles/kubernetes/master/handlers/main.yml b/roles/kubernetes/master/handlers/main.yml index 577b433b5..e6bc321e2 100644 --- a/roles/kubernetes/master/handlers/main.yml +++ b/roles/kubernetes/master/handlers/main.yml @@ -121,21 +121,3 @@ until: result.status == 200 retries: 60 delay: 1 - -- name: Master | set secret_changed - command: /bin/true - notify: - - Master | set secret_changed to true - - Master | Copy new kubeconfig for root user - -- name: Master | set secret_changed to true - set_fact: - secret_changed: true - -- name: Master | Copy new kubeconfig for root user - copy: - src: "{{ kube_config_dir }}/admin.conf" - dest: "{{ ansible_env.HOME | default('/root') }}/.kube/config" - remote_src: yes - mode: "0600" - backup: yes diff --git a/roles/kubernetes/master/tasks/kubeadm-secondary.yml b/roles/kubernetes/master/tasks/kubeadm-secondary.yml index b81c42212..6f961f2bc 100644 --- a/roles/kubernetes/master/tasks/kubeadm-secondary.yml +++ b/roles/kubernetes/master/tasks/kubeadm-secondary.yml @@ -66,7 +66,3 @@ when: - inventory_hostname != groups['kube-master']|first - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists - -- name: Set secret_changed to false to avoid extra token rotation - set_fact: - secret_changed: false diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index 0f12b6d4b..1b78e0526 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -10,15 +10,6 @@ - kube_oidc_auth - kube_oidc_ca_cert is defined -- name: kubeadm | Check serviceaccount key - stat: - path: "{{ kube_cert_dir }}/sa.key" - get_attributes: no - get_checksum: yes - get_mime: no - register: sa_key_before - run_once: true - - name: kubeadm | Check if kubeadm has already run stat: path: "/var/lib/kubelet/config.yaml" @@ -180,20 +171,6 @@ - upgrade_cluster_setup - kubeadm_already_run.stat.exists -- name: kubeadm | Check serviceaccount key again - stat: - path: "{{ kube_cert_dir }}/sa.key" - get_attributes: no - get_checksum: yes - get_mime: no - register: sa_key_after - run_once: true - -- name: kubeadm | Set secret_changed if service account key was updated - command: /bin/true - notify: Master | set secret_changed - when: sa_key_before.stat.checksum|default("") != sa_key_after.stat.checksum - # FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file. - name: kubeadm | Remove taint for master with node role command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-" diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml index 79cd37117..c64a27622 100644 --- a/upgrade-cluster.yml +++ b/upgrade-cluster.yml @@ -134,7 +134,6 @@ environment: "{{ proxy_disable_env }}" roles: - { role: kubespray-defaults } - - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" } - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"] } - hosts: calico-rr