From 5a60d8e9b62bef75ef03da92086cb7dd65ae42c7 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Sat, 23 Sep 2017 12:58:04 +0100 Subject: [PATCH] delete more secrets --- .gitlab-ci.yml | 24 +++++++++---------- docs/upgrades.md | 14 +++++++++++ .../network_plugin/weave/tasks/main.yml | 11 --------- .../rotate_tokens/tasks/main.yml | 24 +++++++------------ 4 files changed, 34 insertions(+), 39 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8dcc18acf..c895074ea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -262,18 +262,18 @@ before_script: tests/testcases/040_check-network-adv.yml $LOG_LEVEL; fi - after_script: - - > - ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local $LOG_LEVEL - -e mode=${CLUSTER_MODE} - -e test_id=${TEST_ID} - -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} - -e gce_project_id=${GCE_PROJECT_ID} - -e gce_service_account_email=${GCE_ACCOUNT} - -e gce_credentials_file=${HOME}/.ssh/gce.json - -e cloud_image=${CLOUD_IMAGE} - -e inventory_path=${PWD}/inventory/inventory.ini - -e cloud_region=${CLOUD_REGION} +# after_script: +# - > +# ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local $LOG_LEVEL +# -e mode=${CLUSTER_MODE} +# -e test_id=${TEST_ID} +# -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} +# -e gce_project_id=${GCE_PROJECT_ID} +# -e gce_service_account_email=${GCE_ACCOUNT} +# -e gce_credentials_file=${HOME}/.ssh/gce.json +# -e cloud_image=${CLOUD_IMAGE} +# -e inventory_path=${PWD}/inventory/inventory.ini +# -e cloud_region=${CLOUD_REGION} # Test matrix. Leave the comments for markup scripts. .coreos_calico_aio_variables: &coreos_calico_aio_variables diff --git a/docs/upgrades.md b/docs/upgrades.md index 9a21cbdc4..6f1d91317 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -67,3 +67,17 @@ follows: * network_plugin (such as Calico or Weave) * kube-apiserver, kube-scheduler, and kube-controller-manager * Add-ons (such as KubeDNS) + +#### Upgrade considerations + +Kubespray supports rotating certificates used for etcd and Kubernetes +components, but some manual steps may be required. If you have a pod that +requires use of a service token and is deployed in a namespace other than +`kube-system`, you will need to manually delete the affected pods after +rotating certificates. This is because all service account tokens are dependent +on the apiserver token that is used to generate them. When the certificate +rotates, all service account tokens must be rotated as well. During the +kubernetes-apps/rotate_tokens role, only pods in kube-system are destroyed and +recreated. All other invalidated service account tokens are cleaned up +automatically, but other pods are not deleted out of an abundance of caution +for impact to user deployed pods. diff --git a/roles/kubernetes-apps/network_plugin/weave/tasks/main.yml b/roles/kubernetes-apps/network_plugin/weave/tasks/main.yml index 3b01d0e66..66d900d55 100644 --- a/roles/kubernetes-apps/network_plugin/weave/tasks/main.yml +++ b/roles/kubernetes-apps/network_plugin/weave/tasks/main.yml @@ -1,15 +1,4 @@ --- -# FIXME: remove if kubernetes/features#124 is implemented -- name: Weave | Purge old weave daemonset - kube: - name: "weave-net" - kubectl: "{{ bin_dir }}/kubectl" - filename: "{{ kube_config_dir }}/weave-net.yml" - resource: "ds" - namespace: "{{system_namespace}}" - state: absent - when: inventory_hostname == groups['kube-master'][0] and weave_manifest.changed - - name: Weave | Start Resources kube: name: "weave-net" diff --git a/roles/kubernetes-apps/rotate_tokens/tasks/main.yml b/roles/kubernetes-apps/rotate_tokens/tasks/main.yml index bb17d9066..5c7678ead 100644 --- a/roles/kubernetes-apps/rotate_tokens/tasks/main.yml +++ b/roles/kubernetes-apps/rotate_tokens/tasks/main.yml @@ -1,16 +1,12 @@ --- -- name: Rotate Tokens | Get list of pods and their current secrets - command: >- - {{ bin_dir }}/kubectl get pods --all-namespaces - -o 'jsonpath={range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.spec.volumes[*].name}{"\n"}{end}' - register: pods_secrets - run_once: true - -- name: Rotate Tokens | Get default tokens to expire +#FIXME(mattymo): Exclude built in secrets that were automatically rotated, +#instead of filtering manually +- name: Rotate Tokens | Get all serviceaccount tokens to expire shell: >- {{ bin_dir }}/kubectl get secrets --all-namespaces - -o 'jsonpath={range .items[*]}{"\n"}{.metadata.namespace}{" "}{.metadata.name}{end}' - | grep default-token + -o 'jsonpath={range .items[*]}{"\n"}{.metadata.namespace}{" "}{.metadata.name}{" "}{.type}{end}' + | grep kubernetes.io/service-account-token + | egrep 'default-token|kube-dns|dnsmasq|netchecker|weave|calico|canal|flannel|dashboard|cluster-proportional-autoscaler|efk|tiller' register: tokens_to_delete run_once: true @@ -19,10 +15,6 @@ with_items: "{{ tokens_to_delete.stdout_lines }}" run_once: true -- name: Rotate Tokens | Delete pods - command: "{{ bin_dir }}/kubectl delete pod -n {{ item.split(' ')[0] }} {{ item.split(' ')[1] }}" - with_items: "{{ pods_secrets.stdout_lines }}" - register: delete_pods - when: item.split(" ")[0] + " " + item.split(" ")[2] in tokens_to_delete.stdout - failed_when: delete_pods.rc != 0 and "not found" not in delete_pods.stderr +- name: Rotate Tokens | Delete pods in system namespace + command: "{{ bin_dir }}/kubectl delete pods -n {{ system_namespace }} --all" run_once: true