delete more secrets

.gitlab-ci.yml

@@ -262,18 +262,18 @@ before_script:
       tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
       fi
 
-  after_script:
-    - >
-      ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local  $LOG_LEVEL
-      -e mode=${CLUSTER_MODE}
-      -e test_id=${TEST_ID}
-      -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
-      -e gce_project_id=${GCE_PROJECT_ID}
-      -e gce_service_account_email=${GCE_ACCOUNT}
-      -e gce_credentials_file=${HOME}/.ssh/gce.json
-      -e cloud_image=${CLOUD_IMAGE}
-      -e inventory_path=${PWD}/inventory/inventory.ini
-      -e cloud_region=${CLOUD_REGION}
+#  after_script:
+#    - >
+#      ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local  $LOG_LEVEL
+#      -e mode=${CLUSTER_MODE}
+#      -e test_id=${TEST_ID}
+#      -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
+#      -e gce_project_id=${GCE_PROJECT_ID}
+#      -e gce_service_account_email=${GCE_ACCOUNT}
+#      -e gce_credentials_file=${HOME}/.ssh/gce.json
+#      -e cloud_image=${CLOUD_IMAGE}
+#      -e inventory_path=${PWD}/inventory/inventory.ini
+#      -e cloud_region=${CLOUD_REGION}
 
 # Test matrix. Leave the comments for markup scripts.
 .coreos_calico_aio_variables: &coreos_calico_aio_variables

docs/upgrades.md

@@ -67,3 +67,17 @@ follows:
 * network_plugin (such as Calico or Weave)
 * kube-apiserver, kube-scheduler, and kube-controller-manager
 * Add-ons (such as KubeDNS)
+
+#### Upgrade considerations
+
+Kubespray supports rotating certificates used for etcd and Kubernetes
+components, but some manual steps may be required. If you have a pod that
+requires use of a service token and is deployed in a namespace other than
+`kube-system`, you will need to manually delete the affected pods after
+rotating certificates. This is because all service account tokens are dependent
+on the apiserver token that is used to generate them. When the certificate
+rotates, all service account tokens must be rotated as well. During the
+kubernetes-apps/rotate_tokens role, only pods in kube-system are destroyed and
+recreated. All other invalidated service account tokens are cleaned up
+automatically, but other pods are not deleted out of an abundance of caution
+for impact to user deployed pods.

roles/kubernetes-apps/network_plugin/weave/tasks/main.yml

@@ -1,15 +1,4 @@
 ---
-# FIXME: remove if kubernetes/features#124 is implemented
-- name: Weave | Purge old weave daemonset
-  kube:
-    name: "weave-net"
-    kubectl: "{{ bin_dir }}/kubectl"
-    filename: "{{ kube_config_dir }}/weave-net.yml"
-    resource: "ds"
-    namespace: "{{system_namespace}}"
-    state: absent
-  when: inventory_hostname == groups['kube-master'][0] and weave_manifest.changed
-
 - name: Weave | Start Resources
   kube:
     name: "weave-net"

roles/kubernetes-apps/rotate_tokens/tasks/main.yml

@@ -1,16 +1,12 @@
 ---
-- name: Rotate Tokens | Get list of pods and their current secrets
-  command: >-
-    {{ bin_dir }}/kubectl get pods --all-namespaces
-    -o 'jsonpath={range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.spec.volumes[*].name}{"\n"}{end}'
-  register: pods_secrets
-  run_once: true
-
-- name: Rotate Tokens | Get default tokens to expire
+#FIXME(mattymo): Exclude built in secrets that were automatically rotated,
+#instead of filtering manually
+- name: Rotate Tokens | Get all serviceaccount tokens to expire
   shell: >-
     {{ bin_dir }}/kubectl get secrets --all-namespaces
-    -o 'jsonpath={range .items[*]}{"\n"}{.metadata.namespace}{" "}{.metadata.name}{end}'
-    | grep default-token
+    -o 'jsonpath={range .items[*]}{"\n"}{.metadata.namespace}{" "}{.metadata.name}{" "}{.type}{end}'
+    | grep kubernetes.io/service-account-token
+    | egrep 'default-token|kube-dns|dnsmasq|netchecker|weave|calico|canal|flannel|dashboard|cluster-proportional-autoscaler|efk|tiller'
   register: tokens_to_delete
   run_once: true
 
@@ -19,10 +15,6 @@
   with_items: "{{ tokens_to_delete.stdout_lines }}"
   run_once: true
 
-- name: Rotate Tokens | Delete pods
-  command: "{{ bin_dir }}/kubectl delete pod -n {{ item.split(' ')[0] }} {{ item.split(' ')[1] }}"
-  with_items: "{{ pods_secrets.stdout_lines }}"
-  register: delete_pods
-  when: item.split(" ")[0] + " " + item.split(" ")[2] in tokens_to_delete.stdout
-  failed_when: delete_pods.rc != 0 and "not found" not in delete_pods.stderr
+- name: Rotate Tokens | Delete pods in system namespace
+  command: "{{ bin_dir }}/kubectl delete pods -n {{ system_namespace }} --all"
   run_once: true