Master component and kubelet container upgrade fixes

* Fixes an issue where apiserver and friends (controller manager, scheduler) were prevented from restarting after manifests/secrets are changed. This occurred when a replaced kubelet doesn't reconcile new master manifests, which caused old master component versions to linger during deployment. In my case this was causing upgrades from k8s 1.6/1.7 -> k8s 1.8 to fail
* Improves transitions from kubelet container to host kubelet by preventing issues where kubelet container reappeared during the deployment
This commit is contained in:
Chad Swenson 2017-11-08 01:40:33 -06:00
parent 8b2bec700a
commit e9f795c5ce
4 changed files with 47 additions and 7 deletions

View file

@ -13,6 +13,24 @@
- Master | wait for kube-scheduler
- Master | wait for kube-controller-manager
- name: Master | Restart apiserver
command: /bin/true
notify:
- Master | Remove apiserver container
- Master | wait for the apiserver to be running
- name: Master | Restart kube-scheduler
command: /bin/true
notify:
- Master | Remove scheduler container
- Master | wait for kube-scheduler
- name: Master | Restart kube-controller-manager
command: /bin/true
notify:
- Master | Remove controller manager container
- Master | wait for kube-controller-manager
- name: Master | reload systemd
command: systemctl daemon-reload
@ -21,6 +39,15 @@
name: kubelet
state: restarted
- name: Master | Remove apiserver container
shell: "docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f"
- name: Master | Remove scheduler container
shell: "docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty docker rm -f"
- name: Master | Remove controller manager container
shell: "docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty docker rm -f"
- name: Master | wait for kube-scheduler
uri:
url: http://localhost:10251/healthz

View file

@ -24,7 +24,7 @@
when: (secret_changed|default(false) or etcd_secret_changed|default(false))
- name: "Pre-upgrade | Delete master containers forcefully"
shell: "docker ps -f name=k8s-{{item}}* -q | xargs --no-run-if-empty docker rm -f"
shell: "docker ps -af name=k8s_{{item}}* -q | xargs --no-run-if-empty docker rm -f"
with_items:
- ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
when: kube_apiserver_manifest_replaced.changed

View file

@ -3,7 +3,7 @@
template:
src: manifests/kube-apiserver.manifest.j2
dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
notify: Master | wait for the apiserver to be running
notify: Master | Restart apiserver
tags:
- kube-apiserver
@ -20,7 +20,7 @@
template:
src: manifests/kube-scheduler.manifest.j2
dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
notify: Master | wait for kube-scheduler
notify: Master | Restart kube-scheduler
tags:
- kube-scheduler
@ -35,6 +35,6 @@
template:
src: manifests/kube-controller-manager.manifest.j2
dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
notify: Master | wait for kube-controller-manager
notify: Master | Restart kube-controller-manager
tags:
- kube-controller-manager

View file

@ -1,12 +1,25 @@
---
- name: "Pre-upgrade | check if kubelet container exists"
shell: docker ps -af name=kubelet | grep kubelet
failed_when: false
changed_when: false
register: kubelet_container_check
- name: "Pre-upgrade | copy /var/lib/cni from kubelet"
command: docker cp kubelet:/var/lib/cni /var/lib/cni
args:
creates: "/var/lib/cni"
failed_when: false
when: kubelet_container_check.rc == 0
- name: "Pre-upgrade | ensure kubelet container is stopped if using host deployment"
command: docker stop kubelet
- name: "Pre-upgrade | ensure kubelet container service is stopped if using host deployment"
service:
name: kubelet
state: stopped
when: kubelet_deployment_type == 'host' and kubelet_container_check.rc == 0
- name: "Pre-upgrade | ensure kubelet container is removed if using host deployment"
command: docker rm -fv kubelet
failed_when: false
changed_when: false
when: kubelet_deployment_type == 'host'
when: kubelet_deployment_type == 'host' and kubelet_container_check.rc == 0