After upgrade, allow cilium to be back before uncordoning (#7978)

* After upgrade, allow cilium to be back before uncordoning

* add eol

* use kube_config_dir variable
resolves https://github.com/kubernetes-sigs/kubespray/pull/7978#discussion_r721685549

roles/upgrade/post-upgrade/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+# how long to wait for cilium after upgrade before uncordoning
+upgrade_post_cilium_wait_timeout: 120s

roles/upgrade/post-upgrade/tasks/main.yml

@@ -1,4 +1,16 @@
 ---
+- name: wait for cilium
+  when:
+    - needs_cordoning|default(false)
+    - kube_network_plugin == 'cilium'
+  command: >
+    {{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf
+    wait pod -n kube-system -l k8s-app=cilium
+    --field-selector 'spec.nodeName=={{ kube_override_hostname|default(inventory_hostname) }}'
+    --for=condition=Ready
+    --timeout={{ upgrade_post_cilium_wait_timeout }}
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+
 - name: Uncordon node
   command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf uncordon {{ kube_override_hostname|default(inventory_hostname) }}"
   delegate_to: "{{ groups['kube_control_plane'][0] }}"