From 53d87e53c5899d4ea2904ab7e3883708dd6363d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20de=20Saint=20Martin?= Date: Thu, 27 Sep 2018 14:28:54 +0200 Subject: [PATCH] All CNIs: support ANY toleration. (#3391) Before, Nodes tainted with NoExecute policy did not have calico/weave Pod. Network pod should run on all nodes whatever happens on a specific node. Also always set the Pods to be critical. Also remove deprecated scheduler.alpha.kubernetes.io/tolerations annotations. --- .../network_plugin/calico/templates/calico-node.yml.j2 | 9 ++++++--- .../network_plugin/canal/templates/canal-node.yaml.j2 | 8 +++++--- roles/network_plugin/cilium/templates/cilium-ds.yml.j2 | 10 ++-------- .../contiv/templates/contiv-api-proxy.yml.j2 | 7 +++++-- .../contiv/templates/contiv-cleanup.yml.j2 | 9 +++++++-- .../network_plugin/contiv/templates/contiv-etcd.yml.j2 | 6 ++++-- .../contiv/templates/contiv-netmaster.yml.j2 | 7 +++++-- .../contiv/templates/contiv-netplugin.yml.j2 | 7 +++++-- .../network_plugin/contiv/templates/contiv-ovs.yml.j2 | 7 +++++-- .../flannel/templates/cni-flannel.yml.j2 | 10 +++++++--- roles/network_plugin/weave/templates/weave-net.yml.j2 | 9 +++++++-- 11 files changed, 58 insertions(+), 31 deletions(-) diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2 index 539ced8a6..c692bc925 100644 --- a/roles/network_plugin/calico/templates/calico-node.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-node.yml.j2 @@ -18,6 +18,7 @@ spec: labels: k8s-app: calico-node annotations: + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) scheduler.alpha.kubernetes.io/critical-pod: '' kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}" spec: @@ -27,8 +28,10 @@ spec: hostNetwork: true serviceAccountName: calico-node tolerations: - - effect: NoSchedule - operator: Exists + - operator: Exists + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) + - key: CriticalAddonsOnly + operator: "Exists" # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 @@ -189,4 +192,4 @@ spec: updateStrategy: rollingUpdate: maxUnavailable: {{ serial | default('20%') }} - type: RollingUpdate \ No newline at end of file + type: RollingUpdate diff --git a/roles/network_plugin/canal/templates/canal-node.yaml.j2 b/roles/network_plugin/canal/templates/canal-node.yaml.j2 index ea34dfa89..e0d0c7cff 100644 --- a/roles/network_plugin/canal/templates/canal-node.yaml.j2 +++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2 @@ -13,8 +13,8 @@ spec: template: metadata: annotations: + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) scheduler.alpha.kubernetes.io/critical-pod: '' - scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' labels: k8s-app: canal-node spec: @@ -24,8 +24,10 @@ spec: hostNetwork: true serviceAccountName: canal tolerations: - - effect: NoSchedule - operator: Exists + - operator: Exists + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) + - key: CriticalAddonsOnly + operator: "Exists" volumes: # Used by calico/node. - name: lib-modules diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 index 4eff22269..ff76d6d7c 100755 --- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 @@ -27,8 +27,6 @@ spec: # gets priority scheduling. # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ scheduler.alpha.kubernetes.io/critical-pod: '' - scheduler.alpha.kubernetes.io/tolerations: >- - [{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}] {% if cilium_enable_prometheus %} prometheus.io/scrape: "true" prometheus.io/port: "9090" @@ -225,11 +223,7 @@ spec: restartPolicy: Always tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - # Mark cilium's pod as critical for rescheduling + - operator: Exists + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - key: CriticalAddonsOnly operator: "Exists" diff --git a/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 b/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 index f37e83847..706027623 100644 --- a/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 @@ -16,6 +16,7 @@ spec: labels: k8s-app: contiv-api-proxy annotations: + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version|version_compare('v1.11.1', '>=') %} @@ -28,8 +29,10 @@ spec: nodeSelector: node-role.kubernetes.io/master: "true" tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule + - operator: Exists + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) + - key: CriticalAddonsOnly + operator: "Exists" serviceAccountName: contiv-netmaster containers: - name: contiv-api-proxy diff --git a/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 b/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 index 8555c133d..3f715a473 100644 --- a/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 @@ -14,6 +14,9 @@ spec: metadata: labels: k8s-app: contiv-cleanup + annotations: + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) + scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version|version_compare('v1.11.1', '>=') %} priorityClassName: system-node-critical @@ -21,8 +24,10 @@ spec: hostNetwork: true hostPID: true tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule + - operator: Exists + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) + - key: CriticalAddonsOnly + operator: "Exists" serviceAccountName: contiv-netplugin containers: - name: contiv-ovs-cleanup diff --git a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 index ba17452fa..134c9c5b5 100644 --- a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 @@ -25,8 +25,10 @@ spec: nodeSelector: node-role.kubernetes.io/master: "true" tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule + - operator: Exists + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) + - key: CriticalAddonsOnly + operator: "Exists" initContainers: - name: contiv-etcd-init image: {{ contiv_etcd_init_image_repo }}:{{ contiv_etcd_init_image_tag }} diff --git a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 index 5731d7c5c..55481b261 100644 --- a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 @@ -16,6 +16,7 @@ spec: labels: k8s-app: contiv-netmaster annotations: + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version|version_compare('v1.11.1', '>=') %} @@ -28,8 +29,10 @@ spec: nodeSelector: node-role.kubernetes.io/master: "true" tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule + - operator: Exists + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) + - key: CriticalAddonsOnly + operator: "Exists" serviceAccountName: contiv-netmaster containers: - name: contiv-netmaster diff --git a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 index e47f711bf..4a996edea 100644 --- a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 @@ -20,6 +20,7 @@ spec: labels: k8s-app: contiv-netplugin annotations: + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version|version_compare('v1.11.1', '>=') %} @@ -28,8 +29,10 @@ spec: hostNetwork: true hostPID: true tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule + - operator: Exists + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) + - key: CriticalAddonsOnly + operator: "Exists" serviceAccountName: contiv-netplugin initContainers: - name: contiv-netplugin-init diff --git a/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 b/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 index 27090c62f..0ded7fe7e 100644 --- a/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 @@ -17,6 +17,7 @@ spec: labels: k8s-app: contiv-ovs annotations: + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version|version_compare('v1.11.1', '>=') %} @@ -25,8 +26,10 @@ spec: hostNetwork: true hostPID: true tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule + - operator: Exists + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) + - key: CriticalAddonsOnly + operator: "Exists" containers: # Runs ovs containers on each Kubernetes node. - name: contiv-ovsdb-server diff --git a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 index c872d9893..d2340eed8 100644 --- a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 +++ b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 @@ -51,6 +51,9 @@ spec: labels: tier: node k8s-app: flannel + annotations: + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) + scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version|version_compare('v1.11.1', '>=') %} priorityClassName: system-node-critical @@ -108,9 +111,10 @@ spec: mountPath: /host/opt/cni/bin/ hostNetwork: true tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule + - operator: Exists + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) + - key: CriticalAddonsOnly + operator: "Exists" volumes: - name: run hostPath: diff --git a/roles/network_plugin/weave/templates/weave-net.yml.j2 b/roles/network_plugin/weave/templates/weave-net.yml.j2 index 59740e67e..b8a9a6871 100644 --- a/roles/network_plugin/weave/templates/weave-net.yml.j2 +++ b/roles/network_plugin/weave/templates/weave-net.yml.j2 @@ -114,6 +114,9 @@ items: metadata: labels: name: weave-net + annotations: + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) + scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version|version_compare('v1.11.1', '>=') %} priorityClassName: system-node-critical @@ -224,8 +227,10 @@ items: seLinuxOptions: {} serviceAccountName: weave-net tolerations: - - effect: NoSchedule - operator: Exists + - operator: Exists + # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) + - key: CriticalAddonsOnly + operator: "Exists" volumes: - name: weavedb hostPath: