All CNIs: support ANY toleration. (#3391)

Before, Nodes tainted with NoExecute policy did not have calico/weave Pod. Network pod should run on all nodes whatever happens on a specific node. Also always set the Pods to be critical. Also remove deprecated scheduler.alpha.kubernetes.io/tolerations annotations.
2018-09-27 14:28:54 +02:00 · 2018-09-27 14:28:54 +02:00 · 53d87e53c5
commit 53d87e53c5
parent 232020ef96
11 changed files with 58 additions and 31 deletions
--- a/roles/network_plugin/calico/templates/calico-node.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-node.yml.j2
@ -18,6 +18,7 @@ spec:
      labels:
        k8s-app: calico-node
      annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
        scheduler.alpha.kubernetes.io/critical-pod: ''
        kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
    spec:
@ -27,8 +28,10 @@ spec:
      hostNetwork: true
      serviceAccountName: calico-node
      tolerations:
-        - effect: NoSchedule
-          operator: Exists
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
      # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
      # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
      terminationGracePeriodSeconds: 0
@ -189,4 +192,4 @@ spec:
  updateStrategy:
    rollingUpdate:
      maxUnavailable: {{ serial | default('20%') }}
-    type: RollingUpdate
+    type: RollingUpdate
--- a/roles/network_plugin/canal/templates/canal-node.yaml.j2
+++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2
@ -13,8 +13,8 @@ spec:
  template:
    metadata:
      annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
        scheduler.alpha.kubernetes.io/critical-pod: ''
-        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
      labels:
        k8s-app: canal-node
    spec:
@ -24,8 +24,10 @@ spec:
      hostNetwork: true
      serviceAccountName: canal
      tolerations:
-        - effect: NoSchedule
-          operator: Exists
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
      volumes:
        # Used by calico/node.
        - name: lib-modules
--- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
@ -27,8 +27,6 @@ spec:
        # gets priority scheduling.
        # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
        scheduler.alpha.kubernetes.io/critical-pod: ''
-        scheduler.alpha.kubernetes.io/tolerations: >-
-          [{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]
 {% if cilium_enable_prometheus %}
        prometheus.io/scrape: "true"
        prometheus.io/port: "9090"
@ -225,11 +223,7 @@ spec:

      restartPolicy: Always
      tolerations:
-        - effect: NoSchedule
-          key: node-role.kubernetes.io/master
-        - effect: NoSchedule
-          key: node.cloudprovider.kubernetes.io/uninitialized
-          value: "true"
-        # Mark cilium's pod as critical for rescheduling
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
        - key: CriticalAddonsOnly
          operator: "Exists"
--- a/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2
@ -16,6 +16,7 @@ spec:
      labels:
        k8s-app: contiv-api-proxy
      annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
@ -28,8 +29,10 @@ spec:
      nodeSelector:
        node-role.kubernetes.io/master: "true"
      tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
      serviceAccountName: contiv-netmaster
      containers:
        - name: contiv-api-proxy
--- a/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2
@ -14,6 +14,9 @@ spec:
    metadata:
      labels:
        k8s-app: contiv-cleanup
+      annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
      priorityClassName: system-node-critical
@ -21,8 +24,10 @@ spec:
      hostNetwork: true
      hostPID: true
      tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
      serviceAccountName: contiv-netplugin
      containers:
      - name: contiv-ovs-cleanup
--- a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2
@ -25,8 +25,10 @@ spec:
      nodeSelector:
        node-role.kubernetes.io/master: "true"
      tolerations:
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
      initContainers:
        - name: contiv-etcd-init
          image: {{ contiv_etcd_init_image_repo }}:{{ contiv_etcd_init_image_tag }}
--- a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2
@ -16,6 +16,7 @@ spec:
      labels:
        k8s-app: contiv-netmaster
      annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
@ -28,8 +29,10 @@ spec:
      nodeSelector:
        node-role.kubernetes.io/master: "true"
      tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
      serviceAccountName: contiv-netmaster
      containers:
        - name: contiv-netmaster
--- a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2
@ -20,6 +20,7 @@ spec:
      labels:
        k8s-app: contiv-netplugin
      annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
@ -28,8 +29,10 @@ spec:
      hostNetwork: true
      hostPID: true
      tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
      serviceAccountName: contiv-netplugin
      initContainers:
        - name: contiv-netplugin-init
--- a/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2
@ -17,6 +17,7 @@ spec:
      labels:
        k8s-app: contiv-ovs
      annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
@ -25,8 +26,10 @@ spec:
      hostNetwork: true
      hostPID: true
      tolerations:
-      - key: node-role.kubernetes.io/master
-        effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
      containers:
      # Runs ovs containers on each Kubernetes node.
      - name: contiv-ovsdb-server
--- a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2
+++ b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2
@ -51,6 +51,9 @@ spec:
      labels:
        tier: node
        k8s-app: flannel
+      annotations:
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
      priorityClassName: system-node-critical
@ -108,9 +111,10 @@ spec:
          mountPath: /host/opt/cni/bin/
      hostNetwork: true
      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
+        - operator: Exists
+        # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+        - key: CriticalAddonsOnly
+          operator: "Exists"
      volumes:
        - name: run
          hostPath:
--- a/roles/network_plugin/weave/templates/weave-net.yml.j2
+++ b/roles/network_plugin/weave/templates/weave-net.yml.j2
@ -114,6 +114,9 @@ items:
        metadata:
          labels:
            name: weave-net
+          annotations:
+            # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+            scheduler.alpha.kubernetes.io/critical-pod: ''
        spec:
 {% if kube_version|version_compare('v1.11.1', '>=') %}
          priorityClassName: system-node-critical
@ -224,8 +227,10 @@ items:
            seLinuxOptions: {}
          serviceAccountName: weave-net
          tolerations:
-            - effect: NoSchedule
-              operator: Exists
+            - operator: Exists
+            # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+            - key: CriticalAddonsOnly
+              operator: "Exists"
          volumes:
            - name: weavedb
              hostPath: