From 222a77dfe754cb05cfcda3fe872f7bf2230b0bd9 Mon Sep 17 00:00:00 2001
From: Florian Ruynat <16313165+floryut@users.noreply.github.com>
Date: Thu, 21 Jan 2021 17:13:03 +0100
Subject: [PATCH] Change node-role.kubernetes.io from master to control-plane
 (#7183)

---
 inventory/sample/group_vars/k8s-cluster/addons.yml         | 4 ++++
 .../ansible/templates/coredns-deployment.yml.j2            | 6 ++++++
 roles/kubernetes-apps/ansible/templates/dashboard.yml.j2   | 4 ++++
 .../ansible/templates/dns-autoscaler.yml.j2                | 7 ++++++-
 .../oci/templates/oci-cloud-provider.yml.j2                | 7 +++++++
 .../templates/azure-csi-azuredisk-controller.yml.j2        | 4 ++--
 .../vsphere/templates/vsphere-csi-controller-ss.yml.j2     | 7 +++++++
 .../external-openstack-cloud-controller-manager-ds.yml.j2  | 6 ++++++
 .../external-vsphere-cloud-controller-manager-ds.yml.j2    | 6 ++++++
 .../ambassador/templates/cr-ambassador-installation.yml.j2 | 5 +++--
 roles/kubernetes-apps/metallb/templates/metallb.yml.j2     | 2 ++
 .../templates/metrics-server-deployment.yaml.j2            | 6 ++++++
 .../calico/templates/calico-kube-controllers.yml.j2        | 2 ++
 roles/kubernetes/master/tasks/kubeadm-setup.yml            | 2 +-
 roles/network_plugin/calico/templates/calico-typha.yml.j2  | 3 +++
 .../ovn4nfv/templates/ovn4nfv-k8s-plugin.yml.j2            | 3 +++
 16 files changed, 68 insertions(+), 6 deletions(-)

diff --git a/inventory/sample/group_vars/k8s-cluster/addons.yml b/inventory/sample/group_vars/k8s-cluster/addons.yml
index 5925a4d2c..32a86e4a5 100644
--- a/inventory/sample/group_vars/k8s-cluster/addons.yml
+++ b/inventory/sample/group_vars/k8s-cluster/addons.yml
@@ -94,6 +94,10 @@ ingress_publish_status_address: ""
 #     operator: "Equal"
 #     value: ""
 #     effect: "NoSchedule"
+#   - key: "node-role.kubernetes.io/control-plane"
+#     operator: "Equal"
+#     value: ""
+#     effect: "NoSchedule"
 # ingress_nginx_namespace: "ingress-nginx"
 # ingress_nginx_insecure_port: 80
 # ingress_nginx_secure_port: 443
diff --git a/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2
index 6a7253422..d14dde08b 100644
--- a/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2
@@ -31,6 +31,8 @@ spec:
       tolerations:
         - key: node-role.kubernetes.io/master
           effect: NoSchedule
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
 {% if dns_extra_tolerations | default(None) %}
         {{ dns_extra_tolerations | list | to_nice_yaml(indent=2) | indent(8) }}
 {% endif %}
@@ -46,7 +48,11 @@ spec:
           - weight: 100
             preference:
               matchExpressions:
+{% if kube_version is version('v1.20.0', '<') %}
               - key: node-role.kubernetes.io/master
+{% else %}
+              - key: node-role.kubernetes.io/control-plane
+{% endif %}
                 operator: In
                 values:
                 - ""
diff --git a/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2 b/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2
index ea8c27845..d75b2cd08 100644
--- a/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2
@@ -219,6 +219,8 @@ spec:
       tolerations:
         - key: node-role.kubernetes.io/master
           effect: NoSchedule
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
 {% endif %}
 
 ---
@@ -316,4 +318,6 @@ spec:
       tolerations:
         - key: node-role.kubernetes.io/master
           effect: NoSchedule
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
 {% endif %}
diff --git a/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2 b/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2
index ead631e07..b49c41264 100644
--- a/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2
@@ -40,8 +40,9 @@ spec:
         kubernetes.io/os: linux
       tolerations:
         - effect: NoSchedule
-          operator: Equal
           key: node-role.kubernetes.io/master
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
       affinity:
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -54,7 +55,11 @@ spec:
           - weight: 100
             preference:
               matchExpressions:
+{% if kube_version is version('v1.20.0', '<') %}
               - key: node-role.kubernetes.io/master
+{% else %}
+              - key: node-role.kubernetes.io/control-plane
+{% endif %}
                 operator: In
                 values:
                 - ""
diff --git a/roles/kubernetes-apps/cloud_controller/oci/templates/oci-cloud-provider.yml.j2 b/roles/kubernetes-apps/cloud_controller/oci/templates/oci-cloud-provider.yml.j2
index 7ed87603c..071432401 100644
--- a/roles/kubernetes-apps/cloud_controller/oci/templates/oci-cloud-provider.yml.j2
+++ b/roles/kubernetes-apps/cloud_controller/oci/templates/oci-cloud-provider.yml.j2
@@ -36,7 +36,11 @@ spec:
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
       nodeSelector:
+{% if kube_version is version('v1.20.0', '<') %}
         node-role.kubernetes.io/master: ""
+{% else %}
+        node-role.kubernetes.io/control-plane: ""
+{% endif %}
       tolerations:
       - key: node.cloudprovider.kubernetes.io/uninitialized
         value: "true"
@@ -44,6 +48,9 @@ spec:
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
       volumes:
         - name: cfg
           secret:
diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2
index ab7a540f8..659c9b965 100644
--- a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2
@@ -21,8 +21,8 @@ spec:
       priorityClassName: system-cluster-critical
       tolerations:
         - key: "node-role.kubernetes.io/master"
-          operator: "Equal"
-          value: "true"
+          effect: "NoSchedule"
+        - key: "node-role.kubernetes.io/control-plane"
           effect: "NoSchedule"
       containers:
         - name: csi-provisioner
diff --git a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-ss.yml.j2 b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-ss.yml.j2
index 24651d90d..b762b496d 100644
--- a/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-ss.yml.j2
+++ b/roles/kubernetes-apps/csi_driver/vsphere/templates/vsphere-csi-controller-ss.yml.j2
@@ -19,11 +19,18 @@ spec:
     spec:
       serviceAccountName: vsphere-csi-controller
       nodeSelector:
+{% if kube_version is version('v1.20.0', '<') %}
         node-role.kubernetes.io/master: ""
+{% else %}
+        node-role.kubernetes.io/control-plane: ""
+{% endif %}
       tolerations:
         - operator: "Exists"
           key: node-role.kubernetes.io/master
           effect: NoSchedule
+        - operator: "Exists"
+          key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
       dnsPolicy: "Default"
       containers:
         - name: csi-attacher
diff --git a/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2
index 295ecee2d..c623fecce 100644
--- a/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2
+++ b/roles/kubernetes-apps/external_cloud_controller/openstack/templates/external-openstack-cloud-controller-manager-ds.yml.j2
@@ -24,7 +24,11 @@ spec:
         k8s-app: openstack-cloud-controller-manager
     spec:
       nodeSelector:
+{% if kube_version is version('v1.20.0', '<') %}
         node-role.kubernetes.io/master: ""
+{% else %}
+        node-role.kubernetes.io/control-plane: ""
+{% endif %}
       securityContext:
         runAsUser: 1001
       tolerations:
@@ -33,6 +37,8 @@ spec:
         effect: NoSchedule
       - key: node-role.kubernetes.io/master
         effect: NoSchedule
+      - key: node-role.kubernetes.io/control-plane
+        effect: NoSchedule
       serviceAccountName: cloud-controller-manager
       containers:
         - name: openstack-cloud-controller-manager
diff --git a/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2
index 8bd4e6410..dc1b7ffda 100644
--- a/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2
+++ b/roles/kubernetes-apps/external_cloud_controller/vsphere/templates/external-vsphere-cloud-controller-manager-ds.yml.j2
@@ -24,7 +24,11 @@ spec:
         k8s-app: vsphere-cloud-controller-manager
     spec:
       nodeSelector:
+{% if kube_version is version('v1.20.0', '<') %}
         node-role.kubernetes.io/master: ""
+{% else %}
+        node-role.kubernetes.io/control-plane: ""
+{% endif %}
       securityContext:
         runAsUser: 0
       tolerations:
@@ -33,6 +37,8 @@ spec:
         effect: NoSchedule
       - key: node-role.kubernetes.io/master
         effect: NoSchedule
+      - key: node-role.kubernetes.io/control-plane
+        effect: NoSchedule
       serviceAccountName: cloud-controller-manager
       containers:
         - name: vsphere-cloud-controller-manager
diff --git a/roles/kubernetes-apps/ingress_controller/ambassador/templates/cr-ambassador-installation.yml.j2 b/roles/kubernetes-apps/ingress_controller/ambassador/templates/cr-ambassador-installation.yml.j2
index d1a6fb216..8449cd5b8 100644
--- a/roles/kubernetes-apps/ingress_controller/ambassador/templates/cr-ambassador-installation.yml.j2
+++ b/roles/kubernetes-apps/ingress_controller/ambassador/templates/cr-ambassador-installation.yml.j2
@@ -16,7 +16,8 @@ spec:
   helmValues:
     tolerations:
       - key: "node-role.kubernetes.io/master"
-        operator: Equal
+        effect: NoSchedule
+      - key: "node-role.kubernetes.io/control-plane"
         effect: NoSchedule
     deploymentTool: amb-oper-kubespray
 {% if ingress_ambassador_host_network %}
@@ -34,4 +35,4 @@ spec:
           port: 443
           hostPort: {{ ingress_ambassador_secure_port }}
           targetPort: 8443
-          protocol: TCP
\ No newline at end of file
+          protocol: TCP
diff --git a/roles/kubernetes-apps/metallb/templates/metallb.yml.j2 b/roles/kubernetes-apps/metallb/templates/metallb.yml.j2
index af4c0f215..392c02137 100644
--- a/roles/kubernetes-apps/metallb/templates/metallb.yml.j2
+++ b/roles/kubernetes-apps/metallb/templates/metallb.yml.j2
@@ -345,6 +345,8 @@ spec:
       tolerations:
       - effect: NoSchedule
         key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/control-plane
 ---
 apiVersion: apps/v1
 kind: Deployment
diff --git a/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2
index d636d6ad9..746d7c352 100644
--- a/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2
+++ b/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2
@@ -126,6 +126,8 @@ spec:
       tolerations:
         - key: node-role.kubernetes.io/master
           effect: NoSchedule
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
 {% endif %}
       affinity:
         nodeAffinity:
@@ -133,7 +135,11 @@ spec:
           - weight: 100
             preference:
               matchExpressions:
+{% if kube_version is version('v1.20.0', '<') %}
               - key: node-role.kubernetes.io/master
+{% else %}
+              - key: node-role.kubernetes.io/control-plane
+{% endif %}
                 operator: In
                 values:
                 - ""
diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
index c1db6b685..f861d918d 100644
--- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
+++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
@@ -26,6 +26,8 @@ spec:
       tolerations:
         - key: node-role.kubernetes.io/master
           effect: NoSchedule
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
       priorityClassName: system-cluster-critical
       containers:
         - name: calico-kube-controllers
diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml
index 43655a30d..1cef72396 100644
--- a/roles/kubernetes/master/tasks/kubeadm-setup.yml
+++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml
@@ -226,7 +226,7 @@
 
 # FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file.
 - name: kubeadm | Remove taint for master with node role
-  command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-"
+  command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule- node-role.kubernetes.io/control-plane:NoSchedule-"
   delegate_to: "{{ groups['kube-master'] | first }}"
   when: inventory_hostname in groups['kube-node']
   failed_when: false
diff --git a/roles/network_plugin/calico/templates/calico-typha.yml.j2 b/roles/network_plugin/calico/templates/calico-typha.yml.j2
index c23e93d46..143a1711e 100644
--- a/roles/network_plugin/calico/templates/calico-typha.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-typha.yml.j2
@@ -54,6 +54,9 @@ spec:
         - key: node-role.kubernetes.io/master
           operator: Exists
           effect: NoSchedule
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
       # Since Calico can't network a pod until Typha is up, we need to run Typha itself
       # as a host-networked pod.
       serviceAccountName: calico-node
diff --git a/roles/network_plugin/ovn4nfv/templates/ovn4nfv-k8s-plugin.yml.j2 b/roles/network_plugin/ovn4nfv/templates/ovn4nfv-k8s-plugin.yml.j2
index 127115276..5e76f8e6a 100644
--- a/roles/network_plugin/ovn4nfv/templates/ovn4nfv-k8s-plugin.yml.j2
+++ b/roles/network_plugin/ovn4nfv/templates/ovn4nfv-k8s-plugin.yml.j2
@@ -414,6 +414,9 @@ spec:
        - key: "node-role.kubernetes.io/master"
          effect: "NoSchedule"
          operator: "Exists"
+       - key: "node-role.kubernetes.io/control-plane"
+         effect: "NoSchedule"
+         operator: "Exists"
       serviceAccountName: k8s-nfn-sa
       containers:
         - name: nfn-operator