Adding pod priority for all the components. (#3361)

* Changes to assign pod priority to kube components.

* Removed the boolean flag pod_priority_assignment

* Created new priorityclass k8s-cluster-critical

* Created new priorityclass k8s-cluster-critical

* Fixed the trailing spaces

* Fixed the trailing spaces

* Added kube version check while creating Priority Class k8s-cluster-critical

* Moved k8s-cluster-critical.yml

* Moved k8s-cluster-critical.yml to kube_config_dir

roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2

@@ -31,6 +31,9 @@ spec:
         scheduler.alpha.kubernetes.io/critical-pod: ''
         scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
       serviceAccountName: dnsmasq
       tolerations:
         - effect: NoSchedule

roles/dnsmasq/templates/dnsmasq-deploy.yml.j2

@@ -21,6 +21,9 @@ spec:
         kubernetes.io/cluster-service: "true"
         kubespray/dnsmasq-checksum: "{{ dnsmasq_stat.stat.checksum }}"
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
       tolerations:
         - effect: NoSchedule
           operator: Exists

roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2

@@ -26,6 +26,9 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
       serviceAccountName: coredns
       tolerations:
         - key: node-role.kubernetes.io/master

roles/kubernetes-apps/ansible/templates/dashboard.yml.j2

@@ -140,6 +140,9 @@ spec:
       labels:
         k8s-app: kubernetes-dashboard
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
       containers:
       - name: kubernetes-dashboard
         image: {{ dashboard_image_repo }}:{{ dashboard_image_tag }}

roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2

@@ -28,6 +28,9 @@ spec:
       labels:
         k8s-app: kubedns-autoscaler
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
       # When having win nodes in cluster without this patch, this pod cloud try to be created in windows
       nodeSelector:
         beta.kubernetes.io/os: linux

roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2

@@ -27,6 +27,9 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
       # When having win nodes in cluster without this patch, this pod cloud try to be created in windows
       nodeSelector:
         beta.kubernetes.io/os: linux

roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2

@@ -12,6 +12,9 @@ spec:
       labels:
         app: netchecker-agent
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if netcheck_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
       tolerations:
         - effect: NoSchedule
           operator: Exists

roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2

@@ -18,6 +18,9 @@ spec:
         beta.kubernetes.io/os: linux
 {% if kube_version | version_compare('v1.6', '>=') %}
       dnsPolicy: ClusterFirstWithHostNet
+{% endif %}
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if netcheck_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
 {% endif %}
       tolerations:
         - effect: NoSchedule

roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2

@@ -11,6 +11,9 @@ spec:
         app: netchecker-server
       namespace: {{ netcheck_namespace }}
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if netcheck_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
       containers:
         - name: netchecker-server
           image: "{{ server_img }}"

roles/kubernetes-apps/cluster_roles/files/k8s-cluster-critical-pc.yml

@@ -0,0 +1,9 @@
+---
+
+apiVersion: scheduling.k8s.io/v1beta1
+kind: PriorityClass
+metadata:
+  name: k8s-cluster-critical
+value: 1000000000
+globalDefault: false
+description: "This priority class should only be used by the pods installed using kubespray."

roles/kubernetes-apps/cluster_roles/tasks/main.yml

@@ -174,3 +174,20 @@
   when:
     - cloud_provider is defined
     - cloud_provider == 'oci'
+
+- name: PriorityClass | Copy k8s-cluster-critical-pc.yml file
+  copy: src=k8s-cluster-critical-pc.yml dest={{ kube_config_dir }}/k8s-cluster-critical-pc.yml
+  when:
+    - kube_version|version_compare('v1.11.1', '>=')
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: PriorityClass | Create k8s-cluster-critical
+  kube:
+    name: k8s-cluster-critical
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "PriorityClass"
+    filename: "{{ kube_config_dir }}/k8s-cluster-critical-pc.yml"
+    state: latest
+  when:
+    - kube_version|version_compare('v1.11.1', '>=')
+    - inventory_hostname == groups['kube-master'][0]

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/deploy-cephfs-provisioner.yml.j2

@@ -19,6 +19,9 @@ spec:
         app: cephfs-provisioner
         version: {{ cephfs_provisioner_image_tag }}
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if cephfs_provisioner_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
       serviceAccount: cephfs-provisioner
       containers:
         - name: cephfs-provisioner

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2

@@ -18,6 +18,9 @@ spec:
         k8s-app: local-volume-provisioner
         version: {{ local_volume_provisioner_image_tag }}
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if local_volume_provisioner_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
       serviceAccountName: local-volume-provisioner
       tolerations:
         - effect: NoSchedule

roles/kubernetes-apps/ingress_controller/cert_manager/templates/deploy-cert-manager.yml.j2

@@ -22,6 +22,9 @@ spec:
         release: cert-manager
       annotations:
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if cert_manager_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
       serviceAccountName: cert-manager
       containers:
         - name: cert-manager

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2

@@ -19,6 +19,9 @@ spec:
         app.kubernetes.io/name: default-backend
         app.kubernetes.io/part-of: ingress-nginx
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if ingress_nginx_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
       terminationGracePeriodSeconds: 60
       containers:
         - name: default-backend

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2

@@ -29,6 +29,9 @@ spec:
       nodeSelector:
         {{ ingress_nginx_nodeselector | to_nice_yaml }}
 {%- endif %}
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if ingress_nginx_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
       containers:
         - name: ingress-nginx-controller
           image: {{ ingress_nginx_controller_image_repo }}:{{ ingress_nginx_controller_image_tag }}

roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2

@@ -29,6 +29,9 @@ spec:
       tolerations:
         - effect: NoSchedule
           operator: Exists
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
       containers:
         - name: calico-kube-controllers
           image: {{ calico_policy_image_repo }}:{{ calico_policy_image_tag }}

roles/kubernetes-apps/registry/templates/registry-proxy-ds.yml.j2

@@ -21,6 +21,9 @@ spec:
         kubernetes.io/cluster-service: "true"
         version: v{{ registry_proxy_image_tag }}
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if registry_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
       serviceAccountName: registry-proxy
       containers:
         - name: registry-proxy

roles/kubernetes-apps/registry/templates/registry-rs.yml.j2

@@ -22,6 +22,9 @@ spec:
         version: v{{ registry_image_tag }}
         kubernetes.io/cluster-service: "true"
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if registry_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
       serviceAccountName: registry
       containers:
         - name: registry

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -13,6 +13,9 @@ spec:
   hostNetwork: true
 {% if kube_version | version_compare('v1.6', '>=')  %}
   dnsPolicy: ClusterFirst
+{% endif %}
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+  priorityClassName: system-node-critical
 {% endif %}
   containers:
   - name: kube-apiserver

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -12,6 +12,9 @@ spec:
   hostNetwork: true
 {% if kube_version | version_compare('v1.6', '>=') %}
   dnsPolicy: ClusterFirst
+{% endif %}
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+  priorityClassName: system-node-critical
 {% endif %}
   containers:
   - name: kube-controller-manager

roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2

@@ -11,6 +11,9 @@ spec:
   hostNetwork: true
 {% if kube_version | version_compare('v1.6', '>=') %}
   dnsPolicy: ClusterFirst
+{% endif %}
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+  priorityClassName: system-node-critical
 {% endif %}
   containers:
   - name: kube-scheduler

roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2

@@ -15,6 +15,9 @@ spec:
   # When having win nodes in cluster without this patch, this pod cloud try to be created in windows
   nodeSelector:
     beta.kubernetes.io/os: linux
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+  priorityClassName: system-node-critical
+{% endif %}
   containers:
   - name: kube-proxy
     image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}

roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2

@@ -10,6 +10,9 @@ spec:
   # When having win nodes in cluster without this patch, this pod cloud try to be created in windows
   nodeSelector:
     beta.kubernetes.io/os: linux
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+  priorityClassName: system-node-critical
+{% endif %}
   containers:
   - name: nginx-proxy
     image: {{ nginx_image_repo }}:{{ nginx_image_tag }}

roles/kubespray-defaults/defaults/main.yaml

@@ -140,7 +140,7 @@ kube_apiserver_insecure_port: 8080
 dynamic_kubelet_configuration: false
 
 # define kubelet config dir for dynamic kubelet
-#kubelet_config_dir:
+# kubelet_config_dir:
 default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir"
 dynamic_kubelet_configuration_dir: "{{ kubelet_config_dir | default(default_kubelet_config_dir) }}"
 

roles/network_plugin/calico/templates/calico-node.yml.j2

@@ -21,6 +21,9 @@ spec:
         scheduler.alpha.kubernetes.io/critical-pod: ''
         kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
       hostNetwork: true
       serviceAccountName: calico-node
       tolerations:

roles/network_plugin/canal/templates/canal-node.yaml.j2

@@ -18,6 +18,9 @@ spec:
       labels:
         k8s-app: canal-node
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
       hostNetwork: true
       serviceAccountName: canal
       tolerations:

roles/network_plugin/cilium/templates/cilium-ds.yml.j2

@@ -34,6 +34,9 @@ spec:
         prometheus.io/port: "9090"
 {% endif %}
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
       serviceAccountName: cilium
       initContainers:
         - name: clean-cilium-state

roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2

@@ -18,6 +18,9 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
       # The API proxy must run in the host network namespace so that
       # it isn't governed by policy that would prevent it from working.
       hostNetwork: true

roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2

@@ -15,6 +15,9 @@ spec:
       labels:
         k8s-app: contiv-cleanup
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
       hostNetwork: true
       hostPID: true
       tolerations:

roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2

@@ -17,6 +17,9 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
       hostNetwork: true
       hostPID: true
       nodeSelector:

roles/network_plugin/contiv/templates/contiv-etcd.yml.j2

@@ -17,6 +17,9 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
       hostNetwork: true
       hostPID: true
       nodeSelector:

roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2

@@ -18,6 +18,9 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
       # The netmaster must run in the host network namespace so that
       # it isn't governed by policy that would prevent it from working.
       hostNetwork: true

roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2

@@ -22,6 +22,9 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
       hostNetwork: true
       hostPID: true
       tolerations:

roles/network_plugin/contiv/templates/contiv-ovs.yml.j2

@@ -19,6 +19,9 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
       hostNetwork: true
       hostPID: true
       tolerations:

roles/network_plugin/flannel/templates/cni-flannel.yml.j2

@@ -52,6 +52,9 @@ spec:
         tier: node
         k8s-app: flannel
     spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
       serviceAccountName: flannel
       # When having win nodes in cluster without this patch, this pod cloud try to be created in windows
       nodeSelector:

roles/network_plugin/weave/templates/weave-net.yml.j2

@@ -115,6 +115,9 @@ items:
           labels:
             name: weave-net
         spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+          priorityClassName: system-node-critical
+{% endif %}
           containers:
             - name: weave
               command: