Adding pod priority for all the components. (#3361)

* Changes to assign pod priority to kube components. * Removed the boolean flag pod_priority_assignment * Created new priorityclass k8s-cluster-critical * Created new priorityclass k8s-cluster-critical * Fixed the trailing spaces * Fixed the trailing spaces * Added kube version check while creating Priority Class k8s-cluster-critical * Moved k8s-cluster-critical.yml * Moved k8s-cluster-critical.yml to kube_config_dir
2018-09-25 09:50:22 -05:00 · 2018-09-25 09:50:22 -05:00 · 36898a2c39
commit 36898a2c39
parent 8526c30b63
37 changed files with 129 additions and 1 deletions
--- a/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2
+++ b/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2
@ -31,6 +31,9 @@ spec:
        scheduler.alpha.kubernetes.io/critical-pod: ''
        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
      serviceAccountName: dnsmasq
      tolerations:
        - effect: NoSchedule
--- a/roles/dnsmasq/templates/dnsmasq-deploy.yml.j2
+++ b/roles/dnsmasq/templates/dnsmasq-deploy.yml.j2
@ -21,6 +21,9 @@ spec:
        kubernetes.io/cluster-service: "true"
        kubespray/dnsmasq-checksum: "{{ dnsmasq_stat.stat.checksum }}"
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
      tolerations:
        - effect: NoSchedule
          operator: Exists
--- a/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/coredns-deployment.yml.j2
@ -26,6 +26,9 @@ spec:
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
      serviceAccountName: coredns
      tolerations:
        - key: node-role.kubernetes.io/master
--- a/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/dashboard.yml.j2
@ -140,6 +140,9 @@ spec:
      labels:
        k8s-app: kubernetes-dashboard
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
      containers:
      - name: kubernetes-dashboard
        image: {{ dashboard_image_repo }}:{{ dashboard_image_tag }}
--- a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml.j2
@ -28,6 +28,9 @@ spec:
      labels:
        k8s-app: kubedns-autoscaler
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
      # When having win nodes in cluster without this patch, this pod cloud try to be created in windows
      nodeSelector:
        beta.kubernetes.io/os: linux
--- a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2
@ -27,6 +27,9 @@ spec:
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
      # When having win nodes in cluster without this patch, this pod cloud try to be created in windows
      nodeSelector:
        beta.kubernetes.io/os: linux
--- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2
@ -12,6 +12,9 @@ spec:
      labels:
        app: netchecker-agent
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if netcheck_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
      tolerations:
        - effect: NoSchedule
          operator: Exists
--- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2
@ -18,6 +18,9 @@ spec:
        beta.kubernetes.io/os: linux
 {% if kube_version | version_compare('v1.6', '>=') %}
      dnsPolicy: ClusterFirstWithHostNet
+{% endif %}
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if netcheck_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
 {% endif %}
      tolerations:
        - effect: NoSchedule
--- a/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2
@ -11,6 +11,9 @@ spec:
        app: netchecker-server
      namespace: {{ netcheck_namespace }}
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if netcheck_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
      containers:
        - name: netchecker-server
          image: "{{ server_img }}"
--- a/roles/kubernetes-apps/cluster_roles/files/k8s-cluster-critical-pc.yml
+++ b/roles/kubernetes-apps/cluster_roles/files/k8s-cluster-critical-pc.yml
@ -0,0 +1,9 @@
+---
+
+apiVersion: scheduling.k8s.io/v1beta1
+kind: PriorityClass
+metadata:
+  name: k8s-cluster-critical
+value: 1000000000
+globalDefault: false
+description: "This priority class should only be used by the pods installed using kubespray."
--- a/roles/kubernetes-apps/cluster_roles/tasks/main.yml
+++ b/roles/kubernetes-apps/cluster_roles/tasks/main.yml
@ -174,3 +174,20 @@
  when:
    - cloud_provider is defined
    - cloud_provider == 'oci'
+
+- name: PriorityClass | Copy k8s-cluster-critical-pc.yml file
+  copy: src=k8s-cluster-critical-pc.yml dest={{ kube_config_dir }}/k8s-cluster-critical-pc.yml
+  when:
+    - kube_version|version_compare('v1.11.1', '>=')
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: PriorityClass | Create k8s-cluster-critical
+  kube:
+    name: k8s-cluster-critical
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "PriorityClass"
+    filename: "{{ kube_config_dir }}/k8s-cluster-critical-pc.yml"
+    state: latest
+  when:
+    - kube_version|version_compare('v1.11.1', '>=')
+    - inventory_hostname == groups['kube-master'][0]
--- a/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/deploy-cephfs-provisioner.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/deploy-cephfs-provisioner.yml.j2
@ -19,6 +19,9 @@ spec:
        app: cephfs-provisioner
        version: {{ cephfs_provisioner_image_tag }}
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if cephfs_provisioner_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
      serviceAccount: cephfs-provisioner
      containers:
        - name: cephfs-provisioner
--- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2
@ -18,6 +18,9 @@ spec:
        k8s-app: local-volume-provisioner
        version: {{ local_volume_provisioner_image_tag }}
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if local_volume_provisioner_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
      serviceAccountName: local-volume-provisioner
      tolerations:
        - effect: NoSchedule
--- a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/deploy-cert-manager.yml.j2
+++ b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/deploy-cert-manager.yml.j2
@ -22,6 +22,9 @@ spec:
        release: cert-manager
      annotations:
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if cert_manager_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
      serviceAccountName: cert-manager
      containers:
        - name: cert-manager
--- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2
+++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2
@ -19,6 +19,9 @@ spec:
        app.kubernetes.io/name: default-backend
        app.kubernetes.io/part-of: ingress-nginx
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if ingress_nginx_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
      terminationGracePeriodSeconds: 60
      containers:
        - name: default-backend
--- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2
+++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2
@ -29,6 +29,9 @@ spec:
      nodeSelector:
        {{ ingress_nginx_nodeselector | to_nice_yaml }}
 {%- endif %}
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if ingress_nginx_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
      containers:
        - name: ingress-nginx-controller
          image: {{ ingress_nginx_controller_image_repo }}:{{ ingress_nginx_controller_image_tag }}
--- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
+++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
@ -29,6 +29,9 @@ spec:
      tolerations:
        - effect: NoSchedule
          operator: Exists
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-cluster-critical
+{% endif %}
      containers:
        - name: calico-kube-controllers
          image: {{ calico_policy_image_repo }}:{{ calico_policy_image_tag }}
--- a/roles/kubernetes-apps/registry/templates/registry-proxy-ds.yml.j2
+++ b/roles/kubernetes-apps/registry/templates/registry-proxy-ds.yml.j2
@ -21,6 +21,9 @@ spec:
        kubernetes.io/cluster-service: "true"
        version: v{{ registry_proxy_image_tag }}
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if registry_namespace == 'kube-system' %}system-node-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
      serviceAccountName: registry-proxy
      containers:
        - name: registry-proxy
--- a/roles/kubernetes-apps/registry/templates/registry-rs.yml.j2
+++ b/roles/kubernetes-apps/registry/templates/registry-rs.yml.j2
@ -22,6 +22,9 @@ spec:
        version: v{{ registry_image_tag }}
        kubernetes.io/cluster-service: "true"
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: {% if registry_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
      serviceAccountName: registry
      containers:
        - name: registry
--- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
@ -13,6 +13,9 @@ spec:
  hostNetwork: true
 {% if kube_version | version_compare('v1.6', '>=')  %}
  dnsPolicy: ClusterFirst
+{% endif %}
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+  priorityClassName: system-node-critical
 {% endif %}
  containers:
  - name: kube-apiserver
--- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
@ -12,6 +12,9 @@ spec:
  hostNetwork: true
 {% if kube_version | version_compare('v1.6', '>=') %}
  dnsPolicy: ClusterFirst
+{% endif %}
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+  priorityClassName: system-node-critical
 {% endif %}
  containers:
  - name: kube-controller-manager
--- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
@ -11,6 +11,9 @@ spec:
  hostNetwork: true
 {% if kube_version | version_compare('v1.6', '>=') %}
  dnsPolicy: ClusterFirst
+{% endif %}
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+  priorityClassName: system-node-critical
 {% endif %}
  containers:
  - name: kube-scheduler
--- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
+++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
@ -15,6 +15,9 @@ spec:
  # When having win nodes in cluster without this patch, this pod cloud try to be created in windows
  nodeSelector:
    beta.kubernetes.io/os: linux
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+  priorityClassName: system-node-critical
+{% endif %}
  containers:
  - name: kube-proxy
    image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
--- a/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2
+++ b/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2
@ -10,6 +10,9 @@ spec:
  # When having win nodes in cluster without this patch, this pod cloud try to be created in windows
  nodeSelector:
    beta.kubernetes.io/os: linux
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+  priorityClassName: system-node-critical
+{% endif %}
  containers:
  - name: nginx-proxy
    image: {{ nginx_image_repo }}:{{ nginx_image_tag }}
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@ -140,7 +140,7 @@ kube_apiserver_insecure_port: 8080
 dynamic_kubelet_configuration: false

 # define kubelet config dir for dynamic kubelet
-#kubelet_config_dir:
+# kubelet_config_dir:
 default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir"
 dynamic_kubelet_configuration_dir: "{{ kubelet_config_dir | default(default_kubelet_config_dir) }}"

--- a/roles/network_plugin/calico/templates/calico-node.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-node.yml.j2
@ -21,6 +21,9 @@ spec:
        scheduler.alpha.kubernetes.io/critical-pod: ''
        kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
      hostNetwork: true
      serviceAccountName: calico-node
      tolerations:
--- a/roles/network_plugin/canal/templates/canal-node.yaml.j2
+++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2
@ -18,6 +18,9 @@ spec:
      labels:
        k8s-app: canal-node
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
      hostNetwork: true
      serviceAccountName: canal
      tolerations:
--- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
@ -34,6 +34,9 @@ spec:
        prometheus.io/port: "9090"
 {% endif %}
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
      serviceAccountName: cilium
      initContainers:
        - name: clean-cilium-state
--- a/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2
@ -18,6 +18,9 @@ spec:
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
      # The API proxy must run in the host network namespace so that
      # it isn't governed by policy that would prevent it from working.
      hostNetwork: true
--- a/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2
@ -15,6 +15,9 @@ spec:
      labels:
        k8s-app: contiv-cleanup
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
      hostNetwork: true
      hostPID: true
      tolerations:
--- a/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2
@ -17,6 +17,9 @@ spec:
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
      hostNetwork: true
      hostPID: true
      nodeSelector:
--- a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2
@ -17,6 +17,9 @@ spec:
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
      hostNetwork: true
      hostPID: true
      nodeSelector:
--- a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2
@ -18,6 +18,9 @@ spec:
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
      # The netmaster must run in the host network namespace so that
      # it isn't governed by policy that would prevent it from working.
      hostNetwork: true
--- a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2
@ -22,6 +22,9 @@ spec:
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
      hostNetwork: true
      hostPID: true
      tolerations:
--- a/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2
@ -19,6 +19,9 @@ spec:
      annotations:
        scheduler.alpha.kubernetes.io/critical-pod: ''
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
      hostNetwork: true
      hostPID: true
      tolerations:
--- a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2
+++ b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2
@ -52,6 +52,9 @@ spec:
        tier: node
        k8s-app: flannel
    spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+      priorityClassName: system-node-critical
+{% endif %}
      serviceAccountName: flannel
      # When having win nodes in cluster without this patch, this pod cloud try to be created in windows
      nodeSelector:
--- a/roles/network_plugin/weave/templates/weave-net.yml.j2
+++ b/roles/network_plugin/weave/templates/weave-net.yml.j2
@ -115,6 +115,9 @@ items:
          labels:
            name: weave-net
        spec:
+{% if kube_version|version_compare('v1.11.1', '>=') %}
+          priorityClassName: system-node-critical
+{% endif %}
          containers:
            - name: weave
              command: