set node label via kubectl label command (#5257)

* set varios node label via kubectl label command, not kubelet options

* remove node_labels from KUBELET_ARGS

cluster.yml

@@ -78,6 +78,7 @@
     - { role: kubespray-defaults}
     - { role: kubernetes/kubeadm, tags: kubeadm}
     - { role: network_plugin, tags: network }
+    - { role: kubernetes/node-label }
 
 - hosts: calico-rr
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"

roles/kubernetes/node-label/tasks/main.yml

@@ -0,0 +1,47 @@
+---
+- name: Kubernetes Apps | Wait for kube-apiserver
+  uri:
+    url: "{{ kube_apiserver_endpoint }}/healthz"
+    validate_certs: no
+    client_cert: "{{ kube_apiserver_client_cert }}"
+    client_key: "{{ kube_apiserver_client_key }}"
+  register: result
+  until: result.status == 200
+  retries: 10
+  delay: 6
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: Set role node label to empty list
+  set_fact:
+    role_node_labels: []
+
+- name: Node label for nvidia GPU nodes
+  set_fact:
+    role_node_labels: "{{ role_node_labels + [ 'nvidia.com/gpu=true' ] }}"
+  when:
+    - nvidia_gpu_nodes is defined
+    - nvidia_accelerator_enabled|bool
+    - inventory_hostname in nvidia_gpu_nodes
+
+- name: Set inventory node label to empty list
+  set_fact:
+    inventory_node_labels: []
+
+- name: Populate inventory node label
+  set_fact:
+    inventory_node_labels: "{{ inventory_node_labels + [ '%s=%s'|format(item.key, item.value) ] }}"
+  loop: "{{ node_labels|d({})|dict2items }}"
+  when:
+    - node_labels is defined
+    - node_labels is mapping
+
+- debug: var=role_node_labels
+- debug: var=inventory_node_labels
+
+- name: Set label to node
+  command: >-
+     {{ bin_dir }}/kubectl label node {{ inventory_hostname }} {{ item }} --overwrite=true
+  loop: "{{ role_node_labels + inventory_node_labels }}"
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  changed_when: false
+...

roles/kubernetes/node/templates/kubelet.env.v1beta1.j2

@@ -23,22 +23,6 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 --runtime-cgroups={{ kubelet_runtime_cgroups }} \
 {% endset %}
 
-{# Kubelet node labels #}
-{% set role_node_labels = [] %}
-{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
-{%   if inventory_hostname in nvidia_gpu_nodes %}
-{%     set dummy = role_node_labels.append('nvidia.com/gpu=true')  %}
-{%   endif %}
-{% endif %}
-
-{% set inventory_node_labels = [] %}
-{% if node_labels is defined and node_labels is mapping %}
-{%   for labelname, labelvalue in node_labels.items() %}
-{%     set dummy = inventory_node_labels.append('%s=%s'|format(labelname, labelvalue)) %}
-{%   endfor %}
-{% endif %}
-{% set all_node_labels = role_node_labels + inventory_node_labels %}
-
 {# Kubelet node taints for gpu #}
 {% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
 {%   if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %}
@@ -49,7 +33,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {%   endif %}
 {% endif %}
 
-KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
+KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
 {% if kubelet_flexvolumes_plugins_dir is defined %}
 KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}"
 {% endif %}

scale.yml

@@ -46,4 +46,5 @@
     - { role: kubernetes/node, tags: node }
     - { role: kubernetes/kubeadm, tags: kubeadm }
     - { role: network_plugin, tags: network }
+    - { role: kubernetes/node-label }
   environment: "{{ proxy_env }}"

upgrade-cluster.yml

@@ -72,6 +72,7 @@
     - { role: kubernetes/node, tags: node }
     - { role: kubernetes/master, tags: master, upgrade_cluster_setup: true }
     - { role: kubernetes/client, tags: client }
+    - { role: kubernetes/node-label }
     - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
     - { role: upgrade/post-upgrade, tags: post-upgrade }
   environment: "{{ proxy_env }}"
@@ -95,6 +96,7 @@
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
     - { role: kubernetes/node, tags: node }
     - { role: kubernetes/kubeadm, tags: kubeadm }
+    - { role: kubernetes/node-label }
     - { role: upgrade/post-upgrade, tags: post-upgrade }
   environment: "{{ proxy_env }}"