From 8331f7b056217e038b7738f1d976cd4a3acaf079 Mon Sep 17 00:00:00 2001 From: Michal Belica Date: Wed, 28 Nov 2018 00:56:49 +0100 Subject: [PATCH] Add support for setting custom node taints (#3774) Introduced variable node_taints which can be set in inventory for specific hosts or in group_vars, which generates --register-with-taints command line argument for kubelet. --- docs/vars.md | 7 +++++++ roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 | 2 +- roles/kubernetes/node/templates/kubelet.standard.env.j2 | 6 ++++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/vars.md b/docs/vars.md index 84340dfab..31aa5f85e 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -126,6 +126,13 @@ node_labels: label1_name: label1_value label2_name: label2_value ``` +* *node_taints* - Taints applied to nodes via kubelet --register-with-taints parameter. + For example, taints can be set in the inventory as variables or more widely in group_vars. + *node_taints* has to be defined as a list of strings in format `key=value:effect`, e.g.: +``` +node_taints: + - "node.example.com/external=true:NoSchedule" +``` * *podsecuritypolicy_enabled* - When set to `true`, enables the PodSecurityPolicy admission controller and defines two policies `privileged` (applying to all resources in `kube-system` namespace and kubelet) and `restricted` (applying all other namespaces). Addons deployed in kube-system namespaces are handled. * *kubernetes_audit* - When set to `true`, enables Auditing. diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index e8e5e7ed5..963a87b43 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -96,7 +96,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} {% set all_node_labels = role_node_labels + inventory_node_labels %} -KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" +KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium", "kube-router"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %} diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2 index 03cf28bc0..fa6d6c3f3 100644 --- a/roles/kubernetes/node/templates/kubelet.standard.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2 @@ -74,16 +74,18 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% set kubelet_args_kubeconfig %}--kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml{% endset %} {% endif %} +{% set role_node_taints = [] %} {% if standalone_kubelet|bool %} {# We are on a master-only host. Make the master unschedulable in this case. #} {% if kube_version is version('v1.6', '>=') %} {# Set taints on the master so that it's unschedulable by default. Use node-role.kubernetes.io/master taint like kubeadm. #} -{% set kubelet_args_kubeconfig %}{{ kubelet_args_kubeconfig }} --register-with-taints=node-role.kubernetes.io/master=:NoSchedule{% endset %} +{% set dummy = role_node_taints.append('node-role.kubernetes.io/master=:NoSchedule') %} {% else %} {# --register-with-taints was added in 1.6 so just register unschedulable if Kubernetes < 1.6 #} {% set kubelet_args_kubeconfig %}{{ kubelet_args_kubeconfig }} --register-schedulable=false{% endset %} {% endif %} {% endif %} +{% set all_node_taints = node_taints|default([]) + role_node_taints %} {# Node reserved CPU/memory #} {% if is_kube_master|bool %} @@ -122,7 +124,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} {% endif %} -KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" +KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} {% if all_node_taints %}--register-with-taints={{ all_node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium", "kube-router"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"