Add Kubelet config, remove deprecated flags and fix minor bugs (#4724)
* Add kubelet config * Change kubelet_authorization_mode_webhook to true * Fix lint * Sync env file * Refactor the kubernetes node folder * Remove deprecated flag and fix lint
This commit is contained in:
parent
8a5eae94ea
commit
044dcbaed0
22 changed files with 221 additions and 43 deletions
|
@ -62,7 +62,7 @@
|
||||||
--client-name kubernetes-admin
|
--client-name kubernetes-admin
|
||||||
--org system:masters
|
--org system:masters
|
||||||
{% endif %}
|
{% endif %}
|
||||||
--cert-dir {{ kube_config_dir }}/ssl
|
--cert-dir {{ kube_cert_dir }}
|
||||||
--apiserver-advertise-address {{ external_apiserver_address }}
|
--apiserver-advertise-address {{ external_apiserver_address }}
|
||||||
--apiserver-bind-port {{ external_apiserver_port }}
|
--apiserver-bind-port {{ external_apiserver_port }}
|
||||||
{% if kubeadm_version is version('v1.14.0', '>=') %}
|
{% if kubeadm_version is version('v1.14.0', '>=') %}
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
apiVersion: kubeadm.k8s.io/v1beta1
|
apiVersion: kubeadm.k8s.io/v1beta1
|
||||||
kind: JoinConfiguration
|
kind: JoinConfiguration
|
||||||
discovery:
|
discovery:
|
||||||
|
@ -8,7 +9,8 @@ discovery:
|
||||||
apiServerEndpoint: {{ kubeadm_discovery_address | replace("https://", "")}}
|
apiServerEndpoint: {{ kubeadm_discovery_address | replace("https://", "")}}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
token: {{ kubeadm_token }}
|
token: {{ kubeadm_token }}
|
||||||
unsafeSkipCAVerification: true
|
caCertHashes:
|
||||||
|
- sha256:{{ kubeadm_ca_hash.stdout }}
|
||||||
timeout: {{ discovery_timeout }}
|
timeout: {{ discovery_timeout }}
|
||||||
tlsBootstrapToken: {{ kubeadm_token }}
|
tlsBootstrapToken: {{ kubeadm_token }}
|
||||||
caCertPath: {{ kube_cert_dir }}/ca.crt
|
caCertPath: {{ kube_cert_dir }}/ca.crt
|
||||||
|
|
|
@ -73,10 +73,3 @@
|
||||||
args:
|
args:
|
||||||
creates: "{{ bin_dir }}/socat"
|
creates: "{{ bin_dir }}/socat"
|
||||||
when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
|
when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
|
||||||
|
|
||||||
- name: install | Write kubelet systemd init file
|
|
||||||
template:
|
|
||||||
src: "kubelet.host.service.j2"
|
|
||||||
dest: "/etc/systemd/system/kubelet.service"
|
|
||||||
backup: "yes"
|
|
||||||
notify: restart kubelet
|
|
||||||
|
|
71
roles/kubernetes/node/tasks/kubelet.yml
Normal file
71
roles/kubernetes/node/tasks/kubelet.yml
Normal file
|
@ -0,0 +1,71 @@
|
||||||
|
---
|
||||||
|
- name: Make sure dynamic kubelet configuration directory is writeable
|
||||||
|
file:
|
||||||
|
path: "{{ dynamic_kubelet_configuration_dir }}"
|
||||||
|
mode: 0600
|
||||||
|
state: directory
|
||||||
|
when: dynamic_kubelet_configuration
|
||||||
|
|
||||||
|
- name: gets the kubeadm version
|
||||||
|
command: "{{ bin_dir }}/kubeadm version -o short"
|
||||||
|
register: kubeadm_output
|
||||||
|
|
||||||
|
- name: sets kubeadm api version to v1beta1
|
||||||
|
set_fact:
|
||||||
|
kubeadmConfig_api_version: v1beta1
|
||||||
|
when: kubeadm_output.stdout is version('v1.13.0', '>=')
|
||||||
|
tags:
|
||||||
|
- kubelet
|
||||||
|
- kubeadm
|
||||||
|
|
||||||
|
- name: Write kubelet environment config file (kubeadm)
|
||||||
|
template:
|
||||||
|
src: "kubelet.env.{{ kubeadmConfig_api_version }}.j2"
|
||||||
|
dest: "{{ kube_config_dir }}/kubelet.env"
|
||||||
|
backup: yes
|
||||||
|
notify: restart kubelet
|
||||||
|
when: kubeadm_output.stdout is version('v1.13.0', '>=')
|
||||||
|
tags:
|
||||||
|
- kubelet
|
||||||
|
- kubeadm
|
||||||
|
|
||||||
|
- name: Write kubelet environment config file (kubeadm)
|
||||||
|
template:
|
||||||
|
src: "kubelet.env.j2"
|
||||||
|
dest: "{{ kube_config_dir }}/kubelet.env"
|
||||||
|
backup: yes
|
||||||
|
notify: restart kubelet
|
||||||
|
when: kubeadm_output.stdout is version('v1.13.0', '<')
|
||||||
|
tags:
|
||||||
|
- kubelet
|
||||||
|
- kubeadm
|
||||||
|
|
||||||
|
- name: Write kubelet config file
|
||||||
|
template:
|
||||||
|
src: "kubelet-config.{{ kubeadmConfig_api_version }}.yaml.j2"
|
||||||
|
dest: "{{ kube_config_dir }}/kubelet-config.yaml"
|
||||||
|
when: kubeadm_output.stdout is version('v1.13.0', '>=')
|
||||||
|
tags:
|
||||||
|
- kubelet
|
||||||
|
- kubeadm
|
||||||
|
|
||||||
|
- name: Write kubelet systemd init file
|
||||||
|
template:
|
||||||
|
src: "kubelet.service.j2"
|
||||||
|
dest: "/etc/systemd/system/kubelet.service"
|
||||||
|
backup: "yes"
|
||||||
|
notify: restart kubelet
|
||||||
|
tags:
|
||||||
|
- kubelet
|
||||||
|
- kubeadm
|
||||||
|
|
||||||
|
- name: flush_handlers and reload-systemd
|
||||||
|
meta: flush_handlers
|
||||||
|
|
||||||
|
- name: Enable kubelet
|
||||||
|
service:
|
||||||
|
name: kubelet
|
||||||
|
enabled: yes
|
||||||
|
state: started
|
||||||
|
tags:
|
||||||
|
- kubelet
|
|
@ -18,7 +18,7 @@
|
||||||
|
|
||||||
- name: haproxy | Write haproxy configuration
|
- name: haproxy | Write haproxy configuration
|
||||||
template:
|
template:
|
||||||
src: haproxy.cfg.j2
|
src: "loadbalancer/haproxy.cfg.j2"
|
||||||
dest: "{{ haproxy_config_dir }}/haproxy.cfg"
|
dest: "{{ haproxy_config_dir }}/haproxy.cfg"
|
||||||
owner: root
|
owner: root
|
||||||
mode: 0755
|
mode: 0755
|
|
@ -18,7 +18,7 @@
|
||||||
|
|
||||||
- name: nginx-proxy | Write nginx-proxy configuration
|
- name: nginx-proxy | Write nginx-proxy configuration
|
||||||
template:
|
template:
|
||||||
src: nginx.conf.j2
|
src: "loadbalancer/nginx.conf.j2"
|
||||||
dest: "{{ nginx_config_dir }}/nginx.conf"
|
dest: "{{ nginx_config_dir }}/nginx.conf"
|
||||||
owner: root
|
owner: root
|
||||||
mode: 0755
|
mode: 0755
|
|
@ -17,7 +17,7 @@
|
||||||
tags:
|
tags:
|
||||||
- kubelet
|
- kubelet
|
||||||
|
|
||||||
- import_tasks: nginx-proxy.yml
|
- import_tasks: loadbalancer/nginx-proxy.yml
|
||||||
when:
|
when:
|
||||||
- not is_kube_master
|
- not is_kube_master
|
||||||
- loadbalancer_apiserver_localhost
|
- loadbalancer_apiserver_localhost
|
||||||
|
@ -25,7 +25,7 @@
|
||||||
tags:
|
tags:
|
||||||
- nginx
|
- nginx
|
||||||
|
|
||||||
- import_tasks: haproxy.yml
|
- import_tasks: loadbalancer/haproxy.yml
|
||||||
when:
|
when:
|
||||||
- not is_kube_master
|
- not is_kube_master
|
||||||
- loadbalancer_apiserver_localhost
|
- loadbalancer_apiserver_localhost
|
||||||
|
@ -33,23 +33,6 @@
|
||||||
tags:
|
tags:
|
||||||
- haproxy
|
- haproxy
|
||||||
|
|
||||||
- name: Make sure dynamic kubelet configuration directory is writeable
|
|
||||||
file:
|
|
||||||
path: "{{ dynamic_kubelet_configuration_dir }}"
|
|
||||||
mode: 0600
|
|
||||||
state: directory
|
|
||||||
when: dynamic_kubelet_configuration
|
|
||||||
|
|
||||||
- name: Write kubelet config file (kubeadm)
|
|
||||||
template:
|
|
||||||
src: kubelet.kubeadm.env.j2
|
|
||||||
dest: "{{ kube_config_dir }}/kubelet.env"
|
|
||||||
backup: yes
|
|
||||||
notify: restart kubelet
|
|
||||||
tags:
|
|
||||||
- kubelet
|
|
||||||
- kubeadm
|
|
||||||
|
|
||||||
- name: Ensure nodePort range is reserved
|
- name: Ensure nodePort range is reserved
|
||||||
sysctl:
|
sysctl:
|
||||||
name: net.ipv4.ip_local_reserved_ports
|
name: net.ipv4.ip_local_reserved_ports
|
||||||
|
@ -99,7 +82,7 @@
|
||||||
name: "{{ item }}"
|
name: "{{ item }}"
|
||||||
state: present
|
state: present
|
||||||
sysctl_file: "{{ sysctl_file_path }}"
|
sysctl_file: "{{ sysctl_file_path }}"
|
||||||
value: 1
|
value: "1"
|
||||||
reload: yes
|
reload: yes
|
||||||
when: sysctl_bridge_nf_call_iptables.rc == 0
|
when: sysctl_bridge_nf_call_iptables.rc == 0
|
||||||
with_items:
|
with_items:
|
||||||
|
@ -175,7 +158,7 @@
|
||||||
tags:
|
tags:
|
||||||
- kube-proxy
|
- kube-proxy
|
||||||
|
|
||||||
- include_tasks: "{{ cloud_provider }}-credential-check.yml"
|
- include_tasks: "cloud-credentials/{{ cloud_provider }}-credential-check.yml"
|
||||||
when:
|
when:
|
||||||
- cloud_provider is defined
|
- cloud_provider is defined
|
||||||
- cloud_provider in [ 'openstack', 'azure', 'vsphere' ]
|
- cloud_provider in [ 'openstack', 'azure', 'vsphere' ]
|
||||||
|
@ -200,7 +183,7 @@
|
||||||
|
|
||||||
- name: Write cloud-config
|
- name: Write cloud-config
|
||||||
template:
|
template:
|
||||||
src: "{{ cloud_provider }}-cloud-config.j2"
|
src: "cloud-configs/{{ cloud_provider }}-cloud-config.j2"
|
||||||
dest: "{{ kube_config_dir }}/cloud_config"
|
dest: "{{ kube_config_dir }}/cloud_config"
|
||||||
group: "{{ kube_cert_group }}"
|
group: "{{ kube_cert_group }}"
|
||||||
mode: 0640
|
mode: 0640
|
||||||
|
@ -211,13 +194,7 @@
|
||||||
tags:
|
tags:
|
||||||
- cloud-provider
|
- cloud-provider
|
||||||
|
|
||||||
# reload-systemd
|
- import_tasks: kubelet.yml
|
||||||
- meta: flush_handlers
|
|
||||||
|
|
||||||
- name: Enable kubelet
|
|
||||||
service:
|
|
||||||
name: kubelet
|
|
||||||
enabled: yes
|
|
||||||
state: started
|
|
||||||
tags:
|
tags:
|
||||||
- kubelet
|
- kubelet
|
||||||
|
- kubeadm
|
|
@ -0,0 +1,57 @@
|
||||||
|
apiVersion: kubelet.config.k8s.io/v1beta1
|
||||||
|
kind: KubeletConfiguration
|
||||||
|
nodeStatusUpdateFrequency: "{{ kubelet_status_update_frequency }}"
|
||||||
|
failSwapOn: {{ kubelet_fail_swap_on|default(true) }}
|
||||||
|
authentication:
|
||||||
|
anonymous:
|
||||||
|
enabled: false
|
||||||
|
webhook:
|
||||||
|
enabled: {{ kubelet_authentication_token_webhook }}
|
||||||
|
x509:
|
||||||
|
clientCAFile: {{ kube_cert_dir }}/ca.crt
|
||||||
|
authorization:
|
||||||
|
{% if kubelet_authorization_mode_webhook %}
|
||||||
|
mode: Webhook
|
||||||
|
{% else %}
|
||||||
|
mode: AlwaysAllow
|
||||||
|
{% endif %}
|
||||||
|
{% if kubelet_enforce_node_allocatable is defined and kubelet_enforce_node_allocatable != "\"\"" %}
|
||||||
|
{% set kubelet_enforce_node_allocatable_list = kubelet_enforce_node_allocatable.split() %}
|
||||||
|
enforceNodeAllocatable:
|
||||||
|
{% for item in kubelet_enforce_node_allocatable_list %}
|
||||||
|
- {{ item }}
|
||||||
|
{% endfor %}
|
||||||
|
{% endif %}
|
||||||
|
staticPodPath: {{ kube_manifest_dir }}
|
||||||
|
cgroupDriver: {{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }}
|
||||||
|
maxPods: {{ kubelet_max_pods }}
|
||||||
|
address: {{ kubelet_bind_address }}
|
||||||
|
readOnlyPort: {{ kube_read_only_port }}
|
||||||
|
kubeletCgroups: {{ kubelet_kubelet_cgroups }}
|
||||||
|
clusterDomain: {{ dns_domain }}
|
||||||
|
{# DNS settings for kubelet #}
|
||||||
|
{% if enable_nodelocaldns %}
|
||||||
|
{% set kubelet_cluster_dns = [nodelocaldns_ip] %}
|
||||||
|
{% elif dns_mode in ['coredns'] %}
|
||||||
|
{% set kubelet_cluster_dns = [skydns_server] %}
|
||||||
|
{% elif dns_mode == 'coredns_dual' %}
|
||||||
|
{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %}
|
||||||
|
{% elif dns_mode == 'manual' %}
|
||||||
|
{% set kubelet_cluster_dns = [manual_dns_server] %}
|
||||||
|
{% else %}
|
||||||
|
{% set kubelet_cluster_dns = [] %}
|
||||||
|
{% endif %}
|
||||||
|
clusterDNS:
|
||||||
|
{% for dns_address in kubelet_cluster_dns %}
|
||||||
|
- {{ dns_address }}
|
||||||
|
{% endfor %}
|
||||||
|
{# Node reserved CPU/memory #}
|
||||||
|
kubeReserved:
|
||||||
|
{% if is_kube_master|bool %}
|
||||||
|
cpu: {{ kube_master_cpu_reserved }}
|
||||||
|
memory: {{ kube_master_memory_reserved|regex_replace('Mi', 'M') }}
|
||||||
|
{% else %}
|
||||||
|
cpu: {{ kube_cpu_reserved }}
|
||||||
|
memory: {{ kube_memory_reserved|regex_replace('Mi', 'M') }}
|
||||||
|
{% endif %}
|
||||||
|
resolvConf: "{{ kube_resolv_conf }}"
|
78
roles/kubernetes/node/templates/kubelet.env.v1beta1.j2
Normal file
78
roles/kubernetes/node/templates/kubelet.env.v1beta1.j2
Normal file
|
@ -0,0 +1,78 @@
|
||||||
|
KUBE_LOGTOSTDERR="--logtostderr=true"
|
||||||
|
KUBE_LOG_LEVEL="--v={{ kube_log_level }}"
|
||||||
|
KUBELET_ADDRESS="--node-ip={{ kubelet_address }}"
|
||||||
|
{% if kube_override_hostname|default('') %}
|
||||||
|
KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{# Base kubelet args #}
|
||||||
|
{% set kubelet_args_base -%}
|
||||||
|
{# start kubeadm specific settings #}
|
||||||
|
--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
|
||||||
|
--config={{ kube_config_dir }}/kubelet-config.yaml \
|
||||||
|
--kubeconfig={{ kube_config_dir }}/kubelet.conf \
|
||||||
|
{% if kube_version is version('v1.8', '<') %}
|
||||||
|
--require-kubeconfig \
|
||||||
|
{% endif %}
|
||||||
|
{% if kube_version is version('v1.12.0', '<') %}
|
||||||
|
--cadvisor-port={{ kube_cadvisor_port }} \
|
||||||
|
{% endif %}
|
||||||
|
{# end kubeadm specific settings #}
|
||||||
|
--pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
|
||||||
|
{% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %}
|
||||||
|
--docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
|
||||||
|
{% endif %}
|
||||||
|
{% if container_manager == 'crio' %}
|
||||||
|
--container-runtime=remote \
|
||||||
|
--container-runtime-endpoint=/var/run/crio/crio.sock \
|
||||||
|
{% endif %}
|
||||||
|
{% if kube_version is version('v1.8', '<') %}
|
||||||
|
--experimental-fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \
|
||||||
|
{% endif %}
|
||||||
|
{% if dynamic_kubelet_configuration %}
|
||||||
|
--dynamic-config-dir={{ dynamic_kubelet_configuration_dir }} \
|
||||||
|
{% endif %}
|
||||||
|
--runtime-cgroups={{ kubelet_runtime_cgroups }} \
|
||||||
|
{% endset %}
|
||||||
|
|
||||||
|
{# Kubelet node labels #}
|
||||||
|
{% set role_node_labels = [] %}
|
||||||
|
{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
|
||||||
|
{% if inventory_hostname in nvidia_gpu_nodes %}
|
||||||
|
{% set dummy = role_node_labels.append('nvidia.com/gpu=true') %}
|
||||||
|
{% endif %}
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
{% set inventory_node_labels = [] %}
|
||||||
|
{% if node_labels is defined and node_labels is mapping %}
|
||||||
|
{% for labelname, labelvalue in node_labels.items() %}
|
||||||
|
{% set dummy = inventory_node_labels.append('%s=%s'|format(labelname, labelvalue)) %}
|
||||||
|
{% endfor %}
|
||||||
|
{% endif %}
|
||||||
|
{% set all_node_labels = role_node_labels + inventory_node_labels %}
|
||||||
|
|
||||||
|
{# Kubelet node taints for gpu #}
|
||||||
|
{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
|
||||||
|
{% if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %}
|
||||||
|
{% set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
|
||||||
|
{% elif inventory_hostname in nvidia_gpu_nodes and node_taints is not defined %}
|
||||||
|
{% set node_taints = [] %}
|
||||||
|
{% set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
|
||||||
|
{% endif %}
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
|
||||||
|
{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router"] %}
|
||||||
|
KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
|
||||||
|
{% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}
|
||||||
|
KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"
|
||||||
|
{% endif %}
|
||||||
|
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
|
||||||
|
KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"
|
||||||
|
{% elif cloud_provider is defined and cloud_provider in ["external"] %}
|
||||||
|
KUBELET_CLOUDPROVIDER="--cloud-provider=external --cloud-config={{ kube_config_dir }}/cloud_config"
|
||||||
|
{% else %}
|
||||||
|
KUBELET_CLOUDPROVIDER=""
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
|
@ -15,7 +15,6 @@ ExecStart={{ bin_dir }}/kubelet \
|
||||||
$KUBELET_ADDRESS \
|
$KUBELET_ADDRESS \
|
||||||
$KUBELET_PORT \
|
$KUBELET_PORT \
|
||||||
$KUBELET_HOSTNAME \
|
$KUBELET_HOSTNAME \
|
||||||
$KUBE_ALLOW_PRIV \
|
|
||||||
$KUBELET_ARGS \
|
$KUBELET_ARGS \
|
||||||
$DOCKER_SOCKET \
|
$DOCKER_SOCKET \
|
||||||
$KUBELET_NETWORK_PLUGIN \
|
$KUBELET_NETWORK_PLUGIN \
|
|
@ -1,3 +1,4 @@
|
||||||
|
---
|
||||||
apiVersion: v1
|
apiVersion: v1
|
||||||
kind: Config
|
kind: Config
|
||||||
clusters:
|
clusters:
|
||||||
|
|
|
@ -314,7 +314,7 @@ rbac_enabled: "{{ 'RBAC' in authorization_modes }}"
|
||||||
kubelet_authentication_token_webhook: true
|
kubelet_authentication_token_webhook: true
|
||||||
|
|
||||||
# When enabled, access to the kubelet API requires authorization by delegation to the API server
|
# When enabled, access to the kubelet API requires authorization by delegation to the API server
|
||||||
kubelet_authorization_mode_webhook: false
|
kubelet_authorization_mode_webhook: true
|
||||||
|
|
||||||
# kubelet uses certificates for authenticating to the Kubernetes API
|
# kubelet uses certificates for authenticating to the Kubernetes API
|
||||||
# Automatically generate a new key and request a new certificate from the Kubernetes API as the current certificate approaches expiration
|
# Automatically generate a new key and request a new certificate from the Kubernetes API as the current certificate approaches expiration
|
||||||
|
|
Loading…
Reference in a new issue