Add Kubelet config, remove deprecated flags and fix minor bugs (#4724)

* Add kubelet config

* Change kubelet_authorization_mode_webhook to true

* Fix lint

* Sync env file

* Refactor the kubernetes node folder

* Remove deprecated flag and fix lint

roles/kubernetes/client/tasks/main.yml

@@ -62,7 +62,7 @@
     --client-name kubernetes-admin
     --org system:masters
     {% endif %}
-    --cert-dir {{ kube_config_dir }}/ssl
+    --cert-dir {{ kube_cert_dir }}
     --apiserver-advertise-address {{ external_apiserver_address }}
     --apiserver-bind-port {{ external_apiserver_port }}
     {% if kubeadm_version is version('v1.14.0', '>=') %}

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2

@@ -1,3 +1,4 @@
+---
 apiVersion: kubeadm.k8s.io/v1beta1
 kind: JoinConfiguration
 discovery:
@@ -8,7 +9,8 @@ discovery:
     apiServerEndpoint: {{ kubeadm_discovery_address | replace("https://", "")}}
 {% endif %}
     token: {{ kubeadm_token }}
-    unsafeSkipCAVerification: true
+    caCertHashes:
+    - sha256:{{ kubeadm_ca_hash.stdout }}
   timeout: {{ discovery_timeout }}
   tlsBootstrapToken: {{ kubeadm_token }}
 caCertPath: {{ kube_cert_dir }}/ca.crt

roles/kubernetes/node/tasks/install.yml

@@ -73,10 +73,3 @@
   args:
     creates: "{{ bin_dir }}/socat"
   when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
-
-- name: install | Write kubelet systemd init file
-  template:
-    src: "kubelet.host.service.j2"
-    dest: "/etc/systemd/system/kubelet.service"
-    backup: "yes"
-  notify: restart kubelet

roles/kubernetes/node/tasks/kubelet.yml

@@ -0,0 +1,71 @@
+---
+- name: Make sure dynamic kubelet configuration directory is writeable
+  file:
+    path: "{{ dynamic_kubelet_configuration_dir }}"
+    mode: 0600
+    state: directory
+  when: dynamic_kubelet_configuration
+
+- name: gets the kubeadm version
+  command: "{{ bin_dir }}/kubeadm version -o short"
+  register: kubeadm_output
+
+- name: sets kubeadm api version to v1beta1
+  set_fact:
+    kubeadmConfig_api_version: v1beta1
+  when: kubeadm_output.stdout is version('v1.13.0', '>=')
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet environment config file (kubeadm)
+  template:
+    src: "kubelet.env.{{ kubeadmConfig_api_version }}.j2"
+    dest: "{{ kube_config_dir }}/kubelet.env"
+    backup: yes
+  notify: restart kubelet
+  when: kubeadm_output.stdout is version('v1.13.0', '>=')
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet environment config file (kubeadm)
+  template:
+    src: "kubelet.env.j2"
+    dest: "{{ kube_config_dir }}/kubelet.env"
+    backup: yes
+  notify: restart kubelet
+  when: kubeadm_output.stdout is version('v1.13.0', '<')
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet config file
+  template:
+    src: "kubelet-config.{{ kubeadmConfig_api_version }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubelet-config.yaml"
+  when: kubeadm_output.stdout is version('v1.13.0', '>=')
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: Write kubelet systemd init file
+  template:
+    src: "kubelet.service.j2"
+    dest: "/etc/systemd/system/kubelet.service"
+    backup: "yes"
+  notify: restart kubelet
+  tags:
+    - kubelet
+    - kubeadm
+
+- name: flush_handlers and reload-systemd
+  meta: flush_handlers
+
+- name: Enable kubelet
+  service:
+    name: kubelet
+    enabled: yes
+    state: started
+  tags:
+    - kubelet

roles/kubernetes/node/tasks/loadbalancer/haproxy.yml

@@ -18,7 +18,7 @@
 
 - name: haproxy | Write haproxy configuration
   template:
-    src: haproxy.cfg.j2
+    src: "loadbalancer/haproxy.cfg.j2"
     dest: "{{ haproxy_config_dir }}/haproxy.cfg"
     owner: root
     mode: 0755

roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml

@@ -18,7 +18,7 @@
 
 - name: nginx-proxy | Write nginx-proxy configuration
   template:
-    src: nginx.conf.j2
+    src: "loadbalancer/nginx.conf.j2"
     dest: "{{ nginx_config_dir }}/nginx.conf"
     owner: root
     mode: 0755

roles/kubernetes/node/tasks/main.yml

@@ -17,7 +17,7 @@
   tags:
     - kubelet
 
-- import_tasks: nginx-proxy.yml
+- import_tasks: loadbalancer/nginx-proxy.yml
   when:
     - not is_kube_master
     - loadbalancer_apiserver_localhost
@@ -25,7 +25,7 @@
   tags:
     - nginx
 
-- import_tasks: haproxy.yml
+- import_tasks: loadbalancer/haproxy.yml
   when:
     - not is_kube_master
     - loadbalancer_apiserver_localhost
@@ -33,23 +33,6 @@
   tags:
     - haproxy
 
-- name: Make sure dynamic kubelet configuration directory is writeable
-  file:
-    path: "{{ dynamic_kubelet_configuration_dir }}"
-    mode: 0600
-    state: directory
-  when: dynamic_kubelet_configuration
-
-- name: Write kubelet config file (kubeadm)
-  template:
-    src: kubelet.kubeadm.env.j2
-    dest: "{{ kube_config_dir }}/kubelet.env"
-    backup: yes
-  notify: restart kubelet
-  tags:
-    - kubelet
-    - kubeadm
-
 - name: Ensure nodePort range is reserved
   sysctl:
     name: net.ipv4.ip_local_reserved_ports
@@ -99,7 +82,7 @@
     name: "{{ item }}"
     state: present
     sysctl_file: "{{ sysctl_file_path }}"
-    value: 1
+    value: "1"
     reload: yes
   when: sysctl_bridge_nf_call_iptables.rc == 0
   with_items:
@@ -175,7 +158,7 @@
   tags:
     - kube-proxy
 
-- include_tasks: "{{ cloud_provider }}-credential-check.yml"
+- include_tasks: "cloud-credentials/{{ cloud_provider }}-credential-check.yml"
   when:
     - cloud_provider is defined
     - cloud_provider in [ 'openstack', 'azure', 'vsphere' ]
@@ -200,7 +183,7 @@
 
 - name: Write cloud-config
   template:
-    src: "{{ cloud_provider }}-cloud-config.j2"
+    src: "cloud-configs/{{ cloud_provider }}-cloud-config.j2"
     dest: "{{ kube_config_dir }}/cloud_config"
     group: "{{ kube_cert_group }}"
     mode: 0640
@@ -211,13 +194,7 @@
   tags:
     - cloud-provider
 
-# reload-systemd
+- import_tasks: kubelet.yml
-- meta: flush_handlers
-
-- name: Enable kubelet
-  service:
-    name: kubelet
-    enabled: yes
-    state: started
   tags:
     - kubelet
+    - kubeadm

roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2

@@ -0,0 +1,57 @@
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+nodeStatusUpdateFrequency: "{{ kubelet_status_update_frequency }}"
+failSwapOn: {{ kubelet_fail_swap_on|default(true) }}
+authentication:
+  anonymous:
+    enabled: false
+  webhook:
+    enabled: {{ kubelet_authentication_token_webhook }}
+  x509:
+    clientCAFile: {{ kube_cert_dir }}/ca.crt
+authorization:
+{% if kubelet_authorization_mode_webhook %}
+  mode: Webhook
+{% else %}
+  mode: AlwaysAllow
+{% endif %}
+{% if kubelet_enforce_node_allocatable is defined and kubelet_enforce_node_allocatable != "\"\"" %}
+{% set kubelet_enforce_node_allocatable_list = kubelet_enforce_node_allocatable.split() %}
+enforceNodeAllocatable:
+{% for item in kubelet_enforce_node_allocatable_list %}
+- {{ item }}
+{% endfor %}
+{% endif %}
+staticPodPath: {{ kube_manifest_dir }}
+cgroupDriver: {{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }}
+maxPods: {{ kubelet_max_pods }}
+address: {{ kubelet_bind_address }}
+readOnlyPort: {{ kube_read_only_port }}
+kubeletCgroups: {{ kubelet_kubelet_cgroups }}
+clusterDomain: {{ dns_domain }}
+{# DNS settings for kubelet #}
+{% if enable_nodelocaldns %}
+{% set kubelet_cluster_dns = [nodelocaldns_ip] %}
+{% elif dns_mode in ['coredns'] %}
+{% set kubelet_cluster_dns = [skydns_server] %}
+{% elif dns_mode == 'coredns_dual' %}
+{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %}
+{% elif dns_mode == 'manual' %}
+{% set kubelet_cluster_dns = [manual_dns_server] %}
+{% else %}
+{% set kubelet_cluster_dns = [] %}
+{% endif %}
+clusterDNS:
+{% for dns_address in kubelet_cluster_dns %}
+- {{ dns_address }}
+{% endfor %}
+{# Node reserved CPU/memory #}
+kubeReserved:
+{% if is_kube_master|bool %}
+  cpu: {{ kube_master_cpu_reserved }}
+  memory: {{ kube_master_memory_reserved|regex_replace('Mi', 'M') }}
+{% else %}
+  cpu: {{ kube_cpu_reserved }}
+  memory: {{ kube_memory_reserved|regex_replace('Mi', 'M') }}
+{% endif %}
+resolvConf: "{{ kube_resolv_conf }}"

roles/kubernetes/node/templates/kubelet.env.v1beta1.j2

@@ -0,0 +1,78 @@
+KUBE_LOGTOSTDERR="--logtostderr=true"
+KUBE_LOG_LEVEL="--v={{ kube_log_level }}"
+KUBELET_ADDRESS="--node-ip={{ kubelet_address }}"
+{% if kube_override_hostname|default('') %}
+KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
+{% endif %}
+
+{# Base kubelet args #}
+{% set kubelet_args_base -%}
+{# start kubeadm specific settings #}
+--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \
+--config={{ kube_config_dir }}/kubelet-config.yaml \
+--kubeconfig={{ kube_config_dir }}/kubelet.conf \
+{% if kube_version is version('v1.8', '<') %}
+--require-kubeconfig \
+{% endif %}
+{% if kube_version is version('v1.12.0', '<') %}
+--cadvisor-port={{ kube_cadvisor_port }} \
+{% endif %}
+{# end kubeadm specific settings #}
+--pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
+{% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %}
+--docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
+{% endif %}
+{% if container_manager == 'crio' %}
+--container-runtime=remote \
+--container-runtime-endpoint=/var/run/crio/crio.sock \
+{% endif %}
+{% if kube_version is version('v1.8', '<') %}
+--experimental-fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \
+{% endif %}
+{% if dynamic_kubelet_configuration %}
+--dynamic-config-dir={{ dynamic_kubelet_configuration_dir }} \
+{% endif %}
+--runtime-cgroups={{ kubelet_runtime_cgroups }} \
+{% endset %}
+
+{# Kubelet node labels #}
+{% set role_node_labels = [] %}
+{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
+{%   if inventory_hostname in nvidia_gpu_nodes %}
+{%     set dummy = role_node_labels.append('nvidia.com/gpu=true')  %}
+{%   endif %}
+{% endif %}
+
+{% set inventory_node_labels = [] %}
+{% if node_labels is defined and node_labels is mapping %}
+{%   for labelname, labelvalue in node_labels.items() %}
+{%     set dummy = inventory_node_labels.append('%s=%s'|format(labelname, labelvalue)) %}
+{%   endfor %}
+{% endif %}
+{% set all_node_labels = role_node_labels + inventory_node_labels %}
+
+{# Kubelet node taints for gpu #}
+{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
+{%   if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %}
+{%       set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
+{%   elif inventory_hostname in nvidia_gpu_nodes and node_taints is not defined %}
+{%       set node_taints = [] %}
+{%       set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
+{%   endif %}
+{% endif %}
+
+KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
+{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router"] %}
+KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
+{% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}
+KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet"
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"
+{% elif cloud_provider is defined and cloud_provider in ["external"] %}
+KUBELET_CLOUDPROVIDER="--cloud-provider=external --cloud-config={{ kube_config_dir }}/cloud_config"
+{% else %}
+KUBELET_CLOUDPROVIDER=""
+{% endif %}
+
+PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

roles/kubernetes/node/templates/kubelet.service.j2

@@ -15,7 +15,6 @@ ExecStart={{ bin_dir }}/kubelet \
 		$KUBELET_ADDRESS \
 		$KUBELET_PORT \
 		$KUBELET_HOSTNAME \
-		$KUBE_ALLOW_PRIV \
 		$KUBELET_ARGS \
 		$DOCKER_SOCKET \
 		$KUBELET_NETWORK_PLUGIN \

roles/kubernetes/node/templates/node-kubeconfig.yaml.j2

@@ -1,3 +1,4 @@
+---
 apiVersion: v1
 kind: Config
 clusters:

roles/kubespray-defaults/defaults/main.yaml

@@ -314,7 +314,7 @@ rbac_enabled: "{{ 'RBAC' in authorization_modes }}"
 kubelet_authentication_token_webhook: true
 
 # When enabled, access to the kubelet API requires authorization by delegation to the API server
-kubelet_authorization_mode_webhook: false
+kubelet_authorization_mode_webhook: true
 
 # kubelet uses certificates for authenticating to the Kubernetes API
 # Automatically generate a new key and request a new certificate from the Kubernetes API as the current certificate approaches expiration