From fa1d222eee41318758289baacbaff0777d76aad2 Mon Sep 17 00:00:00 2001 From: Alessio Greggi Date: Mon, 2 May 2022 20:03:15 +0200 Subject: [PATCH] add support for `EventRateLimit` plugin configuration (#8711) * feat: add support for EventRateLimit admission plugin * docs: add documentation about admission_control_config_file and EventRateLimit configuration --- docs/vars.md | 40 +++++++++++++++++++ .../control-plane/defaults/main/main.yml | 13 ++++++ .../control-plane/tasks/kubeadm-setup.yml | 24 +++++++++++ .../admission-controls.v1beta2.yaml.j2 | 9 +++++ .../templates/eventratelimit.v1beta2.yaml.j2 | 11 +++++ .../templates/kubeadm-config.v1beta2.yaml.j2 | 10 +++++ roles/kubernetes/control-plane/vars/main.yaml | 3 ++ 7 files changed, 110 insertions(+) create mode 100644 roles/kubernetes/control-plane/templates/admission-controls.v1beta2.yaml.j2 create mode 100644 roles/kubernetes/control-plane/templates/eventratelimit.v1beta2.yaml.j2 create mode 100644 roles/kubernetes/control-plane/vars/main.yaml diff --git a/docs/vars.md b/docs/vars.md index 00d35e48a..304163568 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -57,36 +57,55 @@ Kubernetes needs some parameters in order to get deployed. These are the following default cluster parameters: * *cluster_name* - Name of cluster (default is cluster.local) + * *container_manager* - Container Runtime to install in the nodes (default is containerd) + * *image_command_tool* - Tool used to pull images (default depends on `container_manager` and is `nerdctl` for `containerd`, `crictl` for `crio`, `docker` for `docker`) + * *image_command_tool_on_localhost* - Tool used to pull images on localhost (default is equal to `image_command_tool`) + * *dns_domain* - Name of cluster DNS domain (default is cluster.local) + * *kube_network_plugin* - Plugin to use for container networking + * *kube_service_addresses* - Subnet for cluster IPs (default is 10.233.0.0/18). Must not overlap with kube_pods_subnet + * *kube_pods_subnet* - Subnet for Pod IPs (default is 10.233.64.0/18). Must not overlap with kube_service_addresses. + * *kube_network_node_prefix* - Subnet allocated per-node for pod IPs. Remaining bits in kube_pods_subnet dictates how many kube_nodes can be in cluster. Setting this > 25 will raise an assertion in playbooks if the `kubelet_max_pods` var also isn't adjusted accordingly (assertion not applicable to calico which doesn't use this as a hard limit, see [Calico IP block sizes](https://docs.projectcalico.org/reference/resources/ippool#block-sizes). + * *enable_dual_stack_networks* - Setting this to true will provision both IPv4 and IPv6 networking for pods and services. + * *kube_service_addresses_ipv6* - Subnet for cluster IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1000/116``). Must not overlap with ``kube_pods_subnet_ipv6``. + * *kube_pods_subnet_ipv6* - Subnet for Pod IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1:0000/112``). Must not overlap with ``kube_service_addresses_ipv6``. + * *kube_network_node_prefix_ipv6* - Subnet allocated per-node for pod IPv6 IPs. Remaining bits in ``kube_pods_subnet_ipv6`` dictates how many kube_nodes can be in cluster. + * *skydns_server* - Cluster IP for DNS (default is 10.233.0.3) + * *skydns_server_secondary* - Secondary Cluster IP for CoreDNS used with coredns_dual deployment (default is 10.233.0.4) + * *enable_coredns_k8s_external* - If enabled, it configures the [k8s_external plugin](https://coredns.io/plugins/k8s_external/) on the CoreDNS service. + * *coredns_k8s_external_zone* - Zone that will be used when CoreDNS k8s_external plugin is enabled (default is k8s_external.local) + * *enable_coredns_k8s_endpoint_pod_names* - If enabled, it configures endpoint_pod_names option for kubernetes plugin. on the CoreDNS service. + * *cloud_provider* - Enable extra Kubelet option if operating inside GCE or OpenStack (default is unset) + * *kube_feature_gates* - A list of key=value pairs that describe feature gates for alpha/experimental Kubernetes features. (defaults is `[]`). Additionally, you can use also the following variables to individually customize your kubernetes components installation (they works exactly like `kube_feature_gates`): @@ -95,8 +114,10 @@ following default cluster parameters: * *kube_scheduler_feature_gates* * *kube_proxy_feature_gates* * *kubelet_feature_gates* + * *kubeadm_feature_gates* - A list of key=value pairs that describe feature gates for alpha/experimental Kubeadm features. (defaults is `[]`) + * *authorization_modes* - A list of [authorization mode]( https://kubernetes.io/docs/admin/authorization/#using-flags-for-your-authorization-module) that the cluster should be configured for. Defaults to `['Node', 'RBAC']` @@ -106,6 +127,25 @@ following default cluster parameters: require a service account and cluster role bindings. You can override this setting by setting authorization_modes to `[]`. +* *kube_apiserver_admission_control_config_file* - Enable configuration for `kube-apiserver` admission plugins. + Currently this variable allow you to configure the `EventRateLimit` admission plugin. + + To configure the **EventRateLimit** plugin you have to define a data structure like this: + +```yml +kube_apiserver_admission_event_rate_limits: + limit_1: + type: Namespace + qps: 50 + burst: 100 + cache_size: 2000 + limit_2: + type: User + qps: 50 + burst: 100 + ... +``` + Note, if cloud providers have any use of the ``10.233.0.0/16``, like instances' private addresses, make sure to pick another values for ``kube_service_addresses`` and ``kube_pods_subnet``, for example from the ``172.18.0.0/16``. diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index ad7037029..227a53b09 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -89,6 +89,19 @@ kube_apiserver_pod_eviction_unreachable_timeout_seconds: "300" # 1.10+ admission plugins kube_apiserver_enable_admission_plugins: [] +# enable admission plugins configuration +kube_apiserver_admission_control_config_file: false + +# data structure to configure EventRateLimit admission plugin +# this should have the following structure: +# kube_apiserver_admission_event_rate_limits: +# : +# type: +# qps: +# burst: +# cache_size: +kube_apiserver_admission_event_rate_limits: {} + # 1.10+ list of disabled admission plugins kube_apiserver_disable_admission_plugins: [] diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml index 23f798d64..cdb22f318 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml @@ -83,6 +83,30 @@ dest: "{{ kube_config_dir }}/kubeadm-config.yaml" mode: 0640 +- name: kubeadm | Create directory to store admission control configurations + file: + path: "{{ kube_config_dir }}/admission-controls" + state: directory + mode: 0640 + when: kube_apiserver_admission_control_config_file + +- name: kubeadm | Push admission control config file + template: + src: "admission-controls.{{ kubeadmConfig_api_version }}.yaml.j2" + dest: "{{ kube_config_dir }}/admission-controls/admission-controls.yaml" + mode: 0640 + when: kube_apiserver_admission_control_config_file + +- name: kubeadm | Push admission control config files + template: + src: "{{ item|lower }}.{{ kubeadmConfig_api_version }}.yaml.j2" + dest: "{{ kube_config_dir }}/admission-controls/{{ item|lower }}.yaml" + mode: 0640 + when: + - kube_apiserver_admission_control_config_file + - item in kube_apiserver_admission_plugins_needs_configuration + loop: "{{ kube_apiserver_enable_admission_plugins[0].split(',') }}" + - name: kubeadm | Check if apiserver.crt contains all needed SANs shell: | set -o pipefail diff --git a/roles/kubernetes/control-plane/templates/admission-controls.v1beta2.yaml.j2 b/roles/kubernetes/control-plane/templates/admission-controls.v1beta2.yaml.j2 new file mode 100644 index 000000000..0bb4517c2 --- /dev/null +++ b/roles/kubernetes/control-plane/templates/admission-controls.v1beta2.yaml.j2 @@ -0,0 +1,9 @@ +apiVersion: apiserver.config.k8s.io/v1 +kind: AdmissionConfiguration +plugins: +{% for plugin in kube_apiserver_enable_admission_plugins[0].split(',') %} +{% if plugin in kube_apiserver_admission_plugins_needs_configuration %} +- name: {{ plugin }} + path: {{ kube_config_dir }}/{{ plugin|lower }}.yaml +{% endif %} +{% endfor %} diff --git a/roles/kubernetes/control-plane/templates/eventratelimit.v1beta2.yaml.j2 b/roles/kubernetes/control-plane/templates/eventratelimit.v1beta2.yaml.j2 new file mode 100644 index 000000000..0d7867070 --- /dev/null +++ b/roles/kubernetes/control-plane/templates/eventratelimit.v1beta2.yaml.j2 @@ -0,0 +1,11 @@ +apiVersion: eventratelimit.admission.k8s.io/v1alpha1 +kind: Configuration +limits: +{% for limit in kube_apiserver_admission_event_rate_limits.values() %} +- type: {{ limit.type }} + qps: {{ limit.qps }} + burst: {{ limit.burst }} +{% if limit.cache_size is defined %} + cacheSize: {{ limit.cache_size }} +{% endif %} +{% endfor %} diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 index bf7868bd8..a43c549de 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 @@ -126,6 +126,9 @@ apiServer: {% if kube_apiserver_enable_admission_plugins|length > 0 %} enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }} {% endif %} +{% if kube_apiserver_admission_control_config_file %} + admission-control-config-file: {{ kube_config_dir }}/admission-controls.yaml +{% endif %} {% if kube_apiserver_disable_admission_plugins|length > 0 %} disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }} {% endif %} @@ -249,6 +252,13 @@ apiServer: readOnly: false {% endif %} {% endif %} +{% if kube_apiserver_admission_control_config_file %} + - name: admission-control-configs + hostPath: {{ kube_config_dir }}/admission-controls + mountPath: {{ kube_config_dir }} + readOnly: false + pathType: DirectoryOrCreate +{% endif %} {% for volume in apiserver_extra_volumes %} - name: {{ volume.name }} hostPath: {{ volume.hostPath }} diff --git a/roles/kubernetes/control-plane/vars/main.yaml b/roles/kubernetes/control-plane/vars/main.yaml new file mode 100644 index 000000000..57a39f784 --- /dev/null +++ b/roles/kubernetes/control-plane/vars/main.yaml @@ -0,0 +1,3 @@ +--- +# list of admission plugins that needs to be configured +kube_apiserver_admission_plugins_needs_configuration: [EventRateLimit]