From 5a8cf824f60c37dc79c3b1947bd727726424668c Mon Sep 17 00:00:00 2001 From: Ilya Margolin Date: Tue, 8 Nov 2022 15:44:32 +0100 Subject: [PATCH] [containerd] Simplify limiting number of open files per container (#9319) by setting a default runtime spec with a patch for RLIMIT_NOFILE. - Introduces containerd_base_runtime_spec_rlimit_nofile. - Generates base_runtime_spec on-the-fly, to use the containerd version of the node. --- docs/containerd.md | 17 +- .../containerd/defaults/main.yml | 13 +- .../containerd/files/cri-base.json | 214 ------------------ .../containerd/tasks/main.yml | 10 + 4 files changed, 31 insertions(+), 223 deletions(-) delete mode 100644 roles/container-engine/containerd/files/cri-base.json diff --git a/docs/containerd.md b/docs/containerd.md index 847f7c9ca..5b20e7f16 100644 --- a/docs/containerd.md +++ b/docs/containerd.md @@ -64,14 +64,17 @@ is a list of such dictionaries. Default runtime can be changed by setting `containerd_default_runtime`. -#### base_runtime_spec +#### Base runtime specs and limiting number of open files -`base_runtime_spec` key in a runtime dictionary can be used to explicitly -specify a runtime spec json file. We ship the default one which is generated -with `ctr oci spec > /etc/containerd/cri-base.json`. It will be used if you set -`base_runtime_spec: cri-base.json`. The main advantage of doing so is the presence of -`rlimits` section in this configuration, which will restrict the maximum number -of file descriptors(open files) per container to 1024. +`base_runtime_spec` key in a runtime dictionary is used to explicitly +specify a runtime spec json file. `runc` runtime has it set to `cri-base.json`, +which is generated with `ctr oci spec > /etc/containerd/cri-base.json` and +updated to include a custom setting for maximum number of file descriptors per +container. + +You can change maximum number of file descriptors per container for the default +`runc` runtime by setting the `containerd_base_runtime_spec_rlimit_nofile` +variable. You can tune many more [settings][runtime-spec] by supplying your own file name and content with `containerd_base_runtime_specs`: diff --git a/roles/container-engine/containerd/defaults/main.yml b/roles/container-engine/containerd/defaults/main.yml index 5f82fae59..cc630ff20 100644 --- a/roles/container-engine/containerd/defaults/main.yml +++ b/roles/container-engine/containerd/defaults/main.yml @@ -15,7 +15,7 @@ containerd_runc_runtime: type: "io.containerd.runc.v2" engine: "" root: "" - # base_runtime_spec: cri-base.json # use this to limit number of file descriptors per container + base_runtime_spec: cri-base.json options: systemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}" @@ -26,8 +26,17 @@ containerd_additional_runtimes: [] # engine: "" # root: "" +containerd_base_runtime_spec_rlimit_nofile: 16384 + +containerd_default_base_runtime_spec_patch: + process: + rlimits: + - type: RLIMIT_NOFILE + hard: "{{ containerd_base_runtime_spec_rlimit_nofile }}" + soft: "{{ containerd_base_runtime_spec_rlimit_nofile }}" + containerd_base_runtime_specs: - cri-base.json: "{{ lookup('file', 'cri-base.json') }}" + cri-base.json: "{{ containerd_default_base_runtime_spec | combine(containerd_default_base_runtime_spec_patch,recursive=1) }}" containerd_grpc_max_recv_message_size: 16777216 containerd_grpc_max_send_message_size: 16777216 diff --git a/roles/container-engine/containerd/files/cri-base.json b/roles/container-engine/containerd/files/cri-base.json deleted file mode 100644 index f022438a4..000000000 --- a/roles/container-engine/containerd/files/cri-base.json +++ /dev/null @@ -1,214 +0,0 @@ -{ - "ociVersion": "1.0.2-dev", - "process": { - "user": { - "uid": 0, - "gid": 0 - }, - "cwd": "/", - "capabilities": { - "bounding": [ - "CAP_CHOWN", - "CAP_DAC_OVERRIDE", - "CAP_FSETID", - "CAP_FOWNER", - "CAP_MKNOD", - "CAP_NET_RAW", - "CAP_SETGID", - "CAP_SETUID", - "CAP_SETFCAP", - "CAP_SETPCAP", - "CAP_NET_BIND_SERVICE", - "CAP_SYS_CHROOT", - "CAP_KILL", - "CAP_AUDIT_WRITE" - ], - "effective": [ - "CAP_CHOWN", - "CAP_DAC_OVERRIDE", - "CAP_FSETID", - "CAP_FOWNER", - "CAP_MKNOD", - "CAP_NET_RAW", - "CAP_SETGID", - "CAP_SETUID", - "CAP_SETFCAP", - "CAP_SETPCAP", - "CAP_NET_BIND_SERVICE", - "CAP_SYS_CHROOT", - "CAP_KILL", - "CAP_AUDIT_WRITE" - ], - "inheritable": [ - "CAP_CHOWN", - "CAP_DAC_OVERRIDE", - "CAP_FSETID", - "CAP_FOWNER", - "CAP_MKNOD", - "CAP_NET_RAW", - "CAP_SETGID", - "CAP_SETUID", - "CAP_SETFCAP", - "CAP_SETPCAP", - "CAP_NET_BIND_SERVICE", - "CAP_SYS_CHROOT", - "CAP_KILL", - "CAP_AUDIT_WRITE" - ], - "permitted": [ - "CAP_CHOWN", - "CAP_DAC_OVERRIDE", - "CAP_FSETID", - "CAP_FOWNER", - "CAP_MKNOD", - "CAP_NET_RAW", - "CAP_SETGID", - "CAP_SETUID", - "CAP_SETFCAP", - "CAP_SETPCAP", - "CAP_NET_BIND_SERVICE", - "CAP_SYS_CHROOT", - "CAP_KILL", - "CAP_AUDIT_WRITE" - ] - }, - "rlimits": [ - { - "type": "RLIMIT_NOFILE", - "hard": 1024, - "soft": 1024 - } - ], - "noNewPrivileges": true - }, - "root": { - "path": "rootfs" - }, - "mounts": [ - { - "destination": "/proc", - "type": "proc", - "source": "proc", - "options": [ - "nosuid", - "noexec", - "nodev" - ] - }, - { - "destination": "/dev", - "type": "tmpfs", - "source": "tmpfs", - "options": [ - "nosuid", - "strictatime", - "mode=755", - "size=65536k" - ] - }, - { - "destination": "/dev/pts", - "type": "devpts", - "source": "devpts", - "options": [ - "nosuid", - "noexec", - "newinstance", - "ptmxmode=0666", - "mode=0620", - "gid=5" - ] - }, - { - "destination": "/dev/shm", - "type": "tmpfs", - "source": "shm", - "options": [ - "nosuid", - "noexec", - "nodev", - "mode=1777", - "size=65536k" - ] - }, - { - "destination": "/dev/mqueue", - "type": "mqueue", - "source": "mqueue", - "options": [ - "nosuid", - "noexec", - "nodev" - ] - }, - { - "destination": "/sys", - "type": "sysfs", - "source": "sysfs", - "options": [ - "nosuid", - "noexec", - "nodev", - "ro" - ] - }, - { - "destination": "/run", - "type": "tmpfs", - "source": "tmpfs", - "options": [ - "nosuid", - "strictatime", - "mode=755", - "size=65536k" - ] - } - ], - "linux": { - "resources": { - "devices": [ - { - "allow": false, - "access": "rwm" - } - ] - }, - "cgroupsPath": "/default", - "namespaces": [ - { - "type": "pid" - }, - { - "type": "ipc" - }, - { - "type": "uts" - }, - { - "type": "mount" - }, - { - "type": "network" - } - ], - "maskedPaths": [ - "/proc/acpi", - "/proc/asound", - "/proc/kcore", - "/proc/keys", - "/proc/latency_stats", - "/proc/timer_list", - "/proc/timer_stats", - "/proc/sched_debug", - "/sys/firmware", - "/proc/scsi" - ], - "readonlyPaths": [ - "/proc/bus", - "/proc/fs", - "/proc/irq", - "/proc/sys", - "/proc/sysrq-trigger" - ] - } -} diff --git a/roles/container-engine/containerd/tasks/main.yml b/roles/container-engine/containerd/tasks/main.yml index 6bb536413..50efd4add 100644 --- a/roles/container-engine/containerd/tasks/main.yml +++ b/roles/container-engine/containerd/tasks/main.yml @@ -84,6 +84,16 @@ notify: restart containerd when: http_proxy is defined or https_proxy is defined +- name: containerd | Generate default base_runtime_spec + register: ctr_oci_spec + command: "{{ containerd_bin_dir }}/ctr oci spec" + check_mode: false + changed_when: false + +- name: containerd | Store generated default base_runtime_spec + set_fact: + containerd_default_base_runtime_spec: "{{ ctr_oci_spec.stdout | from_json }}" + - name: containerd | Write base_runtime_specs copy: content: "{{ item.value }}"