[containerd] Simplify limiting number of open files per container (#9319)
by setting a default runtime spec with a patch for RLIMIT_NOFILE. - Introduces containerd_base_runtime_spec_rlimit_nofile. - Generates base_runtime_spec on-the-fly, to use the containerd version of the node.
This commit is contained in:
Ilya Margolin
GitHub
parent
5c25b57989
commit
5a8cf824f6
|
|
@ -64,14 +64,17 @@ is a list of such dictionaries.
|
||||||
|
|
||||||
Default runtime can be changed by setting `containerd_default_runtime`.
|
Default runtime can be changed by setting `containerd_default_runtime`.
|
||||||
|
|
||||||
#### base_runtime_spec
|
#### Base runtime specs and limiting number of open files
|
||||||
|
|
||||||
`base_runtime_spec` key in a runtime dictionary can be used to explicitly
|
`base_runtime_spec` key in a runtime dictionary is used to explicitly
|
||||||
specify a runtime spec json file. We ship the default one which is generated
|
specify a runtime spec json file. `runc` runtime has it set to `cri-base.json`,
|
||||||
with `ctr oci spec > /etc/containerd/cri-base.json`. It will be used if you set
|
which is generated with `ctr oci spec > /etc/containerd/cri-base.json` and
|
||||||
`base_runtime_spec: cri-base.json`. The main advantage of doing so is the presence of
|
updated to include a custom setting for maximum number of file descriptors per
|
||||||
`rlimits` section in this configuration, which will restrict the maximum number
|
container.
|
||||||
of file descriptors(open files) per container to 1024.
|
|
||||||
|
You can change maximum number of file descriptors per container for the default
|
||||||
|
`runc` runtime by setting the `containerd_base_runtime_spec_rlimit_nofile`
|
||||||
|
variable.
|
||||||
|
|
||||||
You can tune many more [settings][runtime-spec] by supplying your own file name and content with `containerd_base_runtime_specs`:
|
You can tune many more [settings][runtime-spec] by supplying your own file name and content with `containerd_base_runtime_specs`:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ containerd_runc_runtime:
|
||||||
type: "io.containerd.runc.v2"
|
type: "io.containerd.runc.v2"
|
||||||
engine: ""
|
engine: ""
|
||||||
root: ""
|
root: ""
|
||||||
# base_runtime_spec: cri-base.json # use this to limit number of file descriptors per container
|
base_runtime_spec: cri-base.json
|
||||||
options:
|
options:
|
||||||
systemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}"
|
systemdCgroup: "{{ containerd_use_systemd_cgroup | ternary('true', 'false') }}"
|
||||||
|
|
||||||
|
|
@ -26,8 +26,17 @@ containerd_additional_runtimes: []
|
||||||
# engine: ""
|
# engine: ""
|
||||||
# root: ""
|
# root: ""
|
||||||
|
|
||||||
|
containerd_base_runtime_spec_rlimit_nofile: 16384
|
||||||
|
|
||||||
|
containerd_default_base_runtime_spec_patch:
|
||||||
|
process:
|
||||||
|
rlimits:
|
||||||
|
- type: RLIMIT_NOFILE
|
||||||
|
hard: "{{ containerd_base_runtime_spec_rlimit_nofile }}"
|
||||||
|
soft: "{{ containerd_base_runtime_spec_rlimit_nofile }}"
|
||||||
|
|
||||||
containerd_base_runtime_specs:
|
containerd_base_runtime_specs:
|
||||||
cri-base.json: "{{ lookup('file', 'cri-base.json') }}"
|
cri-base.json: "{{ containerd_default_base_runtime_spec | combine(containerd_default_base_runtime_spec_patch,recursive=1) }}"
|
||||||
|
|
||||||
containerd_grpc_max_recv_message_size: 16777216
|
containerd_grpc_max_recv_message_size: 16777216
|
||||||
containerd_grpc_max_send_message_size: 16777216
|
containerd_grpc_max_send_message_size: 16777216
|
||||||
|
|
|
||||||
|
|
@ -1,214 +0,0 @@
|
||||||
{
|
|
||||||
"ociVersion": "1.0.2-dev",
|
|
||||||
"process": {
|
|
||||||
"user": {
|
|
||||||
"uid": 0,
|
|
||||||
"gid": 0
|
|
||||||
},
|
|
||||||
"cwd": "/",
|
|
||||||
"capabilities": {
|
|
||||||
"bounding": [
|
|
||||||
"CAP_CHOWN",
|
|
||||||
"CAP_DAC_OVERRIDE",
|
|
||||||
"CAP_FSETID",
|
|
||||||
"CAP_FOWNER",
|
|
||||||
"CAP_MKNOD",
|
|
||||||
"CAP_NET_RAW",
|
|
||||||
"CAP_SETGID",
|
|
||||||
"CAP_SETUID",
|
|
||||||
"CAP_SETFCAP",
|
|
||||||
"CAP_SETPCAP",
|
|
||||||
"CAP_NET_BIND_SERVICE",
|
|
||||||
"CAP_SYS_CHROOT",
|
|
||||||
"CAP_KILL",
|
|
||||||
"CAP_AUDIT_WRITE"
|
|
||||||
],
|
|
||||||
"effective": [
|
|
||||||
"CAP_CHOWN",
|
|
||||||
"CAP_DAC_OVERRIDE",
|
|
||||||
"CAP_FSETID",
|
|
||||||
"CAP_FOWNER",
|
|
||||||
"CAP_MKNOD",
|
|
||||||
"CAP_NET_RAW",
|
|
||||||
"CAP_SETGID",
|
|
||||||
"CAP_SETUID",
|
|
||||||
"CAP_SETFCAP",
|
|
||||||
"CAP_SETPCAP",
|
|
||||||
"CAP_NET_BIND_SERVICE",
|
|
||||||
"CAP_SYS_CHROOT",
|
|
||||||
"CAP_KILL",
|
|
||||||
"CAP_AUDIT_WRITE"
|
|
||||||
],
|
|
||||||
"inheritable": [
|
|
||||||
"CAP_CHOWN",
|
|
||||||
"CAP_DAC_OVERRIDE",
|
|
||||||
"CAP_FSETID",
|
|
||||||
"CAP_FOWNER",
|
|
||||||
"CAP_MKNOD",
|
|
||||||
"CAP_NET_RAW",
|
|
||||||
"CAP_SETGID",
|
|
||||||
"CAP_SETUID",
|
|
||||||
"CAP_SETFCAP",
|
|
||||||
"CAP_SETPCAP",
|
|
||||||
"CAP_NET_BIND_SERVICE",
|
|
||||||
"CAP_SYS_CHROOT",
|
|
||||||
"CAP_KILL",
|
|
||||||
"CAP_AUDIT_WRITE"
|
|
||||||
],
|
|
||||||
"permitted": [
|
|
||||||
"CAP_CHOWN",
|
|
||||||
"CAP_DAC_OVERRIDE",
|
|
||||||
"CAP_FSETID",
|
|
||||||
"CAP_FOWNER",
|
|
||||||
"CAP_MKNOD",
|
|
||||||
"CAP_NET_RAW",
|
|
||||||
"CAP_SETGID",
|
|
||||||
"CAP_SETUID",
|
|
||||||
"CAP_SETFCAP",
|
|
||||||
"CAP_SETPCAP",
|
|
||||||
"CAP_NET_BIND_SERVICE",
|
|
||||||
"CAP_SYS_CHROOT",
|
|
||||||
"CAP_KILL",
|
|
||||||
"CAP_AUDIT_WRITE"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"rlimits": [
|
|
||||||
{
|
|
||||||
"type": "RLIMIT_NOFILE",
|
|
||||||
"hard": 1024,
|
|
||||||
"soft": 1024
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"noNewPrivileges": true
|
|
||||||
},
|
|
||||||
"root": {
|
|
||||||
"path": "rootfs"
|
|
||||||
},
|
|
||||||
"mounts": [
|
|
||||||
{
|
|
||||||
"destination": "/proc",
|
|
||||||
"type": "proc",
|
|
||||||
"source": "proc",
|
|
||||||
"options": [
|
|
||||||
"nosuid",
|
|
||||||
"noexec",
|
|
||||||
"nodev"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"destination": "/dev",
|
|
||||||
"type": "tmpfs",
|
|
||||||
"source": "tmpfs",
|
|
||||||
"options": [
|
|
||||||
"nosuid",
|
|
||||||
"strictatime",
|
|
||||||
"mode=755",
|
|
||||||
"size=65536k"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"destination": "/dev/pts",
|
|
||||||
"type": "devpts",
|
|
||||||
"source": "devpts",
|
|
||||||
"options": [
|
|
||||||
"nosuid",
|
|
||||||
"noexec",
|
|
||||||
"newinstance",
|
|
||||||
"ptmxmode=0666",
|
|
||||||
"mode=0620",
|
|
||||||
"gid=5"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"destination": "/dev/shm",
|
|
||||||
"type": "tmpfs",
|
|
||||||
"source": "shm",
|
|
||||||
"options": [
|
|
||||||
"nosuid",
|
|
||||||
"noexec",
|
|
||||||
"nodev",
|
|
||||||
"mode=1777",
|
|
||||||
"size=65536k"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"destination": "/dev/mqueue",
|
|
||||||
"type": "mqueue",
|
|
||||||
"source": "mqueue",
|
|
||||||
"options": [
|
|
||||||
"nosuid",
|
|
||||||
"noexec",
|
|
||||||
"nodev"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"destination": "/sys",
|
|
||||||
"type": "sysfs",
|
|
||||||
"source": "sysfs",
|
|
||||||
"options": [
|
|
||||||
"nosuid",
|
|
||||||
"noexec",
|
|
||||||
"nodev",
|
|
||||||
"ro"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"destination": "/run",
|
|
||||||
"type": "tmpfs",
|
|
||||||
"source": "tmpfs",
|
|
||||||
"options": [
|
|
||||||
"nosuid",
|
|
||||||
"strictatime",
|
|
||||||
"mode=755",
|
|
||||||
"size=65536k"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"linux": {
|
|
||||||
"resources": {
|
|
||||||
"devices": [
|
|
||||||
{
|
|
||||||
"allow": false,
|
|
||||||
"access": "rwm"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"cgroupsPath": "/default",
|
|
||||||
"namespaces": [
|
|
||||||
{
|
|
||||||
"type": "pid"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "ipc"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "uts"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "mount"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "network"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"maskedPaths": [
|
|
||||||
"/proc/acpi",
|
|
||||||
"/proc/asound",
|
|
||||||
"/proc/kcore",
|
|
||||||
"/proc/keys",
|
|
||||||
"/proc/latency_stats",
|
|
||||||
"/proc/timer_list",
|
|
||||||
"/proc/timer_stats",
|
|
||||||
"/proc/sched_debug",
|
|
||||||
"/sys/firmware",
|
|
||||||
"/proc/scsi"
|
|
||||||
],
|
|
||||||
"readonlyPaths": [
|
|
||||||
"/proc/bus",
|
|
||||||
"/proc/fs",
|
|
||||||
"/proc/irq",
|
|
||||||
"/proc/sys",
|
|
||||||
"/proc/sysrq-trigger"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -84,6 +84,16 @@
|
||||||
notify: restart containerd
|
notify: restart containerd
|
||||||
when: http_proxy is defined or https_proxy is defined
|
when: http_proxy is defined or https_proxy is defined
|
||||||
|
|
||||||
|
- name: containerd | Generate default base_runtime_spec
|
||||||
|
register: ctr_oci_spec
|
||||||
|
command: "{{ containerd_bin_dir }}/ctr oci spec"
|
||||||
|
check_mode: false
|
||||||
|
changed_when: false
|
||||||
|
|
||||||
|
- name: containerd | Store generated default base_runtime_spec
|
||||||
|
set_fact:
|
||||||
|
containerd_default_base_runtime_spec: "{{ ctr_oci_spec.stdout | from_json }}"
|
||||||
|
|
||||||
- name: containerd | Write base_runtime_specs
|
- name: containerd | Write base_runtime_specs
|
||||||
copy:
|
copy:
|
||||||
content: "{{ item.value }}"
|
content: "{{ item.value }}"
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue