From e03e3c4582602ce7e2c8d7489ddf1957ad435f12 Mon Sep 17 00:00:00 2001 From: Victor Morales Date: Fri, 23 Oct 2020 03:07:46 -0700 Subject: [PATCH] Add Kata Containers support to CRI-O runtime (#6830) * Enable Kata Containers for CRI-O runtime Kata Containers is an OCI runtime where containers are run inside lightweight VMs. This runtime has been enabled for containerd runtime thru the kata_containers_enabled variable. This change enables Kata Containers to CRI-O container runtime. Signed-off-by: Victor Morales * Set appropiate conmon_cgroup when crio_cgroup_manager is 'cgroupfs' * Set manage_ns_lifecycle=true when KataContainers is enabed * Add preinstall check for katacontainers Signed-off-by: Victor Morales Co-authored-by: Pasquale Toscano --- .../container-engine/cri-o/defaults/main.yml | 23 +++++++++++++- roles/container-engine/cri-o/tasks/main.yaml | 6 ++++ .../cri-o/templates/crio.conf.j2 | 30 +++++++++---------- roles/container-engine/cri-o/vars/debian.yml | 7 ++++- roles/container-engine/cri-o/vars/ubuntu.yml | 7 ++++- .../preinstall/tasks/0020-verify-settings.yml | 6 ++++ roles/kubespray-defaults/defaults/main.yaml | 2 +- 7 files changed, 61 insertions(+), 20 deletions(-) diff --git a/roles/container-engine/cri-o/defaults/main.yml b/roles/container-engine/cri-o/defaults/main.yml index b76bf27ea..a2d690b3f 100644 --- a/roles/container-engine/cri-o/defaults/main.yml +++ b/roles/container-engine/cri-o/defaults/main.yml @@ -11,7 +11,6 @@ crio_pause_image: "{{ pod_infra_image_repo }}:{{ pod_infra_version }}" # By default unqualified images are not allowed for security reasons crio_registries: [] -crio_runc_path: "/usr/bin/runc" crio_seccomp_profile: "" crio_selinux: "{{ (preinstall_selinux_state == 'enforcing')|lower }}" crio_signature_policy: "{% if ansible_os_family == 'ClearLinux' %}/usr/share/defaults/crio/policy.json{% endif %}" @@ -29,3 +28,25 @@ crio_kubernetes_version_matrix: "1.17": "1.17" crio_version: "{{ crio_kubernetes_version_matrix[crio_required_version] | default('1.19') }}" + +# The crio_runtimes variable defines a list of OCI compatible runtimes. +crio_runtimes: + - name: runc + path: /usr/bin/runc + type: oci + root: /run/runc + +# Kata Containers is an OCI runtime, where containers are run inside lightweight +# VMs. Kata provides additional isolation towards the host, minimizing the host attack +# surface and mitigating the consequences of containers breakout. +kata_runtimes: + # Kata Containers with the default configured VMM + - name: kata-runtime + path: /opt/kata/bin/kata-runtime + type: oci + root: /run/kata-containers + # Kata Containers with the QEMU VMM + - name: kata-qemu + path: /opt/kata/bin/kata-qemu + type: oci + root: /run/kata-containers diff --git a/roles/container-engine/cri-o/tasks/main.yaml b/roles/container-engine/cri-o/tasks/main.yaml index 504330d13..61bea7af6 100644 --- a/roles/container-engine/cri-o/tasks/main.yaml +++ b/roles/container-engine/cri-o/tasks/main.yaml @@ -29,6 +29,12 @@ - import_tasks: "crictl.yml" +- name: Build a list of crio runtimes + set_fact: + crio_runtimes: "{{ crio_runtimes + kata_runtimes }}" + when: + - kata_containers_enabled + - name: Make sure needed folders exist in the system with_items: - /etc/crio diff --git a/roles/container-engine/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2 index fd85e5eb4..a456d16d4 100644 --- a/roles/container-engine/cri-o/templates/crio.conf.j2 +++ b/roles/container-engine/cri-o/templates/crio.conf.j2 @@ -110,7 +110,11 @@ decryption_keys_path = "/etc/crio/keys/" conmon = "{{ crio_conmon }}" # Cgroup setting for conmon +{% if crio_cgroup_manager == "cgroupfs" %} +conmon_cgroup = "pod" +{% else %} conmon_cgroup = "system.slice" +{% endif %} # Environment variable list for the conmon process, used for passing necessary # environment variables to conmon or the runtime. @@ -250,7 +254,11 @@ ctr_stop_timeout = 30 # manage_ns_lifecycle determines whether we pin and remove namespaces # and manage their lifecycle +{% if kata_containers_enabled %} +manage_ns_lifecycle = true +{% else %} manage_ns_lifecycle = false +{% endif %} # The directory where the state of the managed namespaces gets tracked. # Only used when manage_ns_lifecycle is true. @@ -280,22 +288,12 @@ pinns_path = "" # - runtime_root (optional, string): root directory for storage of containers # state. - -[crio.runtime.runtimes.runc] -runtime_path = "{{ crio_runc_path }}" -runtime_type = "oci" -runtime_root = "/run/runc" - - -# Kata Containers is an OCI runtime, where containers are run inside lightweight -# VMs. Kata provides additional isolation towards the host, minimizing the host attack -# surface and mitigating the consequences of containers breakout. - -# Kata Containers with the default configured VMM -#[crio.runtime.runtimes.kata-runtime] - -# Kata Containers with the QEMU VMM -#[crio.runtime.runtimes.kata-qemu] +{% for runtime in crio_runtimes %} +[crio.runtime.runtimes.{{ runtime.name }}] +runtime_path = "{{ runtime.path }}" +runtime_type = "{{ runtime.type }}" +runtime_root = "{{ runtime.root }}" +{% endfor %} # Kata Containers with the Firecracker VMM #[crio.runtime.runtimes.kata-fc] diff --git a/roles/container-engine/cri-o/vars/debian.yml b/roles/container-engine/cri-o/vars/debian.yml index 0ba558e02..cc9ca8442 100644 --- a/roles/container-engine/cri-o/vars/debian.yml +++ b/roles/container-engine/cri-o/vars/debian.yml @@ -4,4 +4,9 @@ crio_packages: - "cri-o" - "cri-o-runc" -crio_runc_path: /usr/sbin/runc +# The crio_runtimes variable defines a list of OCI compatible runtimes. +crio_runtimes: + - name: runc + path: /usr/sbin/runc + type: oci + root: /run/runc diff --git a/roles/container-engine/cri-o/vars/ubuntu.yml b/roles/container-engine/cri-o/vars/ubuntu.yml index 0ba558e02..cc9ca8442 100644 --- a/roles/container-engine/cri-o/vars/ubuntu.yml +++ b/roles/container-engine/cri-o/vars/ubuntu.yml @@ -4,4 +4,9 @@ crio_packages: - "cri-o" - "cri-o-runc" -crio_runc_path: /usr/sbin/runc +# The crio_runtimes variable defines a list of OCI compatible runtimes. +crio_runtimes: + - name: runc + path: /usr/sbin/runc + type: oci + root: /run/runc diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index e590d6609..7157299c8 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -286,6 +286,12 @@ msg: "download_run_once support only for docker. See https://github.com/containerd/containerd/issues/4075 for details" when: download_run_once or download_force_cache +- name: Stop if kata_containers_enabled is enabled when container_manager is docker + assert: + that: container_manager != 'docker' + msg: "kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details" + when: kata_containers_enabled + - name: Stop if download_localhost is enabled for Flatcar Container Linux assert: that: ansible_os_family not in ["Flatcar Container Linux by Kinvolk"] diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 7a9c31dc2..b412b7da5 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -221,7 +221,7 @@ kube_profiling: false container_manager: docker # Enable Kata Containers as additional container runtime -# When enabled, it requires container_manager=containerd +# When enabled, it requires `container_manager` different than Docker kata_containers_enabled: false # Container on localhost (download images when download_localhost is true)