diff --git a/docs/_sidebar.md b/docs/_sidebar.md index e07deab4e..d704e8b44 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -41,6 +41,7 @@ * [Containerd](docs/containerd.md) * [CRI-O](docs/cri-o.md) * [Kata Containers](docs/kata-containers.md) + * [gVisor](docs/gvisor.md) * Advanced * [Proxy](/docs/proxy.md) * [Downloads](docs/downloads.md) diff --git a/docs/gvisor.md b/docs/gvisor.md new file mode 100644 index 000000000..ef0a64b10 --- /dev/null +++ b/docs/gvisor.md @@ -0,0 +1,16 @@ +# gVisor + +[gVisor](https://gvisor.dev/docs/) is an application kernel, written in Go, that implements a substantial portion of the Linux system call interface. It provides an additional layer of isolation between running applications and the host operating system. + +gVisor includes an Open Container Initiative (OCI) runtime called runsc that makes it easy to work with existing container tooling. The runsc runtime integrates with Docker and Kubernetes, making it simple to run sandboxed containers. + +## Usage + +To enable gVisor you should be using a container manager that is compatible with selecting the [RuntimeClass](https://kubernetes.io/docs/concepts/containers/runtime-class/) such as `containerd`. + +Containerd support: + +```yaml +container_manager: containerd +gvisor_enabled: true +``` diff --git a/roles/container-engine/containerd-common/defaults/main.yml b/roles/container-engine/containerd-common/defaults/main.yml index 5ee766409..e1555e986 100644 --- a/roles/container-engine/containerd-common/defaults/main.yml +++ b/roles/container-engine/containerd-common/defaults/main.yml @@ -1,2 +1,17 @@ --- containerd_package: 'containerd.io' + +# Fedora docker-ce repo +docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable' +docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg' +# CentOS/RedHat docker-ce repo +docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable' +docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg' +# Ubuntu docker-ce repo +docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu" +docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg' +docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88' +# Debian docker-ce repo +docker_debian_repo_base_url: "https://download.docker.com/linux/debian" +docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg' +docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88' diff --git a/roles/container-engine/containerd/templates/config.toml.j2 b/roles/container-engine/containerd/templates/config.toml.j2 index 8fca56814..a344e4b96 100644 --- a/roles/container-engine/containerd/templates/config.toml.j2 +++ b/roles/container-engine/containerd/templates/config.toml.j2 @@ -35,6 +35,10 @@ oom_score = {{ containerd_oom_score }} {% if kata_containers_enabled %} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu] runtime_type = "io.containerd.kata-qemu.v2" +{% endif %} +{% if gvisor_enabled %} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc] + runtime_type = "io.containerd.runsc.v1" {% endif %} [plugins."io.containerd.grpc.v1.cri".registry] [plugins."io.containerd.grpc.v1.cri".registry.mirrors] diff --git a/roles/container-engine/docker/defaults/main.yml b/roles/container-engine/docker/defaults/main.yml index 09cdbb6a8..5f376b03e 100644 --- a/roles/container-engine/docker/defaults/main.yml +++ b/roles/container-engine/docker/defaults/main.yml @@ -15,20 +15,6 @@ docker_cgroup_driver: systemd yum_repo_dir: /etc/yum.repos.d -# Fedora docker-ce repo -docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable' -docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg' -# CentOS/RedHat docker-ce repo -docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable' -docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg' -# Ubuntu docker-ce repo -docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu" -docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg' -docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88' -# Debian docker-ce repo -docker_debian_repo_base_url: "https://download.docker.com/linux/debian" -docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg' -docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88' docker_bin_dir: "/usr/bin" # flag to enable/disable docker cleanup diff --git a/roles/container-engine/gvisor/molecule/default/converge.yml b/roles/container-engine/gvisor/molecule/default/converge.yml new file mode 100644 index 000000000..8bf5478e8 --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/converge.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + become: true + vars: + gvisor_enabled: true + container_manager: containerd + roles: + - role: kubespray-defaults + - role: containerd + - role: gvisor diff --git a/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf b/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf new file mode 100644 index 000000000..f10935b75 --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf @@ -0,0 +1,17 @@ +{ + "cniVersion": "0.2.0", + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.19.0.0/24", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } +} diff --git a/roles/container-engine/gvisor/molecule/default/files/container.json b/roles/container-engine/gvisor/molecule/default/files/container.json new file mode 100644 index 000000000..3490359d0 --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/files/container.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "gvisor1" + }, + "image": { + "image": "docker.io/library/hello-world:latest" + }, + "log_path": "gvisor1.0.log", + "linux": {} +} diff --git a/roles/container-engine/gvisor/molecule/default/files/sandbox.json b/roles/container-engine/gvisor/molecule/default/files/sandbox.json new file mode 100644 index 000000000..a8da54d91 --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/files/sandbox.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "gvisor1", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "linux": {}, + "log_directory": "/tmp" +} diff --git a/roles/container-engine/gvisor/molecule/default/molecule.yml b/roles/container-engine/gvisor/molecule/default/molecule.yml new file mode 100644 index 000000000..fc4ec0276 --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/molecule.yml @@ -0,0 +1,44 @@ +--- +driver: + name: vagrant + provider: + name: libvirt + options: + driver: kvm +lint: | + set -e + yamllint -c ../../../.yamllint . +platforms: + - name: ubuntu20 + box: generic/ubuntu2004 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane + - name: centos8 + box: generic/centos8 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../ + config_options: + defaults: + callback_whitelist: profile_tasks + lint: + name: ansible-lint + options: + c: ../../../.ansible-lint + inventory: + group_vars: + all: + become: true +verifier: + name: testinfra + lint: + name: flake8 diff --git a/roles/container-engine/gvisor/molecule/default/prepare.yml b/roles/container-engine/gvisor/molecule/default/prepare.yml new file mode 100644 index 000000000..084824830 --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/prepare.yml @@ -0,0 +1,48 @@ +--- +- name: Prepare generic + hosts: all + become: true + roles: + - role: kubespray-defaults + - role: bootstrap-os + - role: ../adduser + user: "{{ addusers.kube }}" + tasks: + - include_tasks: "../../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.cni) }}" + +- name: Prepare container runtime + hosts: all + become: true + vars: + container_manager: containerd + kube_network_plugin: cni + roles: + - role: kubespray-defaults + - role: ../network_plugin/cni + - role: crictl + tasks: + - name: Copy test container files + copy: + src: "{{ item }}" + dest: "/tmp/{{ item }}" + owner: root + mode: 0644 + with_items: + - container.json + - sandbox.json + - name: Create /etc/cni/net.d directory + file: + path: /etc/cni/net.d + state: directory + owner: kube + mode: 0755 + - name: Setup CNI + copy: + src: "{{ item }}" + dest: "/etc/cni/net.d/{{ item }}" + owner: root + mode: 0644 + with_items: + - 10-mynet.conf diff --git a/roles/container-engine/gvisor/molecule/default/tests/test_default.py b/roles/container-engine/gvisor/molecule/default/tests/test_default.py new file mode 100644 index 000000000..eb40b00cb --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/tests/test_default.py @@ -0,0 +1,29 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_run(host): + gvisorruntime = "/usr/local/bin/runsc" + with host.sudo(): + cmd = host.command(gvisorruntime + " --version") + assert cmd.rc == 0 + assert "runsc version" in cmd.stdout + + +def test_run_pod(host): + runtime = "runsc" + + run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime) + with host.sudo(): + cmd = host.command(run_command) + assert cmd.rc == 0 + + with host.sudo(): + log_f = host.file("/tmp/gvisor1.0.log") + + assert log_f.exists + assert b"Hello from Docker!" in log_f.content diff --git a/roles/container-engine/gvisor/tasks/main.yml b/roles/container-engine/gvisor/tasks/main.yml new file mode 100644 index 000000000..fa5bd725e --- /dev/null +++ b/roles/container-engine/gvisor/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: gVisor | Download runsc binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.gvisor_runsc) }}" + +- name: gVisor | Download containerd-shim-runsc-v1 binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.gvisor_containerd_shim) }}" + +- name: gVisor | Copy binaries + copy: + src: "{{ local_release_dir }}/gvisor-{{ item }}" + dest: "{{ bin_dir }}/{{ item }}" + mode: 0755 + remote_src: yes + with_items: + - runsc + - containerd-shim-runsc-v1 diff --git a/roles/container-engine/meta/main.yml b/roles/container-engine/meta/main.yml index 6218db932..2f6bff147 100644 --- a/roles/container-engine/meta/main.yml +++ b/roles/container-engine/meta/main.yml @@ -7,6 +7,14 @@ dependencies: - container-engine - kata-containers + - role: container-engine/gvisor + when: + - gvisor_enabled + - container_manager in ['docker', 'containerd'] + tags: + - container-engine + - gvisor + - role: container-engine/crun when: - crun_enabled diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index afcfc944f..7b465384d 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -53,6 +53,7 @@ kubeadm_version: "{{ kube_version }}" etcd_version: v3.4.13 crun_version: 0.19 kata_containers_version: 2.1.0 +gvisor_version: 20210518.0 # gcr and kubernetes image repo define gcr_image_repo: "gcr.io" @@ -108,6 +109,8 @@ crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/down helm_download_url: "https://get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz" crun_download_url: "https://github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}" kata_containers_download_url: "https://github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz" +# gVisor only supports amd64 and uses x86_64 to in the download link +gvisor_download_url: "https://storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}" nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz" @@ -401,6 +404,22 @@ kata_containers_binary_checksums: 2.0.4: 0 2.1.0: 0 +gvisor_runsc_binary_checksums: + arm: + 20210518.0: 0 + arm64: + 20210518.0: 0 + amd64: + 20210518.0: 907811e10576d31b6f63d2f91f1c3be2b2df94e222fb48eb509fce48cd6ae9e0 + +gvisor_containerd_shim_binary_checksums: + arm: + 20210518.0: 0 + arm64: + 20210518.0: 0 + amd64: + 20210518.0: d6deda79562cadf4fc98ccf1d6d9fd1d2dff3890b184c7b546167bd6eff241ec + nerdctl_archive_checksums: arm: 0.8.1: 27bdad3f9e2667620f70617c48d595c5c4e24a10fbcd00d31202cd8d571c9233 @@ -420,6 +439,8 @@ crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}" helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}" crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}" kata_containers_binary_checksum: "{{ kata_containers_binary_checksums[image_arch][kata_containers_version] }}" +gvisor_runsc_binary_checksum: "{{ gvisor_runsc_binary_checksums[image_arch][gvisor_version] }}" +gvisor_containerd_shim_binary_checksum: "{{ gvisor_containerd_shim_binary_checksums[image_arch][gvisor_version] }}" nerdctl_archive_checksum: "{{ nerdctl_archive_checksums[image_arch][nerdctl_version] }}" krew_archive_checksum: "{{ krew_archive_checksums[krew_version] }}" @@ -715,6 +736,32 @@ downloads: groups: - k8s_cluster + gvisor_runsc: + enabled: "{{ gvisor_enabled }}" + file: true + version: "{{ gvisor_version }}" + dest: "{{ local_release_dir }}/gvisor-runsc" + sha256: "{{ gvisor_runsc_binary_checksum }}" + url: "{{ gvisor_download_url }}/runsc" + unarchive: false + owner: "root" + mode: 755 + groups: + - k8s_cluster + + gvisor_containerd_shim: + enabled: "{{ gvisor_enabled }}" + file: true + version: "{{ gvisor_version }}" + dest: "{{ local_release_dir }}/gvisor-containerd-shim-runsc-v1" + sha256: "{{ gvisor_containerd_shim_binary_checksum }}" + url: "{{ gvisor_download_url }}/containerd-shim-runsc-v1" + unarchive: false + owner: "root" + mode: 755 + groups: + - k8s_cluster + nerdctl: file: true enabled: "{{ nerdctl_enabled }}" diff --git a/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml b/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml new file mode 100644 index 000000000..f12f5feae --- /dev/null +++ b/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml @@ -0,0 +1,33 @@ +--- +- name: gVisor | Create addon dir + file: + path: "{{ kube_config_dir}}/addons/gvisor" + owner: root + group: root + mode: 0755 + recurse: true + +- name: gVisor | Templates List + set_fact: + gvisor_templates: + - { name: runtimeclass-gvisor, file: runtimeclass-gvisor.yml, type: runtimeclass } + +- name: gVisort | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir}}/addons/gvisor/{{ item.file }}" + with_items: "{{ gvisor_templates }}" + register: gvisor_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: gVisor | Apply manifests + kube: + name: "{{ item.item.name }}" + kubectl: "{{ bin_dir}}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/gvisor/{{ item.item.file }}" + state: "latest" + with_items: "{{ gvisor_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 b/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 new file mode 100644 index 000000000..92b20b4c6 --- /dev/null +++ b/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 @@ -0,0 +1,6 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1{{ 'beta1' if kube_version is version('v1.20.0', '<') else '' }} +metadata: + name: gvisor +handler: runsc diff --git a/roles/kubernetes-apps/container_runtimes/meta/main.yml b/roles/kubernetes-apps/container_runtimes/meta/main.yml index 3c56e992e..f63d38d61 100644 --- a/roles/kubernetes-apps/container_runtimes/meta/main.yml +++ b/roles/kubernetes-apps/container_runtimes/meta/main.yml @@ -7,6 +7,13 @@ dependencies: - kata-containers - container-runtimes + - role: kubernetes-apps/container_runtimes/gvisor + when: gvisor_enabled + tags: + - apps + - gvisor + - container-runtimes + - role: kubernetes-apps/container_runtimes/crun when: crun_enabled tags: diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index ae618243d..0585a1116 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -312,6 +312,12 @@ msg: "kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details" when: kata_containers_enabled +- name: Stop if gvisor_enabled is enabled when container_manager is not containerd + assert: + that: container_manager == 'containerd' + msg: "gvisor_enabled support only compatible with containerd. See https://github.com/kubernetes-sigs/kubespray/issues/7650 for details" + when: gvisor_enabled + - name: Stop if download_localhost is enabled for Flatcar Container Linux assert: that: ansible_os_family not in ["Flatcar Container Linux by Kinvolk"] diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 2bcba470f..960aceff3 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -237,6 +237,10 @@ container_manager: docker # When enabled, it requires `container_manager` different than Docker kata_containers_enabled: false +# Enable gVisor as an additional container runtime +# gVisor is only supported with container_manager Docker or containerd +gvisor_enabled: false + # Enable crun as additional container runtime # When enabled, it requires container_manager=crio crun_enabled: false