gVisor: initial support for gVisor container runtime (#7661)

* Docker/Containerd: move downloads urls to containerd-common

* gVisor: initial support for gVisor container runtime
This commit is contained in:
Cristian Calin 2021-06-21 15:18:51 +03:00 committed by GitHub
parent 3fe6dbb65c
commit 282a27a07c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 336 additions and 14 deletions

View file

@ -41,6 +41,7 @@
* [Containerd](docs/containerd.md)
* [CRI-O](docs/cri-o.md)
* [Kata Containers](docs/kata-containers.md)
* [gVisor](docs/gvisor.md)
* Advanced
* [Proxy](/docs/proxy.md)
* [Downloads](docs/downloads.md)

16
docs/gvisor.md Normal file
View file

@ -0,0 +1,16 @@
# gVisor
[gVisor](https://gvisor.dev/docs/) is an application kernel, written in Go, that implements a substantial portion of the Linux system call interface. It provides an additional layer of isolation between running applications and the host operating system.
gVisor includes an Open Container Initiative (OCI) runtime called runsc that makes it easy to work with existing container tooling. The runsc runtime integrates with Docker and Kubernetes, making it simple to run sandboxed containers.
## Usage
To enable gVisor you should be using a container manager that is compatible with selecting the [RuntimeClass](https://kubernetes.io/docs/concepts/containers/runtime-class/) such as `containerd`.
Containerd support:
```yaml
container_manager: containerd
gvisor_enabled: true
```

View file

@ -1,2 +1,17 @@
---
containerd_package: 'containerd.io'
# Fedora docker-ce repo
docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable'
docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg'
# CentOS/RedHat docker-ce repo
docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable'
docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg'
# Ubuntu docker-ce repo
docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg'
docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
# Debian docker-ce repo
docker_debian_repo_base_url: "https://download.docker.com/linux/debian"
docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg'
docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'

View file

@ -35,6 +35,10 @@ oom_score = {{ containerd_oom_score }}
{% if kata_containers_enabled %}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu]
runtime_type = "io.containerd.kata-qemu.v2"
{% endif %}
{% if gvisor_enabled %}
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc]
runtime_type = "io.containerd.runsc.v1"
{% endif %}
[plugins."io.containerd.grpc.v1.cri".registry]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]

View file

@ -15,20 +15,6 @@ docker_cgroup_driver: systemd
yum_repo_dir: /etc/yum.repos.d
# Fedora docker-ce repo
docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable'
docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg'
# CentOS/RedHat docker-ce repo
docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable'
docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg'
# Ubuntu docker-ce repo
docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg'
docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
# Debian docker-ce repo
docker_debian_repo_base_url: "https://download.docker.com/linux/debian"
docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg'
docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
docker_bin_dir: "/usr/bin"
# flag to enable/disable docker cleanup

View file

@ -0,0 +1,11 @@
---
- name: Converge
hosts: all
become: true
vars:
gvisor_enabled: true
container_manager: containerd
roles:
- role: kubespray-defaults
- role: containerd
- role: gvisor

View file

@ -0,0 +1,17 @@
{
"cniVersion": "0.2.0",
"name": "mynet",
"type": "bridge",
"bridge": "cni0",
"isGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"subnet": "172.19.0.0/24",
"routes": [
{
"dst": "0.0.0.0/0"
}
]
}
}

View file

@ -0,0 +1,10 @@
{
"metadata": {
"name": "gvisor1"
},
"image": {
"image": "docker.io/library/hello-world:latest"
},
"log_path": "gvisor1.0.log",
"linux": {}
}

View file

@ -0,0 +1,10 @@
{
"metadata": {
"name": "gvisor1",
"namespace": "default",
"attempt": 1,
"uid": "hdishd83djaidwnduwk28bcsb"
},
"linux": {},
"log_directory": "/tmp"
}

View file

@ -0,0 +1,44 @@
---
driver:
name: vagrant
provider:
name: libvirt
options:
driver: kvm
lint: |
set -e
yamllint -c ../../../.yamllint .
platforms:
- name: ubuntu20
box: generic/ubuntu2004
cpus: 1
memory: 1024
nested: true
groups:
- kube_control_plane
- name: centos8
box: generic/centos8
cpus: 1
memory: 1024
nested: true
groups:
- kube_control_plane
provisioner:
name: ansible
env:
ANSIBLE_ROLES_PATH: ../../../../
config_options:
defaults:
callback_whitelist: profile_tasks
lint:
name: ansible-lint
options:
c: ../../../.ansible-lint
inventory:
group_vars:
all:
become: true
verifier:
name: testinfra
lint:
name: flake8

View file

@ -0,0 +1,48 @@
---
- name: Prepare generic
hosts: all
become: true
roles:
- role: kubespray-defaults
- role: bootstrap-os
- role: ../adduser
user: "{{ addusers.kube }}"
tasks:
- include_tasks: "../../../../download/tasks/download_file.yml"
vars:
download: "{{ download_defaults | combine(downloads.cni) }}"
- name: Prepare container runtime
hosts: all
become: true
vars:
container_manager: containerd
kube_network_plugin: cni
roles:
- role: kubespray-defaults
- role: ../network_plugin/cni
- role: crictl
tasks:
- name: Copy test container files
copy:
src: "{{ item }}"
dest: "/tmp/{{ item }}"
owner: root
mode: 0644
with_items:
- container.json
- sandbox.json
- name: Create /etc/cni/net.d directory
file:
path: /etc/cni/net.d
state: directory
owner: kube
mode: 0755
- name: Setup CNI
copy:
src: "{{ item }}"
dest: "/etc/cni/net.d/{{ item }}"
owner: root
mode: 0644
with_items:
- 10-mynet.conf

View file

@ -0,0 +1,29 @@
import os
import testinfra.utils.ansible_runner
testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
def test_run(host):
gvisorruntime = "/usr/local/bin/runsc"
with host.sudo():
cmd = host.command(gvisorruntime + " --version")
assert cmd.rc == 0
assert "runsc version" in cmd.stdout
def test_run_pod(host):
runtime = "runsc"
run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime)
with host.sudo():
cmd = host.command(run_command)
assert cmd.rc == 0
with host.sudo():
log_f = host.file("/tmp/gvisor1.0.log")
assert log_f.exists
assert b"Hello from Docker!" in log_f.content

View file

@ -0,0 +1,20 @@
---
- name: gVisor | Download runsc binary
include_tasks: "../../../download/tasks/download_file.yml"
vars:
download: "{{ download_defaults | combine(downloads.gvisor_runsc) }}"
- name: gVisor | Download containerd-shim-runsc-v1 binary
include_tasks: "../../../download/tasks/download_file.yml"
vars:
download: "{{ download_defaults | combine(downloads.gvisor_containerd_shim) }}"
- name: gVisor | Copy binaries
copy:
src: "{{ local_release_dir }}/gvisor-{{ item }}"
dest: "{{ bin_dir }}/{{ item }}"
mode: 0755
remote_src: yes
with_items:
- runsc
- containerd-shim-runsc-v1

View file

@ -7,6 +7,14 @@ dependencies:
- container-engine
- kata-containers
- role: container-engine/gvisor
when:
- gvisor_enabled
- container_manager in ['docker', 'containerd']
tags:
- container-engine
- gvisor
- role: container-engine/crun
when:
- crun_enabled

View file

@ -53,6 +53,7 @@ kubeadm_version: "{{ kube_version }}"
etcd_version: v3.4.13
crun_version: 0.19
kata_containers_version: 2.1.0
gvisor_version: 20210518.0
# gcr and kubernetes image repo define
gcr_image_repo: "gcr.io"
@ -108,6 +109,8 @@ crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/down
helm_download_url: "https://get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
crun_download_url: "https://github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
kata_containers_download_url: "https://github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz"
# gVisor only supports amd64 and uses x86_64 to in the download link
gvisor_download_url: "https://storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}"
nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz"
@ -401,6 +404,22 @@ kata_containers_binary_checksums:
2.0.4: 0
2.1.0: 0
gvisor_runsc_binary_checksums:
arm:
20210518.0: 0
arm64:
20210518.0: 0
amd64:
20210518.0: 907811e10576d31b6f63d2f91f1c3be2b2df94e222fb48eb509fce48cd6ae9e0
gvisor_containerd_shim_binary_checksums:
arm:
20210518.0: 0
arm64:
20210518.0: 0
amd64:
20210518.0: d6deda79562cadf4fc98ccf1d6d9fd1d2dff3890b184c7b546167bd6eff241ec
nerdctl_archive_checksums:
arm:
0.8.1: 27bdad3f9e2667620f70617c48d595c5c4e24a10fbcd00d31202cd8d571c9233
@ -420,6 +439,8 @@ crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}"
helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}"
crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}"
kata_containers_binary_checksum: "{{ kata_containers_binary_checksums[image_arch][kata_containers_version] }}"
gvisor_runsc_binary_checksum: "{{ gvisor_runsc_binary_checksums[image_arch][gvisor_version] }}"
gvisor_containerd_shim_binary_checksum: "{{ gvisor_containerd_shim_binary_checksums[image_arch][gvisor_version] }}"
nerdctl_archive_checksum: "{{ nerdctl_archive_checksums[image_arch][nerdctl_version] }}"
krew_archive_checksum: "{{ krew_archive_checksums[krew_version] }}"
@ -715,6 +736,32 @@ downloads:
groups:
- k8s_cluster
gvisor_runsc:
enabled: "{{ gvisor_enabled }}"
file: true
version: "{{ gvisor_version }}"
dest: "{{ local_release_dir }}/gvisor-runsc"
sha256: "{{ gvisor_runsc_binary_checksum }}"
url: "{{ gvisor_download_url }}/runsc"
unarchive: false
owner: "root"
mode: 755
groups:
- k8s_cluster
gvisor_containerd_shim:
enabled: "{{ gvisor_enabled }}"
file: true
version: "{{ gvisor_version }}"
dest: "{{ local_release_dir }}/gvisor-containerd-shim-runsc-v1"
sha256: "{{ gvisor_containerd_shim_binary_checksum }}"
url: "{{ gvisor_download_url }}/containerd-shim-runsc-v1"
unarchive: false
owner: "root"
mode: 755
groups:
- k8s_cluster
nerdctl:
file: true
enabled: "{{ nerdctl_enabled }}"

View file

@ -0,0 +1,33 @@
---
- name: gVisor | Create addon dir
file:
path: "{{ kube_config_dir}}/addons/gvisor"
owner: root
group: root
mode: 0755
recurse: true
- name: gVisor | Templates List
set_fact:
gvisor_templates:
- { name: runtimeclass-gvisor, file: runtimeclass-gvisor.yml, type: runtimeclass }
- name: gVisort | Create manifests
template:
src: "{{ item.file }}.j2"
dest: "{{ kube_config_dir}}/addons/gvisor/{{ item.file }}"
with_items: "{{ gvisor_templates }}"
register: gvisor_manifests
when:
- inventory_hostname == groups['kube_control_plane'][0]
- name: gVisor | Apply manifests
kube:
name: "{{ item.item.name }}"
kubectl: "{{ bin_dir}}/kubectl"
resource: "{{ item.item.type }}"
filename: "{{ kube_config_dir }}/addons/gvisor/{{ item.item.file }}"
state: "latest"
with_items: "{{ gvisor_manifests.results }}"
when:
- inventory_hostname == groups['kube_control_plane'][0]

View file

@ -0,0 +1,6 @@
---
kind: RuntimeClass
apiVersion: node.k8s.io/v1{{ 'beta1' if kube_version is version('v1.20.0', '<') else '' }}
metadata:
name: gvisor
handler: runsc

View file

@ -7,6 +7,13 @@ dependencies:
- kata-containers
- container-runtimes
- role: kubernetes-apps/container_runtimes/gvisor
when: gvisor_enabled
tags:
- apps
- gvisor
- container-runtimes
- role: kubernetes-apps/container_runtimes/crun
when: crun_enabled
tags:

View file

@ -312,6 +312,12 @@
msg: "kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details"
when: kata_containers_enabled
- name: Stop if gvisor_enabled is enabled when container_manager is not containerd
assert:
that: container_manager == 'containerd'
msg: "gvisor_enabled support only compatible with containerd. See https://github.com/kubernetes-sigs/kubespray/issues/7650 for details"
when: gvisor_enabled
- name: Stop if download_localhost is enabled for Flatcar Container Linux
assert:
that: ansible_os_family not in ["Flatcar Container Linux by Kinvolk"]

View file

@ -237,6 +237,10 @@ container_manager: docker
# When enabled, it requires `container_manager` different than Docker
kata_containers_enabled: false
# Enable gVisor as an additional container runtime
# gVisor is only supported with container_manager Docker or containerd
gvisor_enabled: false
# Enable crun as additional container runtime
# When enabled, it requires container_manager=crio
crun_enabled: false