Set containerd version to 1.4.4 (#7398)

* Set containerd version to 1.4.3

* Set containerd version to 1.4.4

Co-authored-by: Barry Melbourne <9964974+bmelbourne@users.noreply.github.com>

README.md

@@ -131,7 +131,7 @@ Note: Upstart/SysV init based OS types are not supported.
   - [kubernetes](https://github.com/kubernetes/kubernetes) v1.20.5
   - [etcd](https://github.com/coreos/etcd) v3.4.13
   - [docker](https://www.docker.com/) v19.03 (see note)
-  - [containerd](https://containerd.io/) v1.3.9
+  - [containerd](https://containerd.io/) v1.4.4
   - [cri-o](http://cri-o.io/) v1.19 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) v0.9.0

docs/containerd.md

@@ -22,13 +22,7 @@ etcd_deployment_type: host
 Example: define registry mirror for docker hub
 
 ```yaml
-containerd_config:
-  grpc:
-    max_recv_message_size: 16777216
-    max_send_message_size: 16777216
-  debug:
-    level: ""
-  registries:
+containerd_registries:
   "docker.io":
     - "https://mirror.gcr.io"
     - "https://registry-1.docker.io"

docs/upgrades.md

@@ -7,6 +7,7 @@ You can also individually control versions of components by explicitly defining
 versions. Here are all version vars for each component:
 
 * docker_version
+* containerd_version
 * kube_version
 * etcd_version
 * calico_version
@@ -283,6 +284,7 @@ installed in the Ansible playbook. The order of component installation is as
 follows:
 
 * Docker
+* Containerd
 * etcd
 * kubelet and kube-proxy
 * network_plugin (such as Calico or Weave)

docs/vars.md

@@ -18,6 +18,7 @@ Some variables of note include:
 * *docker_version* - Specify version of Docker to used (should be quoted
   string). Must match one of the keys defined for *docker_versioned_pkg*
   in `roles/container-engine/docker/vars/*.yml`.
+* *containerd_version* - Specify version of Containerd to use
 * *etcd_version* - Specify version of ETCD to use
 * *ipip* - Enables Calico ipip encapsulation by default
 * *kube_network_plugin* - Sets k8s network plugin (default Calico)
@@ -115,7 +116,8 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m
 * *docker_options* - Commonly used to set
   ``--insecure-registry=myregistry.mydomain:5000``
 * *docker_plugins* - This list can be used to define [Docker plugins](https://docs.docker.com/engine/extend/) to install.
-* *containerd_config* - Controls some parameters in containerd configuration file (usually /etc/containerd/config.toml).
+* *containerd_default_runtime* - Sets the default Containerd runtime used by the Kubernetes CRI plugin.
+* *containerd_runtimes* - Sets the Containerd runtime attributes used by the Kubernetes CRI plugin.
   [Default config](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/container-engine/containerd/defaults/main.yml) can be overriden in inventory vars.
 * *http_proxy/https_proxy/no_proxy/no_proxy_exclude_workers/additional_no_proxy* - Proxy variables for deploying behind a
   proxy. Note that no_proxy defaults to all internal cluster IPs and hostnames

inventory/sample/group_vars/all/containerd.yml

@@ -1,19 +1,30 @@
 ---
 # Please see roles/container-engine/containerd/defaults/main.yml for more configuration options
 
-# Example: define registry mirror for docker hub
+# containerd_default_runtime: "runc"
+# containerd_snapshotter: "native"
 
-# containerd_config:
-#   grpc:
-#     max_recv_message_size: 16777216
-#     max_send_message_size: 16777216
-#   debug:
-#     level: ""
-#   registries:
-#     "docker.io":
-#       - "https://mirror.gcr.io"
-#       - "https://registry-1.docker.io"
-#   max_container_log_line_size: -1
-#   metrics:
-#     address: ""
-#     grpc_histogram: false
+# containerd_runtimes:
+#   - name: runc
+#     type: "io.containerd.runc.v2"
+#     engine: ""
+#     root: ""
+# Example for Kata Containers as additional runtime:
+#   - name: kata
+#     type: "io.containerd.kata.v2"
+#     engine: ""
+#     root: ""
+
+# containerd_grpc_max_recv_message_size: 16777216
+# containerd_grpc_max_send_message_size: 16777216
+
+# containerd_debug_level: "info"
+
+# containerd_metrics_address: ""
+
+# containerd_metrics_grpc_histogram: false
+
+# containerd_registries:
+#   "docker.io": "https://registry-1.docker.io"
+
+# containerd_max_container_log_line_size: -1

inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml

@@ -198,30 +198,16 @@ container_manager: docker
 kata_containers_enabled: false
 
 ## Settings for containerd runtimes (only used when container_manager is set to containerd)
-#
-# Settings for default containerd runtime
-# containerd_default_runtime:
-#   type: io.containerd.runtime.v1.linux
-#   engine: ''
-#   root: ''
-#
-# Settings for additional runtimes for containerd configuration
 # containerd_runtimes:
-#   - name: ""
-#     type: ""
+#  - name: runc
+#    type: "io.containerd.runc.v2"
 #    engine: ""
 #    root: ""
 # Example for Kata Containers as additional runtime:
-# containerd_runtimes:
 #  - name: kata
 #    type: io.containerd.kata.v2
 #    engine: ""
 #    root: ""
-#
-# Settings for untrusted containerd runtime
-# containerd_untrusted_runtime_type: ''
-# containerd_untrusted_runtime_engine: ''
-# containerd_untrusted_runtime_root: ''
 
 kubeadm_certificate_key: "{{ lookup('password', credentials_dir + '/kubeadm_certificate_key.creds length=64 chars=hexdigits') | lower }}"
 

roles/container-engine/containerd-common/defaults/main.yml

@@ -1,3 +1,2 @@
 ---
-containerd_version: '1.3.9'
 containerd_package: 'containerd.io'

roles/container-engine/containerd-common/vars/debian-stretch.yml

@@ -0,0 +1,8 @@
+---
+containerd_versioned_pkg:
+  'latest': "{{ containerd_package }}"
+  '1.3.7': "{{ containerd_package }}=1.3.7-1"
+  '1.3.9': "{{ containerd_package }}=1.3.9-1"
+  '1.4.3': "{{ containerd_package }}=1.4.3-1"
+  'stable': "{{ containerd_package }}=1.4.3-1"
+  'edge': "{{ containerd_package }}=1.4.3-1"

roles/container-engine/containerd-common/vars/debian.yml

@@ -1,13 +1,9 @@
 ---
 containerd_versioned_pkg:
   'latest': "{{ containerd_package }}"
-  '1.2.4': "{{ containerd_package }}=1.2.4-1"
-  '1.2.5': "{{ containerd_package }}=1.2.5-1"
-  '1.2.6': "{{ containerd_package }}=1.2.6-3"
-  '1.2.10': "{{ containerd_package }}=1.2.10-3"
-  '1.2.12': "{{ containerd_package }}=1.2.12-1"
-  '1.2.13': "{{ containerd_package }}=1.2.13-2"
   '1.3.7': "{{ containerd_package }}=1.3.7-1"
   '1.3.9': "{{ containerd_package }}=1.3.9-1"
-  'stable': "{{ containerd_package }}=1.3.9-1"
-  'edge': "{{ containerd_package }}=1.3.9-1"
+  '1.4.3': "{{ containerd_package }}=1.4.3-2"
+  '1.4.4': "{{ containerd_package }}=1.4.4-1"
+  'stable': "{{ containerd_package }}=1.4.4-1"
+  'edge': "{{ containerd_package }}=1.4.4-1"

roles/container-engine/containerd-common/vars/fedora.yml

@@ -1,10 +1,9 @@
 ---
 containerd_versioned_pkg:
   'latest': "{{ containerd_package }}"
-  '1.2.10': "{{ containerd_package }}-1.2.10-3.2.fc{{ ansible_distribution_major_version }}"
-  '1.2.12': "{{ containerd_package }}-1.2.12-3.1.fc{{ ansible_distribution_major_version }}"
-  '1.2.13': "{{ containerd_package }}-1.2.13-3.2.fc{{ ansible_distribution_major_version }}"
   '1.3.7': "{{ containerd_package }}-1.3.7-3.1.fc{{ ansible_distribution_major_version }}"
   '1.3.9': "{{ containerd_package }}-1.3.9-3.1.fc{{ ansible_distribution_major_version }}"
-  'stable': "{{ containerd_package }}-1.3.9-3.1.fc{{ ansible_distribution_major_version }}"
-  'edge': "{{ containerd_package }}-1.3.9-3.1.fc{{ ansible_distribution_major_version }}"
+  '1.4.3': "{{ containerd_package }}-1.4.3-3.2.fc{{ ansible_distribution_major_version }}"
+  '1.4.4': "{{ containerd_package }}-1.4.4-3.1.fc{{ ansible_distribution_major_version }}"
+  'stable': "{{ containerd_package }}-1.4.4-3.1.fc{{ ansible_distribution_major_version }}"
+  'edge': "{{ containerd_package }}-1.4.4-3.1.fc{{ ansible_distribution_major_version }}"

roles/container-engine/containerd-common/vars/redhat.yml

@@ -1,13 +1,9 @@
 ---
 containerd_versioned_pkg:
   'latest': "{{ containerd_package }}"
-  '1.2.4': "{{ containerd_package }}-1.2.4-3.1.el7"
-  '1.2.5': "{{ containerd_package }}-1.2.5-3.1.el7"
-  '1.2.6': "{{ containerd_package }}-1.2.6-3.3.el7"
-  '1.2.10': "{{ containerd_package }}-1.2.10-3.2.el7"
-  '1.2.12': "{{ containerd_package }}-1.2.12-3.1.el7"
-  '1.2.13': "{{ containerd_package }}-1.2.13-3.2.el7"
   '1.3.7': "{{ containerd_package }}-1.3.7-3.1.el{{ ansible_distribution_major_version }}"
   '1.3.9': "{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}"
-  'stable': "{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}"
-  'edge': "{{ containerd_package }}-1.3.9-3.1.el{{ ansible_distribution_major_version }}"
+  '1.4.3': "{{ containerd_package }}-1.4.3-3.2.el{{ ansible_distribution_major_version }}"
+  '1.4.4': "{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}"
+  'stable': "{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}"
+  'edge': "{{ containerd_package }}-1.4.4-3.1.el{{ ansible_distribution_major_version }}"

roles/container-engine/containerd-common/vars/ubuntu.yml

@@ -1,10 +1,9 @@
 ---
 containerd_versioned_pkg:
   'latest': "{{ containerd_package }}"
-  '1.2.10': "{{ containerd_package }}=1.2.10-3"
-  '1.2.12': "{{ containerd_package }}=1.2.12-1"
-  '1.2.13': "{{ containerd_package }}=1.2.13-2"
   '1.3.7': "{{ containerd_package }}=1.3.7-1"
   '1.3.9': "{{ containerd_package }}=1.3.9-1"
-  'stable': "{{ containerd_package }}=1.3.9-1"
-  'edge': "{{ containerd_package }}=1.3.9-1"
+  '1.4.3': "{{ containerd_package }}=1.4.3-2"
+  '1.4.4': "{{ containerd_package }}=1.4.4-1"
+  'stable': "{{ containerd_package }}=1.4.4-1"
+  'edge': "{{ containerd_package }}=1.4.4-1"

roles/container-engine/containerd/defaults/main.yml

@@ -1,27 +1,40 @@
 ---
-# The root directory for containerd metadata
-containerd_metadata_root_dir: /var/lib/containerd
-# The state directory for containerd
-containerd_state_dir: /run/containerd
 
-containerd_config:
-  grpc:
-    max_recv_message_size: 16777216
-    max_send_message_size: 16777216
-  debug:
-    level: ""
-  registries:
+containerd_default_runtime: "runc"
+# containerd_snapshotter: "native"
+
+containerd_runtimes:
+  - name: runc
+    type: "io.containerd.runc.v2"
+    engine: ""
+    root: ""
+    options:
+      systemdCgroup: "true"
+# Example for Kata Containers as additional runtime:
+#  - name: kata
+#    type: "io.containerd.kata.v2"
+#    engine: ""
+#    root: ""
+
+containerd_grpc_max_recv_message_size: 16777216
+containerd_grpc_max_send_message_size: 16777216
+
+containerd_debug_level: "info"
+
+containerd_metrics_address: ""
+
+containerd_metrics_grpc_histogram: false
+
+containerd_registries:
   "docker.io": "https://registry-1.docker.io"
-  max_container_log_line_size: -1
-  # containerd:
-  #   snapshotter: native
+
+containerd_max_container_log_line_size: -1
 
 containerd_cfg_dir: /etc/containerd
 
 # Path to runc binary
 runc_binary: /usr/bin/runc
 
-
 yum_repo_dir: /etc/yum.repos.d
 
 # Optional values for containerd apt repo
@@ -36,38 +49,18 @@ containerd_repo_info:
 
 # Ubuntu docker-ce repo
 containerd_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu"
-containerd_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg'
-containerd_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
-containerd_ubuntu_repo_component: 'stable'
+containerd_ubuntu_repo_gpgkey: "https://download.docker.com/linux/ubuntu/gpg"
+containerd_ubuntu_repo_repokey: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88"
+containerd_ubuntu_repo_component: "stable"
 
 # Debian docker-ce repo
-containerd_debian_repo_base_url: 'https://download.docker.com/linux/debian'
-containerd_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg'
-containerd_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
-containerd_debian_repo_component: 'stable'
+containerd_debian_repo_base_url: "https://download.docker.com/linux/debian"
+containerd_debian_repo_gpgkey: "https://download.docker.com/linux/debian/gpg"
+containerd_debian_repo_repokey: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88"
+containerd_debian_repo_component: "stable"
 
 # Fedora docker-ce repo
-containerd_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable'
-containerd_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg'
-containerd_fedora_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88'
-containerd_fedora_repo_component: 'stable'
-
-containerd_default_runtime:
-  type: io.containerd.runtime.v1.linux
-  engine: ''
-  root: ''
-
-# Additional runtimes for containerd configuration
-#
-# Example for Kata Containers:
-# containerd_runtimes:
-#   - name: kata
-#     type: io.containerd.kata.v2
-#     engine: ""
-#     root: ""
-#     privileged_without_host_devices: true
-containerd_runtimes: []
-
-containerd_untrusted_runtime_type: ''
-containerd_untrusted_runtime_engine: ''
-containerd_untrusted_runtime_root: ''
+containerd_fedora_repo_base_url: "https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable"
+containerd_fedora_repo_gpgkey: "https://download.docker.com/linux/fedora/gpg"
+containerd_fedora_repo_repokey: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88"
+containerd_fedora_repo_component: "stable"

roles/container-engine/containerd/templates/config.toml.j2

@@ -1,80 +1,41 @@
-# persistent data location
-root = "{{ containerd_metadata_root_dir }}"
-# runtime state information
-state = "{{ containerd_state_dir }}"
+version = 2
 
-# Kubernetes doesn't use containerd restart manager.
-disabled_plugins = ["restart"]
+[grpc]
+  max_recv_message_size = {{ containerd_grpc_max_recv_message_size | default(16777216) }}
+  max_send_message_size = {{ containerd_grpc_max_send_message_size | default(16777216) }}
 
 [debug]
-  level = "{{ containerd_config.debug.level | default("") }}"
+  level = "{{ containerd_debug_level | default('info') }}"
 
-{% if 'grpc' in containerd_config %}
-[grpc]
-{% for param, value in containerd_config.grpc.items() %}
-  {{ param }} = {{ value }}
-{% endfor %}
-{% endif %}
+[metrics]
+  address = "{{ containerd_metrics_address | default('') }}"
+  grpc_histogram = {{ containerd_metrics_grpc_histogram | default(false) | lower }}
 
-[plugins.linux]
-  shim = "/usr/bin/containerd-shim"
-  runtime = "{{ runc_binary }}"
-
-[plugins.cri]
-  stream_server_address = "127.0.0.1"
-  max_container_log_line_size = {{ containerd_config.max_container_log_line_size }}
+[plugins]
+  [plugins."io.containerd.grpc.v1.cri"]
     sandbox_image = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
-  systemd_cgroup = {{ containerd_use_systemd_cgroup|lower }}
-
-[plugins.cri.cni]
-  bin_dir = "/opt/cni/bin"
-  conf_dir = "/etc/cni/net.d"
-  conf_template = ""
-
-{% if 'containerd' in containerd_config %}
-[plugins.cri.containerd]
-{% for param, value in containerd_config.containerd.items() %}
-  {{ param }} = "{{ value }}"
-{% endfor %}
-{% endif %}
-
-[plugins.cri.containerd.default_runtime]
-  runtime_type = "{{ containerd_default_runtime.type }}"
-  runtime_engine = "{{ containerd_default_runtime.engine }}"
-  runtime_root = "{{ containerd_default_runtime.root }}"
-  privileged_without_host_devices = {{ containerd_default_runtime.privileged_without_host_devices|default(false)|lower }}
-
-{% if kata_containers_enabled %}
-[plugins.cri.containerd.runtimes.kata-qemu]
-  runtime_type = "io.containerd.kata-qemu.v2"
-  [plugins.cri.containerd.runtimes.kata-qemu.options]
-    ConfigPath = "/etc/kata-containers/configuration-qemu.toml"
-{% endif %}
-
+    max_container_log_line_size = {{ containerd_max_container_log_line_size }}
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "{{ containerd_default_runtime | default('runc') }}"
+      snapshotter = "{{ containerd_snapshotter | default('overlayfs') }}"
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
 {% for runtime in containerd_runtimes %}
-[plugins.cri.containerd.runtimes.{{ runtime.name }}]
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}]
           runtime_type = "{{ runtime.type }}"
           runtime_engine = "{{ runtime.engine }}"
           runtime_root = "{{ runtime.root }}"
-  privileged_without_host_devices = {{ runtime.privileged_without_host_devices|default(false)|lower }}
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.{{ runtime.name }}.options]
+{% for key, value in runtime.options.items() %}
+            {{ key }} = {{ value }}
 {% endfor %}
-
-[plugins.cri.containerd.untrusted_workload_runtime]
-  runtime_type = "{{ containerd_untrusted_runtime_type }}"
-  runtime_engine = "{{ containerd_untrusted_runtime_engine }}"
-  runtime_root = "{{ containerd_untrusted_runtime_root }}"
-
-{% if 'registries' in containerd_config %}
-[plugins.cri.registry]
-[plugins.cri.registry.mirrors]
-{% for registry, addr in containerd_config.registries.items() %}
-[plugins.cri.registry.mirrors."{{ registry }}"]
+{% endfor %}
+{% if kata_containers_enabled %}
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu]
+          runtime_type = "io.containerd.kata-qemu.v2"
+{% endif %}
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+{% for registry, addr in containerd_registries.items() %}
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry }}"]
           endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"]
 {% endfor %}
-{% endif %}
-
-{% if 'metrics' in containerd_config %}
-[metrics]
-  address = "{{ containerd_config.metrics.address | default('') }}"
-  grpc_histogram = {{ containerd_config.metrics.grpc_histogram | default(false) | lower }}
-{% endif %}

roles/kubernetes/preinstall/tasks/0020-verify-settings.yml

@@ -298,3 +298,19 @@
     that: ansible_os_family not in ["Flatcar Container Linux by Kinvolk"]
     msg: "download_run_once not supported for Flatcar Container Linux"
   when: download_run_once or download_force_cache
+
+- name: Ensure minimum containerd version
+  assert:
+    that: containerd_version is version(containerd_min_version_required, '>=')
+    msg: "containerd_version is too low. Minimum version {{ containerd_min_version_required }}"
+  run_once: yes
+  when:
+    - container_manager == 'containerd'
+
+- name: Stop if using deprecated containerd_config variable
+  assert:
+    that: containerd_config is not defined
+    msg: "Variable containerd_config is now deprecated. See https://github.com/kubernetes-sigs/kubespray/blob/master/inventory/sample/group_vars/all/containerd.yml for details."
+  when:
+    - containerd_config is defined
+    - not ignore_assert_errors

roles/kubespray-defaults/defaults/main.yaml

@@ -309,7 +309,13 @@ docker_plugins: []
 etcd_kubeadm_enabled: false
 
 # Containerd options
-containerd_use_systemd_cgroup: false
+containerd_version: >-
+  {%- if ansible_os_family == "Debian" and ansible_distribution_release == "stretch" -%}
+  1.4.3
+  {%- else -%}
+  1.4.4
+  {%- endif -%}
+containerd_use_systemd_cgroup: true
 
 # Settings for containerized control plane (etcd/kubelet/secrets)
 # deployment type for legacy etcd mode

roles/kubespray-defaults/vars/main.yml

@@ -5,3 +5,5 @@ kube_proxy_deployed: "{{ 'addon/kube-proxy' not in kubeadm_init_phases_skip }}"
 
 # The lowest version allowed to upgrade from (same as calico_version in the previous branch)
 calico_min_version_required: "v3.15.2"
+
+containerd_min_version_required: "1.3.7"

tests/common/_docker_hub_registry_mirror.yml

@@ -2,17 +2,15 @@
 docker_registry_mirrors:
   - "https://mirror.gcr.io"
 
-containerd_config:
-  grpc:
-    max_recv_message_size: 16777216
-    max_send_message_size: 16777216
-  debug:
-    level: ""
-  registries:
+containerd_grpc_max_recv_message_size: 16777216
+containerd_grpc_max_send_message_size: 16777216
+
+containerd_registries:
   "docker.io":
     - "https://mirror.gcr.io"
     - "https://registry-1.docker.io"
-  max_container_log_line_size: -1
+
+containerd_max_container_log_line_size: -1
 
 crio_registries_mirrors:
   - prefix: docker.io

tests/files/packet_centos7-flannel-containerd-addons-ha.yml

@@ -38,7 +38,3 @@ tls_cipher_suites:
 etcd_tls_cipher_suites:
   - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
   - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
-
-# Containerd path
-containerd_metadata_root_dir: /var/data/containerd
-containerd_state_dir: /run/cri/containerd