[docker] add support for cri-dockerd as a replacement for dockershim (#8623)

.gitignore

@@ -102,7 +102,6 @@ ENV/
 
 # molecule
 roles/**/molecule/**/__pycache__/
-roles/**/molecule/**/*.conf
 
 # macOS
 .DS_Store

docs/_sidebar.md

@@ -37,6 +37,7 @@
   * [Amazon Linux 2](docs/amazonlinux.md)
 * CRI
   * [Containerd](docs/containerd.md)
+  * [Docker](docs/docker.md)
   * [CRI-O](docs/cri-o.md)
   * [Kata Containers](docs/kata-containers.md)
   * [gVisor](docs/gvisor.md)

docs/docker.md

@@ -0,0 +1,105 @@
+# Docker support
+
+The docker runtime is supported by kubespray and while the `dockershim` is deprecated to be removed in kubernetes 1.24+ there are alternative ways to use docker such as through the [cri-dockerd](https://github.com/Mirantis/cri-dockerd) project supported by Mirantis.
+
+Using the docker container manager:
+
+```yaml
+container_manager: docker
+```
+
+Using `cri-dockerd` instead of `dockershim`:
+
+```yaml
+cri_dockerd_enabled: false
+```
+
+*Note:* The `cri_dockerd_enabled: true` setting will become the default in a future kubespray release once kubespray 1.24+ is supported and `dockershim` is removed. At that point, changing this option will be deprecated and silently ignored.
+
+Enabling the `overlay2` graph driver:
+
+```yaml
+docker_storage_options: -s overlay2
+```
+
+Enabling `docker_container_storage_setup`, it will configure devicemapper driver on Centos7 or RedHat7.
+Deployers must be define a disk path for `docker_container_storage_setup_devs`, otherwise docker-storage-setup will be executed incorrectly.
+
+```yaml
+docker_container_storage_setup: true
+docker_container_storage_setup_devs: /dev/vdb
+```
+
+Changing the Docker cgroup driver (native.cgroupdriver); valid options are `systemd` or `cgroupfs`, default is `systemd`:
+
+```yaml
+docker_cgroup_driver: systemd
+```
+
+If you have more than 3 nameservers kubespray will only use the first 3 else it will fail. Set the `docker_dns_servers_strict` to `false` to prevent deployment failure.
+
+```yaml
+docker_dns_servers_strict: false
+```
+
+Set the path used to store Docker data:
+
+```yaml
+docker_daemon_graph: "/var/lib/docker"
+```
+
+Changing the docker daemon iptables support:
+
+```yaml
+docker_iptables_enabled: "false"
+```
+
+Docker log options:
+
+```yaml
+# Rotate container stderr/stdout logs at 50m and keep last 5
+docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5"
+```
+
+Changre the docker `bin_dir`, this should not be changed unless you use a custom docker package:
+
+```yaml
+docker_bin_dir: "/usr/bin"
+```
+
+To keep docker packages after installation; speeds up repeated ansible provisioning runs when '1'.
+kubespray deletes the docker package on each run, so caching the package makes sense:
+
+```yaml
+docker_rpm_keepcache: 1
+```
+
+Allowing insecure-registry access to self hosted registries. Can be ipaddress and domain_name.
+
+```yaml
+## example define 172.19.16.11 or mirror.registry.io
+docker_insecure_registries:
+  - mirror.registry.io
+  - 172.19.16.11
+```
+
+Adding other registry, i.e. China registry mirror:
+
+```yaml
+docker_registry_mirrors:
+  - https://registry.docker-cn.com
+  - https://mirror.aliyuncs.com
+```
+
+Overriding default system MountFlags value. This option takes a mount propagation flag: `shared`, `slave` or `private`, which control whether mounts in the file system namespace set up for docker will receive or propagate mounts and unmounts. Leave empty for system default:
+
+```yaml
+docker_mount_flags:
+```
+
+Adding extra options to pass to the docker daemon:
+
+```yaml
+## This string should be exactly as you wish it to appear.
+docker_options: ""
+```

inventory/sample/group_vars/all/docker.yml

@@ -57,3 +57,6 @@ docker_rpm_keepcache: 1
 ## A string of extra options to pass to the docker daemon.
 ## This string should be exactly as you wish it to appear.
 # docker_options: ""
+
+## Use CRI-DockerD instead of dockershim
+# cri_dockerd_enabled: false

roles/container-engine/cri-dockerd/handlers/main.yml

@@ -0,0 +1,21 @@
+---
+- name: restart cri-dockerd
+  command: /bin/true
+  notify:
+    - cri-dockerd | reload systemd
+    - cri-dockerd | reload cri-dockerd.socket
+    - cri-dockerd | reload cri-dockerd.service
+
+- name: cri-dockerd | reload systemd
+  systemd:
+    daemon_reload: true
+
+- name: cri-dockerd | reload cri-dockerd.socket
+  service:
+    name: cri-dockerd.socket
+    state: restarted
+
+- name: cri-dockerd | reload cri-dockerd.service
+  service:
+    name: cri-dockerd.service
+    state: restarted

roles/container-engine/cri-dockerd/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - role: container-engine/docker
+  - role: container-engine/crictl

roles/container-engine/cri-dockerd/molecule/default/converge.yml

@@ -0,0 +1,10 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+  vars:
+    container_manager: docker
+    cri_dockerd_enabled: true
+  roles:
+    - role: kubespray-defaults
+    - role: container-engine/cri-dockerd

roles/container-engine/cri-dockerd/molecule/default/files/10-mynet.conf

@@ -0,0 +1,17 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "mynet",
+  "type": "bridge",
+  "bridge": "cni0",
+  "isGateway": true,
+  "ipMasq": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "172.19.0.0/24",
+    "routes": [
+      {
+        "dst": "0.0.0.0/0"
+      }
+    ]
+  }
+}

roles/container-engine/cri-dockerd/molecule/default/files/container.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "cri-dockerd1"
+  },
+  "image": {
+    "image": "quay.io/kubespray/hello-world:latest"
+  },
+  "log_path": "cri-dockerd1.0.log",
+  "linux": {}
+}

roles/container-engine/cri-dockerd/molecule/default/files/sandbox.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "cri-dockerd1",
+    "namespace": "default",
+    "attempt": 1,
+    "uid": "hdishd83djaidwnduwk28bcsb"
+  },
+  "linux": {},
+  "log_directory": "/tmp"
+}

roles/container-engine/cri-dockerd/molecule/default/molecule.yml

@@ -0,0 +1,45 @@
+---
+driver:
+  name: vagrant
+  provider:
+    name: libvirt
+    options:
+      driver: kvm
+lint: |
+  set -e
+  yamllint -c ../../../.yamllint .
+platforms:
+  - name: almalinux8
+    box: almalinux/8
+    cpus: 1
+    memory: 1024
+    nested: true
+    groups:
+      - kube_control_plane
+  - name: ubuntu20
+    box: generic/ubuntu2004
+    cpus: 1
+    memory: 1024
+    nested: true
+    groups:
+      - kube_control_plane
+provisioner:
+  name: ansible
+  env:
+    ANSIBLE_ROLES_PATH: ../../../../
+  config_options:
+    defaults:
+      callback_whitelist: profile_tasks
+      timeout: 120
+  lint:
+    name: ansible-lint
+    options:
+      c: ../../../.ansible-lint
+  inventory:
+    group_vars:
+      all:
+        become: true
+verifier:
+  name: testinfra
+  lint:
+    name: flake8

roles/container-engine/cri-dockerd/molecule/default/prepare.yml

@@ -0,0 +1,47 @@
+---
+- name: Prepare
+  hosts: all
+  become: true
+  roles:
+    - role: kubespray-defaults
+    - role: bootstrap-os
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare container runtime
+  hosts: all
+  become: true
+  vars:
+    container_manager: containerd
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray-defaults
+    - role: network_plugin/cni
+  tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - container.json
+        - sandbox.json
+    - name: Create /etc/cni/net.d directory
+      file:
+        path: /etc/cni/net.d
+        state: directory
+        owner: kube
+        mode: 0755
+    - name: Setup CNI
+      copy:
+        src: "{{ item }}"
+        dest: "/etc/cni/net.d/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - 10-mynet.conf

roles/container-engine/cri-dockerd/molecule/default/tests/test_default.py

@@ -0,0 +1,19 @@
+import os
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
+
+
+def test_run_pod(host):
+    run_command = "/usr/local/bin/crictl run --with-pull /tmp/container.json /tmp/sandbox.json"
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/cri-dockerd1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker" in log_f.content

roles/container-engine/cri-dockerd/tasks/main.yml

@@ -0,0 +1,25 @@
+---
+- name: runc | Download cri-dockerd binary
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.cri_dockerd) }}"
+
+- name: Copy cri-dockerd binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/cri-dockerd"
+    dest: "{{ bin_dir }}/cri-dockerd"
+    mode: 0755
+    remote_src: true
+  notify:
+    - restart cri-dockerd
+
+- name: Generate cri-dockerd systemd unit files
+  template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: 0644
+  with_items:
+    - cri-dockerd.service
+    - cri-dockerd.socket
+  notify:
+    - restart cri-dockerd

roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2

@@ -0,0 +1,39 @@
+[Unit]
+Description=CRI Interface for Docker Application Container Engine
+Documentation=https://docs.mirantis.com
+After=network-online.target firewalld.service docker.service
+Wants=network-online.target docker.service
+Requires=cri-dockerd.socket
+
+[Service]
+Type=notify
+ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint fd:// --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnet }}
+ExecReload=/bin/kill -s HUP $MAINPID
+TimeoutSec=0
+RestartSec=2
+Restart=always
+
+# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
+# Both the old, and new location are accepted by systemd 229 and up, so using the old location
+# to make them work for either version of systemd.
+StartLimitBurst=3
+
+# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
+# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
+# this option work for either version of systemd.
+StartLimitInterval=60s
+
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+
+# Comment TasksMax if your systemd version does not support it.
+# Only systemd 226 and above support this option.
+TasksMax=infinity
+Delegate=yes
+KillMode=process
+
+[Install]
+WantedBy=multi-user.target

roles/container-engine/cri-dockerd/templates/cri-dockerd.socket.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=CRI Docker Socket for the API
+PartOf=cri-dockerd.service
+
+[Socket]
+ListenStream=%t/cri-dockerd.sock
+SocketMode=0660
+SocketUser=root
+SocketGroup=docker
+
+[Install]
+WantedBy=sockets.target

roles/container-engine/meta/main.yml

@@ -56,3 +56,11 @@ dependencies:
     tags:
       - container-engine
       - docker
+
+  - role: container-engine/cri-dockerd
+    when:
+      - cri_dockerd_enabled
+      - container_manager == 'docker'
+    tags:
+      - container-engine
+      - docker

roles/download/defaults/main.yml

@@ -75,6 +75,7 @@ kata_containers_version: 2.2.3
 youki_version: 0.0.1
 gvisor_version: 20210921
 containerd_version: 1.6.1
+cri_dockerd_version: v0.2.0
 
 # this is relevant when container_manager == 'docker'
 docker_containerd_version: 1.4.12
@@ -152,6 +153,7 @@ gvisor_containerd_shim_runsc_download_url: "https://storage.googleapis.com/gviso
 nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
 krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz"
 containerd_download_url: "https://github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
+cri_dockerd_download_url: "https://github.com/Mirantis/cri-dockerd/releases/download/{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}-linux-{{ image_arch }}.tar.gz"
 
 crictl_checksums:
   arm:
@@ -577,6 +579,16 @@ helm_archive_checksums:
   ppc64le:
     v3.8.0: 5070fa5188e7bc798dd54bc1ea9fc4cda623d9ff45eedb05ec93db234309f391
 
+cri_dockerd_archive_checksums:
+  arm:
+    v0.2.0: 0
+  arm64:
+    v0.2.0: 6ddcb77bc7ca9d93008fdcb81659e3acfb45105dc935782030afb7dab4a3250b
+  amd64:
+    v0.2.0: ad0e5f2ac6e7a57af4b560d1638ee26baab68c56a49ff3337e10ff9cea0a3539
+  ppc64le:
+    v0.2.0: 0
+
 runc_checksums:
   arm:
     v1.0.2: 0
@@ -738,6 +750,7 @@ kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}"
 calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}"
 calico_crds_archive_checksum: "{{ calico_crds_archive_checksums[calico_version] }}"
 crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}"
+cri_dockerd_archive_checksum: "{{ cri_dockerd_archive_checksums[image_arch][cri_dockerd_version] }}"
 helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}"
 runc_binary_checksum: "{{ runc_checksums[image_arch][runc_version] }}"
 crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}"
@@ -1011,7 +1024,7 @@ downloads:
 
   crictl:
     file: true
-    enabled: "{{ container_manager in ['crio', 'cri', 'containerd'] }}"
+    enabled: "{{ container_manager in ['crio', 'cri', 'containerd'] or cri_dockerd_enabled }}"
     version: "{{ crictl_version }}"
     dest: "{{ local_release_dir }}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz"
     sha256: "{{ crictl_binary_checksum }}"
@@ -1022,6 +1035,19 @@ downloads:
     groups:
     - k8s_cluster
 
+  cri_dockerd:
+    file: true
+    enabled: "{{ cri_dockerd_enabled }}"
+    version: "{{ cri_dockerd_version }}"
+    dest: "{{ local_release_dir }}/cri-dockerd-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz"
+    sha256: "{{ cri_dockerd_archive_checksum }}"
+    url: "{{ cri_dockerd_download_url }}"
+    unarchive: true
+    owner: "root"
+    mode: "0755"
+    groups:
+    - k8s_cluster
+
   crun:
     file: true
     enabled: "{{ crun_enabled }}"

roles/kubernetes/node/templates/kubelet.env.v1beta1.j2

@@ -12,7 +12,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 --config={{ kube_config_dir }}/kubelet-config.yaml \
 --kubeconfig={{ kube_config_dir }}/kubelet.conf \
 {# end kubeadm specific settings #}
-{% if container_manager == 'docker' %}
+{% if container_manager == 'docker' and not cri_dockerd_enabled %}
 --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
 {% else %}
 --container-runtime=remote \

roles/kubespray-defaults/defaults/main.yaml

@@ -263,6 +263,9 @@ deploy_container_engine: "{{ inventory_hostname in groups['k8s_cluster'] or etcd
 # Container for runtime
 container_manager: containerd
 
+# Enable CRI Docker interface
+cri_dockerd_enabled: false
+
 # Enable Kata Containers as additional container runtime
 # When enabled, it requires `container_manager` different than Docker
 kata_containers_enabled: false
@@ -288,6 +291,8 @@ cri_socket: >-
   /var/run/crio/crio.sock
   {%- elif container_manager == 'containerd' -%}
   /var/run/containerd/containerd.sock
+  {%- elif cri_dockerd_enabled -%}
+  /var/run/cri-dockerd.sock
   {%- else -%}
   /var/run/dockershim.sock
   {%- endif -%}

roles/reset/tasks/main.yml

@@ -4,7 +4,9 @@
     name: "{{ item }}"
     state: stopped
   with_items:
-    - kubelet
+    - kubelet.service
+    - cri-dockerd.service
+    - cri-dockerd.socket
   failed_when: false
   tags:
     - services
@@ -15,6 +17,8 @@
     state: absent
   with_items:
     - kubelet.service
+    - cri-dockerd.service
+    - cri-dockerd.socket
     - calico-node.service
     - containerd.service.d/http-proxy.conf
     - crio.service.d/http-proxy.conf
@@ -299,6 +303,7 @@
     - /etc/pki/ca-trust/source/anchors/kube-ca.crt
     - /var/log/pods/
     - "{{ bin_dir }}/kubelet"
+    - "{{ bin_dir }}/cri-dockerd"
     - "{{ bin_dir }}/etcd-scripts"
     - "{{ bin_dir }}/etcd"
     - "{{ bin_dir }}/etcd-events"

tests/files/packet_almalinux8-docker.yml

@@ -8,3 +8,4 @@ vm_memory: 3072Mi
 container_manager: docker
 etcd_deployment_type: docker
 resolvconf_mode: docker_dns
+cri_dockerd_enabled: true

tests/files/packet_ubuntu20-aio-docker.yml

@@ -15,3 +15,4 @@ enable_nodelocaldns: False
 container_manager: docker
 etcd_deployment_type: docker
 resolvconf_mode: docker_dns
+cri_dockerd_enabled: true