containerd support (#4664)

* Add limited containerd support

Containerd support for Ubuntu + Calico

* Added CRI-O support for ubuntu

* containerd support.

* Reset  containerd support.

* fix lint.

* implemented feedback

* Change task name cri xx instead of cri-o in reset task and timeout condition.

* set crictl to fixed version

* Use docker-ce's container.io package for containerd.

* Add check containerd is installable or not.

* Avoid stop docker when use containerd and optimize retry for reset.

* Add config.toml.

* Fixed containerd for kubelet.env.

* Merge PR #4629

* Remove unused ubuntu variable for containerd

* Polish code for containerd and cri-o

* Refactoring cri socket configuration.

* Configurable conmon.

* Remove unused crictl/runc download

* Now crictl and runc is downloaded by common crictl.yml.

* fixed yamllint error

* Fixed brokenfiles by conflict.

* Remove commented line in config.toml

* Remove readded v1.12.x version

* Fixed broken set_docker_image_facts

* Fix yamllint errors.

* Remove unused apt source

* Fix crictl could not be installed

* Add containerd config from skolekonov's PR #4601

inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml

@@ -145,7 +145,7 @@ skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipad
 dns_domain: "{{ cluster_name }}"
 
 ## Container runtime
-## docker for docker and crio for cri-o.
+## docker for docker, crio for cri-o and containerd for containerd.
 container_manager: docker
 
 ## Settings for containerized control plane (etcd/kubelet/secrets)

roles/container-engine/containerd/defaults/main.yml

@@ -0,0 +1,12 @@
+---
+kubelet_cgroup_driver: systemd
+
+containerd_config:
+  grpc:
+    max_recv_message_size: 16777216
+    max_send_message_size: 16777216
+  debug:
+    level: ""
+  registries:
+    "docker.io": "https://registry-1.docker.io"
+  max_container_log_line_size: -1

roles/container-engine/containerd/handlers/main.yml

@@ -0,0 +1,24 @@
+---
+- name: restart containerd
+  command: /bin/true
+  notify:
+    - Containerd | reload containerd
+    - Containerd | pause while containerd restarts
+    - Containerd | wait for containerd
+
+- name: Containerd | reload containerd
+  service:
+    name: containerd
+    state: restarted
+
+- name: Containerd | pause while containerd restarts
+  pause:
+    seconds: 5
+    prompt: "Waiting for containerd restart"
+
+- name: Containerd | wait for containerd
+  command: "{{ containerd_bin_dir }}/ctr images ls -q"
+  register: containerd_ready
+  retries: 10
+  delay: 5
+  until: containerd_ready.rc == 0

roles/container-engine/containerd/tasks/crictl.yml

@@ -0,0 +1,26 @@
+---
+- name: crictl | Download crictl
+  include_tasks: "roles/download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.crictl) }}"
+
+- name: Install crictl config
+  template:
+    src: ../templates/crictl.yaml.j2
+    dest: /etc/crictl.yaml
+    owner: bin
+    mode: 0644
+
+- name: Copy crictl binary from download dir
+  synchronize:
+    src: "{{ local_release_dir }}/crictl"
+    dest: "{{ bin_dir }}/crictl"
+    compress: no
+    perms: yes
+    owner: no
+    group: no
+  delegate_to: "{{ inventory_hostname }}"
+
+- name: Install crictl completion
+  shell: /usr/local/bin/crictl completion >/etc/bash_completion.d/crictl
+  ignore_errors: True

roles/container-engine/containerd/tasks/main.yml

@@ -0,0 +1,50 @@
+---
+- name: Fail containerd setup if distribution is not supported
+  fail:
+    msg: "{{ ansible_distribution }} is not supported by containerd."
+  when:
+    - not ansible_distribution in ["CentOS","RedHat", "Ubuntu", "Debian"]
+
+- name: Install Docker
+  include_role:
+    name: container-engine/docker
+
+- name: Install config.toml
+  template:
+    src: config.toml.j2
+    dest: /etc/containerd/config.toml
+    owner: bin
+    mode: 0644
+
+- name: Stop and disabled Docker
+  systemd:
+    name: docker
+    state: stopped
+    enabled: no
+
+- name: Restart containerd
+  systemd:
+    name: containerd
+    state: restarted
+
+- name: Install crictl config
+  template:
+    src: crictl.yaml.j2
+    dest: /etc/crictl.yaml
+    owner: bin
+    mode: 0644
+
+- name: Install crictl completion
+  shell: /usr/local/bin/crictl completion >/etc/bash_completion.d/crictl
+  ignore_errors: True
+  when: ansible_distribution in ["CentOS","RedHat", "Ubuntu", "Debian"]
+
+- name: Enable containerd
+  systemd:
+    name: containerd.service
+    state: started
+    enabled: yes
+    daemon-reload: yes
+
+- name: flush handlers so we can wait for containerd to come up
+  meta: flush_handlers

roles/container-engine/containerd/templates/config.toml.j2

@@ -0,0 +1,40 @@
+# Kubernetes doesn't use containerd restart manager.
+disabled_plugins = ["restart"]
+
+[debug]
+  level = "{{ containerd_config.debug.level | default("") }}"
+
+{% if 'grpc' in containerd_config %}
+[grpc]
+{% for param, value in containerd_config.grpc.items() %}
+  {{ param }} = {{ value }}
+{% endfor %}
+{% endif %}
+
+[plugins.linux]
+  shim = "/usr/bin/containerd-shim"
+  runtime = "/usr/sbin/runc"
+
+[plugins.cri]
+  stream_server_address = "127.0.0.1"
+  max_container_log_line_size = {{ containerd_config.max_container_log_line_size }}
+  sandbox_image = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
+
+[plugins.cri.cni]
+  bin_dir = "/opt/cni/bin"
+  conf_dir = "/etc/cni/net.d"
+  conf_template = ""
+
+[plugins.cri.containerd.untrusted_workload_runtime]
+  runtime_type = ""
+  runtime_engine = ""
+  runtime_root = ""
+
+{% if 'registries' in containerd_config %}
+[plugins.cri.registry]
+[plugins.cri.registry.mirrors]
+{% for registry, addr in containerd_config.registries.items() %}
+[plugins.cri.registry.mirrors."{{ registry }}"]
+  endpoint = ["{{ addr }}"]
+{% endfor %}
+{% endif %}

roles/container-engine/containerd/templates/crictl.yaml.j2

@@ -0,0 +1,4 @@
+runtime-endpoint: unix://{{ cri_socket }}
+image-endpoint: unix://{{ cri_socket }}
+timeout: 30
+debug: false

roles/container-engine/cri-o/tasks/main.yaml

@@ -24,6 +24,12 @@
     gpgcheck: no
   when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic
 
+- name: Add CRI-O PPA
+  apt_repository:
+    repo: ppa:projectatomic/ppa
+    state: present
+  when: ansible_distribution in ["Ubuntu"]
+
 - name: Make sure needed folders exist in the system
   with_items:
     - /etc/crio

roles/container-engine/cri-o/templates/crio.conf.j2

@@ -64,7 +64,7 @@ file_locking = true
 # This is a mandatory setting as this runtime will be the default one
 # and will also be used for untrusted container workloads if
 # runtime_untrusted_workload is not set.
-{% if ansible_os_family == "ClearLinux" or ansible_os_family == "RedHat" %}
+{% if ansible_os_family == "ClearLinux" or ansible_os_family == "RedHat" or ansible_distribution == "Ubuntu" %}
 runtime = "/usr/bin/runc"
 {% else %}
 runtime = "/usr/sbin/runc"
@@ -96,7 +96,7 @@ default_workload_trust = "trusted"
 no_pivot = false
 
 # conmon is the path to conmon binary, used for managing the runtime.
-conmon = "/usr/libexec/crio/conmon"
+conmon = "{{ crio_conmon }}"
 
 # conmon_env is the environment variable list for conmon process,
 # used for passing necessary environment variable to conmon or runtime.

roles/container-engine/cri-o/vars/clearlinux.yml

@@ -3,3 +3,4 @@ crio_packages:
   - containers-basic
 
 crio_service: crio
+crio_conmon: /usr/libexec/crio/conmon

roles/container-engine/cri-o/vars/fedora.yml

@@ -4,3 +4,4 @@ crio_packages:
   - cri-tools
 
 crio_service: cri-o
+crio_conmon: /usr/libexec/crio/conmon

roles/container-engine/cri-o/vars/redhat.yml

@@ -5,3 +5,4 @@ crio_packages:
   - oci-systemd-hook
 
 crio_service: crio
+crio_conmon: /usr/libexec/crio/conmon

roles/container-engine/cri-o/vars/ubuntu.yml

@@ -0,0 +1,6 @@
+---
+crio_packages:
+  - "cri-o-{{ kube_version | regex_replace('^v(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)$', '\\g<major>.\\g<minor>') }}"
+
+crio_service: crio
+crio_conmon: /usr/lib/crio/bin/conmon

roles/container-engine/meta/main.yml

@@ -7,9 +7,23 @@ dependencies:
       - container-engine
       - crio
 
+  - role: container-engine/containerd
+    when:
+      - container_manager == 'containerd'
+    tags:
+      - container-engine
+      - containerd
+
   - role: container-engine/docker
     when:
       - container_manager == 'docker'
     tags:
       - container-engine
       - docker
+
+  - role: container-engine/containerd
+    when:
+      - container_manager == 'containerd'
+    tags:
+      - container-engine
+      - containerd

roles/download/defaults/main.yml

@@ -74,7 +74,9 @@ pod_infra_version: 3.1
 contiv_version: 1.2.1
 cilium_version: "v1.3.0"
 kube_router_version: "v0.2.5"
-multus_version: "v3.2.1"
+multus_version: "v3.1.autoconf"
+
+crictl_version: "v1.14.0"
 
 # Download URLs
 kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm"
@@ -82,6 +84,18 @@ hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/relea
 etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
 cni_download_url: "https://github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
 calicoctl_download_url: "https://github.com/projectcalico/calicoctl/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+
+crictl_checksums:
+  arm:
+    v1.14.0: 9910cecfd6558239ba015323066c7233d8371af359b9ddd0b2a35d5223bcf945
+    v1.13.0: 2e478ebed85f9d70d49fd8f1d1089c8fba6e37d3461aeef91813f1ab0f0df586
+  arm64:
+    v1.14.0: f76b3d00a272c8d210e9a45f77d07d3770bee310d99c4fd9a72d6f55278882e5
+    v1.13.0: 68949c0cb5a37e7604c145d189cf1e109c08c93d9c710ba663db026b9c6f2746
+  amd64:
+    v1.14.0: 483c90a9fe679590df4332ba807991c49232e8cd326c307c575ecef7fe22327b
+    v1.13.0: 9bdbea7a2b382494aff2ff014da328a042c5aba9096a7772e57fdf487e5a1d51
 
 # Checksums
 hyperkube_checksums:
@@ -164,6 +178,9 @@ kubeadm_checksums:
     v1.13.2: 7cb0ce57c1e6e2d85e05de3780a2f35a191fe93f89cfc5816b424efcf39834b9
     v1.13.1: 438173bfa0b7014ecae994c5b9e1f27e1328ab971a3fdb06a393a8095a176ba0
     v1.13.0: f5366206416dc4cfc840a7add2289957b56ccc479cc1b74f7397a4df995d6b06
+crictl_binary_checksums:
+  amd64:
+    v1.14.0: 483c90a9fe679590df4332ba807991c49232e8cd326c307c575ecef7fe22327b
 
 etcd_binary_checksums:
   # Etcd does not have arm32 builds at the moment, having some dummy value is
@@ -194,6 +211,7 @@ cni_binary_checksum: "{{ cni_binary_checksums[image_arch] }}"
 hyperkube_binary_checksum: "{{ hyperkube_checksums[image_arch][kube_version] }}"
 kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}"
 calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}"
+crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}"
 
 # Containers
 # In some cases, we need a way to set --registry-mirror or --insecure-registry for docker,
@@ -304,6 +322,9 @@ addon_resizer_image_tag: "{{ addon_resizer_version }}"
 dashboard_image_repo: "gcr.io/google_containers/kubernetes-dashboard-{{ image_arch }}"
 dashboard_image_tag: "v1.10.1"
 
+image_pull_command: "{{ docker_bin_dir }}/docker pull"
+image_info_command: "{{ docker_bin_dir }}/docker images -q | xargs {{ docker_bin_dir }}/docker inspect -f \"{{ '{{' }} if .RepoTags {{ '}}' }}{{ '{{' }} (index .RepoTags 0) {{ '}}' }}{{ '{{' }} end {{ '}}' }}{{ '{{' }} if .RepoDigests {{ '}}' }},{{ '{{' }} (index .RepoDigests 0) {{ '}}' }}{{ '{{' }} end {{ '}}' }}\" | tr '\n' ','"
+
 downloads:
   netcheck_server:
     enabled: "{{ deploy_netchecker }}"
@@ -378,6 +399,19 @@ downloads:
     groups:
       - k8s-cluster
 
+  crictl:
+    file: true
+    enabled: "{{ container_manager in ['crio', 'cri', 'containerd'] }}"
+    version: "{{ crictl_version }}"
+    dest: "{{local_release_dir}}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz"
+    sha256: "{{ crictl_binary_checksum }}"
+    url: "{{ crictl_download_url }}"
+    unarchive: true
+    owner: "root"
+    mode: "0755"
+    groups:
+      - k8s-cluster
+
   cilium:
     enabled: "{{ kube_network_plugin == 'cilium' }}"
     container: true

roles/download/tasks/download_container.yml

@@ -1,129 +1,137 @@
 ---
+- name: container_download | Make download decision if pull is required by tag or sha256
+  include_tasks: set_docker_image_facts.yml
+  when:
+    - download.enabled
+    - download.container
+  tags:
+    - facts
+
 - block:
-  - name: download_container | Set a few facts
+    - name: download_container | Set a few facts
-    import_tasks: set_container_facts.yml
+      import_tasks: set_container_facts.yml
-    run_once: "{{ download_run_once }}"
+      run_once: "{{ download_run_once }}"
-    tags:
+      tags:
-    - facts
+        - facts
 
-  - name: download_container | Determine if image is in cache
+    - name: download_container | Determine if image is in cache
-    stat:
+      stat:
-      path: "{{ image_path_cached }}"
+        path: "{{ image_path_cached }}"
-    delegate_to: localhost
+      delegate_to: localhost
-    delegate_facts: no
+      delegate_facts: no
-    register: cache_image
+      register: cache_image
-    changed_when: false
+      changed_when: false
-    become: false
+      become: false
-    when:
+      when:
-    - download_force_cache
+        - download_force_cache
 
-  - name: download_container | Set fact indicating if image is in cache
+    - name: download_container | Set fact indicating if image is in cache
-    set_fact:
+      set_fact:
-      image_is_cached: "{{ cache_image.stat.exists | default(false) }}"
+        image_is_cached: "{{ cache_image.stat.exists | default(false) }}"
-    tags:
+      tags:
-    - facts
+        - facts
-    when:
+      when:
-    - download_force_cache
+        - download_force_cache
 
-  - name: download_container | Upload image to node if it is cached
+    - name: download_container | Upload image to node if it is cached
-    synchronize:
+      synchronize:
-      src: "{{ image_path_cached }}"
+        src: "{{ image_path_cached }}"
-      dest: "{{ image_path_final }}"
+        dest: "{{ image_path_final }}"
-      use_ssh_args: "{{ has_bastion | default(false) }}"
+        use_ssh_args: "{{ has_bastion | default(false) }}"
-      mode: push
+        mode: push
-    delegate_facts: no
+      delegate_facts: no
-    register: upload_image
+      register: upload_image
-    failed_when: not upload_image
+      failed_when: not upload_image
-    run_once: "{{ download_run_once }}"
+      run_once: "{{ download_run_once }}"
-    until: upload_image is succeeded
+      until: upload_image is succeeded
-    retries: 4
+      retries: 4
-    delay: "{{ retry_stagger | random + 3 }}"
+      delay: "{{ retry_stagger | random + 3 }}"
-    when:
+      when:
-    - download_force_cache
+        - download_force_cache
-    - image_is_cached
+        - image_is_cached
-    - not download_localhost
+        - not download_localhost
-    - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
+        - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
 
-  - name: download_container | Load image into docker
+    - name: download_container | Load image into docker
-    shell: "{{ docker_bin_dir }}/docker load < {{ image_path_cached if download_localhost else image_path_final }}"
+      shell: "{{ docker_bin_dir }}/docker load < {{ image_path_cached if download_localhost else image_path_final }}"
-    delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
+      delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
-    run_once: "{{ download_run_once }}"
+      run_once: "{{ download_run_once }}"
-    register: container_load_status
+      register: container_load_status
-    failed_when: container_load_status | failed
+      failed_when: container_load_status | failed
-    become: "{{ user_can_become_root | default(false) or not (download_run_once and download_localhost) }}"
+      become: "{{ user_can_become_root | default(false) or not (download_run_once and download_localhost) }}"
-    when:
+      when:
-    - download_force_cache
+        - download_force_cache
-    - image_is_cached
+        - image_is_cached
-    - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
+        - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
 
-  - name: download_container | Prepare container download
+    - name: download_container | Prepare container download
-    import_tasks: check_pull_required.yml
+      import_tasks: check_pull_required.yml
-    run_once: "{{ download_run_once }}"
+      run_once: "{{ download_run_once }}"
-    when:
+      when:
-    - not download_always_pull
+        - not download_always_pull
 
-  - debug:
+    - debug:
-      msg: "XXX Pull required is: {{ pull_required }}"
+        msg: "XXX Pull required is: {{ pull_required }}"
 
-  # NOTE: Pre-loading docker images will not prevent 'docker pull' from re-downloading the layers in that image
+    # NOTE: Pre-loading docker images will not prevent 'docker pull' from re-downloading the layers in that image
-  # if a pull is forced. This is a known issue with docker. See https://github.com/moby/moby/issues/23684
+    # if a pull is forced. This is a known issue with docker. See https://github.com/moby/moby/issues/23684
-  - name: download_container | Download image if required
+    - name: download_container | Download image if required
-    command: "{{ docker_bin_dir }}/docker pull {{ image_reponame }}"
+      command: "{{ image_pull_command }} {{ image_reponame }}"
-    delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
+      delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
-    delegate_facts: yes
+      delegate_facts: yes
-    run_once: "{{ download_run_once }}"
+      run_once: "{{ download_run_once }}"
-    register: pull_task_result
+      register: pull_task_result
-    until: pull_task_result is succeeded
+      until: pull_task_result is succeeded
-    delay: "{{ retry_stagger | random + 3 }}"
+      delay: "{{ retry_stagger | random + 3 }}"
-    retries: 4
+      retries: 4
-    become: "{{ user_can_become_root | default(false) or not download_localhost }}"
+      become: "{{ user_can_become_root | default(false) or not download_localhost }}"
-    when:
+      when:
-    - pull_required | default(download_always_pull)
+        - pull_required | default(download_always_pull)
 
-  # NOTE: image_changed is only valid if a pull is was needed or forced.
+    # NOTE: image_changed is only valid if a pull is was needed or forced.
-  - name: download_container | Check if image changed
+    - name: download_container | Check if image changed
-    set_fact:
+      set_fact:
-      image_changed: "{{ true if pull_task_result.stdout is defined and not 'up to date' in pull_task_result.stdout else false }}"
+        image_changed: "{{ true if pull_task_result.stdout is defined and not 'up to date' in pull_task_result.stdout else false }}"
-    run_once: true
+      run_once: true
-    when:
+      when:
-    - download_force_cache
+        - download_force_cache
-    tags:
+      tags:
-    - facts
+        - facts
 
-  - name: download_container | Save and compress image
+    - name: download_container | Save and compress image
-    shell: "{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_cached if download_localhost else image_path_final }}"
+      shell: "{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_cached if download_localhost else image_path_final }}"
-    delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
+      delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
-    delegate_facts: no
+      delegate_facts: no
-    register: container_save_status
+      register: container_save_status
-    failed_when: container_save_status.stderr
+      failed_when: container_save_status.stderr
-    run_once: true
+      run_once: true
-    become: "{{ user_can_become_root | default(false) or not download_localhost }}"
+      become: "{{ user_can_become_root | default(false) or not download_localhost }}"
-    when:
+      when:
-    - download_force_cache
+        - download_force_cache
-    - not image_is_cached or (image_changed | default(true))
+        - not image_is_cached or (image_changed | default(true))
-    - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
+        - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
 
-  - name: download_container | Copy image to ansible host cache
+    - name: download_container | Copy image to ansible host cache
-    synchronize:
+      synchronize:
-      src: "{{ image_path_final }}"
+        src: "{{ image_path_final }}"
-      dest: "{{ image_path_cached }}"
+        dest: "{{ image_path_cached }}"
-      use_ssh_args: "{{ has_bastion | default(false) }}"
+        use_ssh_args: "{{ has_bastion | default(false) }}"
-      mode: pull
+        mode: pull
-    delegate_facts: no
+      delegate_facts: no
-    run_once: true
+      run_once: true
-    when:
+      when:
-    - download_force_cache
+        - download_force_cache
-    - not download_localhost
+        - not download_localhost
-    - not image_is_cached or (image_changed | default(true))
+        - not image_is_cached or (image_changed | default(true))
-    - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
+        - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
 
-  - name: download_container | Remove container image from cache
+    - name: download_container | Remove container image from cache
-    file:
+      file:
-      state: absent
+        state: absent
-      path: "{{ image_path_final }}"
+        path: "{{ image_path_final }}"
-    when:
+      when:
-    - not download_keep_remote_cache
+        - not download_keep_remote_cache
-    - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
+        - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
 
   tags:
-  - download
+    - download

roles/download/tasks/download_prep.yml

@@ -0,0 +1,42 @@
+---
+# Use the same format for Containerd images as for Docker images
+# ctr doesn't have inspect command
+
+- name: Set image info command for containerd
+  set_fact:
+    image_info_command: "{{ containerd_bin_dir }}/ctr images ls | tail -n +2 | awk -F '[ :]+' '{print $1\":\"$2\",\"$1\":\"$4\"@\"$5}' | tr '\n' ','"
+  when: container_manager == 'containerd'
+
+- name: Register docker images info
+  shell: "{{ image_info_command }}"
+  no_log: true
+  register: docker_images
+  failed_when: false
+  changed_when: false
+  check_mode: no
+  when: download_container
+
+- name: container_download | Create dest directory for saved/loaded container images
+  file:
+    path: "{{ local_release_dir }}/containers"
+    state: directory
+    recurse: yes
+    mode: 0755
+    owner: "{{ ansible_ssh_user|default(ansible_user_id) }}"
+  when: download_container
+
+- name: container_download | create local directory for saved/loaded container images
+  file:
+    path: "{{ local_release_dir }}/containers"
+    state: directory
+    recurse: yes
+  delegate_to: localhost
+  delegate_facts: false
+  become: false
+  run_once: true
+  when:
+    - download_run_once
+    - download_delegate == 'localhost'
+    - download_container
+  tags:
+    - localhost

roles/download/tasks/main.yml

@@ -7,6 +7,25 @@
     - download
     - upload
 
+- name: Use cri-o for cri connection
+  set_fact:
+    cri_socket: /var/run/crio/crio.sock
+  when: container_manager == 'crio'
+
+- name: Use containerd for cri connetion
+  set_fact:
+    cri_socket: /var/run/containerd/containerd.sock
+  when: container_manager == 'containerd'
+
+- name: Use docker for cri connetion
+  set_fact:
+    cri_socket: /var/run/dockershim.sock
+  when: container_manager == 'docker'
+
+- include_tasks: ../../container-engine/containerd/tasks/crictl.yml
+  when:
+    - container_manager in ['containerd', 'crio']
+
 - name: download | Get kubeadm binary and list of required images
   import_tasks: prep_kubeadm_images.yml
   when:

roles/download/tasks/set_docker_image_facts.yml

@@ -0,0 +1,54 @@
+---
+- name: Set if containers should be pulled by digest
+  set_fact:
+    pull_by_digest: >-
+      {%- if download.sha256 is defined and download.sha256 -%}true{%- else -%}false{%- endif -%}
+
+- name: Set pull_args
+  set_fact:
+    pull_args: >-
+      {%- if pull_by_digest %}{{ download.repo }}@sha256:{{ download.sha256 }}{%- else -%}{{ download.repo }}:{{ download.tag }}{%- endif -%}
+
+- name: Set image pull command for containerd
+  set_fact:
+    image_pull_command: "{{ bin_dir }}/crictl pull"
+  when: container_manager in ['crio' ,'containerd']
+
+- name: Register docker images info
+  shell: "{{ image_info_command }}"
+  no_log: true
+  register: docker_images
+  failed_when: false
+  changed_when: false
+  check_mode: no
+  when:
+    - not download_always_pull
+    - group_names | intersect(download.groups) | length
+
+- name: Set if pull is required per container
+  set_fact:
+    pull_required: >-
+      {%- if pull_args in docker_images.stdout.split(',') %}false{%- else -%}true{%- endif -%}
+  when:
+    - not download_always_pull
+    - group_names | intersect(download.groups) | length
+
+- name: Does any host require container pull?
+  vars:
+    hosts_pull_required: "{{ hostvars.values() | map(attribute='pull_required') | select('defined') | list }}"
+  set_fact:
+    any_pull_required: "{{ True in hosts_pull_required }}"
+  run_once: true
+  changed_when: false
+  when: not download_always_pull
+
+- name: Check the local digest sha256 corresponds to the given image tag
+  assert:
+    that: "{{ download.repo }}:{{ download.tag }} in docker_images.stdout.split(',')"
+  when:
+    - group_names | intersect(download.groups) | length
+    - not download_always_pull
+    - not pull_required
+    - pull_by_digest
+  tags:
+    - asserts

roles/download/templates/kubeadm-images.yaml.j2

@@ -6,11 +6,7 @@ apiVersion: kubeadm.k8s.io/v1beta1
 {% endif %}
 kind: InitConfiguration
 nodeRegistration:
-{% if container_manager == 'crio' %}
+  criSocket: {{ cri_socket }}
-  criSocket: /var/run/crio/crio.sock
-{% else %}
-  criSocket: /var/run/dockershim.sock
-{% endif %}
 ---
 {% endif %}
 {% if kube_version is version('v1.11.0', '<') %}
@@ -37,9 +33,5 @@ etcd:
 {% endfor %}
 {% if kube_version is version('v1.12.0', '<') %}
 nodeRegistration:
-{% if container_manager == 'crio' %}
+  criSocket: {{ cri_socket }}
-  criSocket: /var/run/crio/crio.sock
-{% else %}
-  criSocket: /var/run/dockershim.sock
-{% endif %}
 {% endif %}

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2

@@ -16,8 +16,4 @@ discoveryTokenAPIServers:
 discoveryTokenUnsafeSkipCAVerification: true
 nodeRegistration:
   name: {{ kube_override_hostname }}
-{% if container_manager == 'crio' %}
+  criSocket: {{ cri_socket }}
-  criSocket: /var/run/crio/crio.sock
-{% else %}
-  criSocket: /var/run/dockershim.sock
-{% endif %}

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2

@@ -16,8 +16,4 @@ discoveryTokenAPIServers:
 discoveryTokenUnsafeSkipCAVerification: true
 nodeRegistration:
   name: {{ kube_override_hostname }}
-{% if container_manager == 'crio' %}
+  criSocket: {{ cri_socket }}
-  criSocket: /var/run/crio/crio.sock
-{% else %}
-  criSocket: /var/run/dockershim.sock
-{% endif %}

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2

@@ -16,8 +16,4 @@ discovery:
 caCertPath: {{ kube_cert_dir }}/ca.crt
 nodeRegistration:
   name: {{ kube_override_hostname }}
-{% if container_manager == 'crio' %}
+  criSocket: {{ cri_socket }}
-  criSocket: /var/run/crio/crio.sock
-{% else %}
-  criSocket: /var/run/dockershim.sock
-{% endif %}

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -228,11 +228,7 @@ nodeRegistration:
 {% else %}
   taints: {}
 {% endif %}
-{% if container_manager == 'crio' %}
+  criSocket: {{ cri_socket }}
-  criSocket: /var/run/crio/crio.sock
-{% else %}
-  criSocket: /var/run/dockershim.sock
-{% endif %}
 {% if dynamic_kubelet_configuration %}
 featureGates:
   DynamicKubeletConfig: true

roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2

@@ -14,11 +14,7 @@ nodeRegistration:
 {% else %}
   taints: {}
 {% endif %}
-{% if container_manager == 'crio' %}
+  criSocket: {{ cri_socket }}
-  criSocket: /var/run/crio/crio.sock
-{% else %}
-  criSocket: /var/run/dockershim.sock
-{% endif %}
 ---
 apiVersion: kubeadm.k8s.io/v1alpha3
 kind: ClusterConfiguration

roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2

@@ -14,11 +14,7 @@ nodeRegistration:
 {% else %}
   taints: []
 {% endif %}
-{% if container_manager == 'crio' %}
+  criSocket: {{ cri_socket }}
-  criSocket: /var/run/crio/crio.sock
-{% else %}
-  criSocket: /var/run/dockershim.sock
-{% endif %}
 ---
 apiVersion: kubeadm.k8s.io/v1beta1
 kind: ClusterConfiguration

roles/kubernetes/master/templates/kubeadm-controlplane.v1beta1.yaml.j2

@@ -17,8 +17,4 @@ controlPlane:
     bindPort: {{ kube_apiserver_port }}
 nodeRegistration:
   name: {{ kube_override_hostname|default(inventory_hostname) }}
-{% if container_manager == 'crio' %}
+  criSocket: {{ cri_socket }}
-  criSocket: /var/run/crio/crio.sock
-{% else %}
-  criSocket: /var/run/dockershim.sock
-{% endif %}

roles/kubernetes/node/tasks/facts.yml

@@ -3,12 +3,23 @@
   shell: "docker info | grep 'Cgroup Driver' | awk -F': ' '{ print $2; }'"
   register: docker_cgroup_driver_result
   changed_when: false
+  when: container_manager in ['crio', 'docker', 'rkt']
 
-- name: set facts
+- name: set standalone_kubelet fact
   set_fact:
     standalone_kubelet: >-
       {%- if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] -%}true{%- else -%}false{%- endif -%}
+
+- name: set kubelet_cgroup_driver_detected fact for containerd
+  set_fact:
+    kubelet_cgroup_driver_detected: >-
+      {%- if containerd_use_systemd_cgroup -%}systemd{%- else -%}cgroupfs{%- endif -%}
+  when: container_manager == 'containerd'
+
+- name: set kubelet_cgroup_driver_detected fact for other engines
+  set_fact:
     kubelet_cgroup_driver_detected: "{{ docker_cgroup_driver_result.stdout }}"
+  when: container_manager in ['crio', 'docker', 'rkt']
 
 - name: os specific vars
   include_vars: "{{ item }}"

roles/kubernetes/node/tasks/pre_upgrade.yml

@@ -1,12 +1,22 @@
 ---
 - name: "Pre-upgrade | check if kubelet container exists"
-  shell: docker ps -af name=kubelet | grep kubelet
+  shell: >-
+    {% if container_manager in ['crio', 'docker', 'rkt'] %}
+    docker ps -af name=kubelet | grep kubelet
+    {% elif container_manager == 'containerd' %}
+    crictl ps --all --name kubelet | grep kubelet
+    {% endif %}
   failed_when: false
   changed_when: false
   register: kubelet_container_check
 
 - name: "Pre-upgrade | copy /var/lib/cni from kubelet"
-  command: docker cp kubelet:/var/lib/cni /var/lib/cni
+  command: >-
+    {% if container_manager in ['crio', 'docker', 'rkt'] %}
+    docker cp kubelet:/var/lib/cni /var/lib/cni
+    {% elif container_manager == 'containerd' %}
+    ctr run --rm --mount type=bind,src=/var/lib/cni,dst=/cnilibdir,options=rbind:rw kubelet kubelet-tmp sh -c 'cp /var/lib/cni/* /cnilibdir/'
+    {% endif %}
   args:
     creates: "/var/lib/cni"
   failed_when: false
@@ -19,7 +29,12 @@
   when: kubelet_container_check.rc == 0
 
 - name: "Pre-upgrade | ensure kubelet container is removed if using host deployment"
-  command: docker rm -fv kubelet
+  shell: >-
+    {% if container_manager in ['crio', 'docker', 'rkt'] %}
+    docker rm -fv kubelet
+    {% elif container_manager == 'containerd' %}
+    crictl stop kubelet && crictl rm kubelet
+    {% endif %}
   failed_when: false
   changed_when: false
   register: remove_kubelet_container

roles/kubernetes/node/templates/kubelet.env.j2

@@ -43,9 +43,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %}
 --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
 {% endif %}
-{% if container_manager == 'crio' %}
+{% if container_manager != 'docker' %}
 --container-runtime=remote \
---container-runtime-endpoint=/var/run/crio/crio.sock \
+--container-runtime-endpoint={{ cri_socket }} \
 {% endif %}
 --anonymous-auth=false \
 --read-only-port={{ kube_read_only_port }} \

roles/kubernetes/node/templates/kubelet.env.v1beta1.j2

@@ -22,9 +22,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %}
 --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
 {% endif %}
-{% if container_manager == 'crio' %}
+{% if container_manager != 'docker' %}
 --container-runtime=remote \
---container-runtime-endpoint=/var/run/crio/crio.sock \
+--container-runtime-endpoint={{ cri_socket }} \
 {% endif %}
 {% if kube_version is version('v1.8', '<') %}
 --experimental-fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \

roles/kubespray-defaults/defaults/main.yaml

@@ -46,6 +46,7 @@ haproxy_config_dir: "/etc/haproxy"
 # Directory where the binaries will be installed
 bin_dir: /usr/local/bin
 docker_bin_dir: /usr/bin
+containerd_bin_dir: /usr/bin
 etcd_data_dir: /var/lib/etcd
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
@@ -258,6 +259,9 @@ docker_options: >-
 # Experimental kubeadm etcd deployment mode. Available only for new deployment
 etcd_kubeadm_enabled: false
 
+# Containerd options
+containerd_use_systemd_cgroup: false
+
 # Settings for containerized control plane (etcd/kubelet/secrets)
 # deployment type for legacy etcd mode
 etcd_deployment_type: docker

roles/network_plugin/calico/rr/tasks/main.yml

@@ -42,9 +42,19 @@
 
 - name: Calico-rr | Write calico-rr systemd init file
   template:
-    src: calico-rr.service.j2
+    src: calico-rr-docker.service.j2
     dest: /etc/systemd/system/calico-rr.service
   notify: restart calico-rr
+  when:
+    - container_manager in ['crio', 'docker', 'rkt']
+
+- name: Calico-rr | Write calico-rr systemd init file
+  template:
+    src: calico-rr-containerd.service.j2
+    dest: /etc/systemd/system/calico-rr.service
+  notify: restart calico-rr
+  when:
+    - container_manager == 'containerd'
 
 - name: Calico-rr | Configure route reflector
   command: |-

roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2

@@ -0,0 +1,27 @@
+[Unit]
+Description=calico-rr
+After=containerd.service
+Requires=containerd.service
+
+[Service]
+EnvironmentFile=/etc/calico/calico-rr.env
+ExecStartPre=-{{ containerd_bin_dir }}/ctr t delete -f calico-rr
+ExecStart={{ containerd_bin_dir }}/ctr run --net-host --privileged \
+ --env IP=${IP} \
+ --env IP6=${IP6} \
+ --env ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \
+ --env ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \
+ --env ETCD_CERT_FILE=${ETCD_CERT_FILE} \
+ --env ETCD_KEY_FILE=${ETCD_KEY_FILE} \
+ --mount type=bind,src=/var/log/calico-rr,dst=/var/log/calico,options=rbind:rw \
+ --mount type=bind,src={{ calico_cert_dir }},dst={{ calico_cert_dir }},options=rbind:ro \
+ {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }} \
+ calico-rr
+
+Restart=always
+RestartSec=10s
+
+ExecStop=-{{ containerd_bin_dir }}/ctr c rm calico-rr
+
+[Install]
+WantedBy=multi-user.target

roles/network_plugin/cilium/templates/cilium-ds.yml.j2

@@ -151,14 +151,14 @@ spec:
               mountPath: /host/opt/cni/bin
             - name: etc-cni-netd
               mountPath: /host/etc/cni/net.d
-{% if container_manager == 'crio' %}
+{% if container_manager == 'docker' %}
-            - name: crio-socket
-              mountPath: /var/run/crio.sock
-              readOnly: true
-{% else %}
             - name: docker-socket
               mountPath: /var/run/docker.sock
               readOnly: true
+{% else %}
+            - name: "{{ container_manager }}-socket"
+              mountPath: {{ cri_socket }}
+              readOnly: true
 {% endif %}
             - name: etcd-config-path
               mountPath: /var/lib/etcd-config
@@ -182,16 +182,16 @@ spec:
         - name: bpf-maps
           hostPath:
             path: /sys/fs/bpf
-{% if container_manager == 'crio' %}
+{% if container_manager == 'docker' %}
-        # To read crio events from the node
-        - name: crio-socket
-          hostPath:
-            path: /var/run/crio/crio.sock
-{% else %}
         # To read docker events from the node
         - name: docker-socket
           hostPath:
             path: /var/run/docker.sock
+{% else %}
+        # To read crio events from the node
+        - name: {{ container_manager }}-socket
+          hostPath:
+            path: {{ cri_socket }}
 {% endif %}
         # To install cilium cni plugin in the host
         - name: cni-path

roles/reset/tasks/main.yml

@@ -14,8 +14,6 @@
   with_items:
     - kubelet
     - vault
-    - etcd
-    - etcd-events
   failed_when: false
   tags:
     - services
@@ -26,8 +24,6 @@
     state: absent
   with_items:
     - kubelet
-    - etcd
-    - etcd-events
     - vault
     - calico-node
   register: services_removed
@@ -57,6 +53,7 @@
   retries: 4
   until: remove_all_containers.rc == 0
   delay: 5
+  when: container_manager == "docker"
   tags:
     - docker
 
@@ -64,50 +61,76 @@
   service:
     name: docker
     state: restarted
-  when: docker_dropins_removed.changed
+  when: docker_dropins_removed.changed and container_manager == "docker"
   tags:
     - docker
 
-- name: reset | stop all cri-o containers
+- name: reset | stop all cri containers
-  shell: "crictl ps -aq | xargs -r crictl stop"
+  shell: "crictl ps -aq | xargs -r crictl -t 60s stop"
-  register: remove_all_crio_containers
+  register: remove_all_cri_containers
-  retries: 4
+  retries: 5
-  until: remove_all_crio_containers.rc == 0
+  until: remove_all_cri_containers.rc == 0
   delay: 5
   tags:
     - crio
-  when: container_manager == 'crio'
+    - containerd
+  when: container_manager in ["crio", "containerd"]
 
-
+- name: reset | remove all cri containers
-- name: reset | remove all cri-o containers
+  shell: "crictl ps -aq | xargs -r crictl -t 60s rm"
-  shell: "crictl ps -aq | xargs -r crictl rm"
+  register: remove_all_cri_containers
-  register: remove_all_crio_containers
+  retries: 5
-  retries: 4
+  until: remove_all_cri_containers.rc == 0
-  until: remove_all_crio_containers.rc == 0
   delay: 5
   tags:
     - crio
-  when: container_manager == 'crio' and deploy_container_engine|default(true)
+    - containerd
+  when: container_manager in ["crio", "containerd"] and deploy_container_engine|default(true)
 
-- name: reset | stop all cri-o pods
+- name: reset | stop all cri pods
-  shell: "crictl pods -q | xargs -r crictl stopp"
+  shell: "crictl pods -q | xargs -r crictl -t 60s stopp"
-  register: remove_all_crio_containers
+  register: remove_all_cri_containers
-  retries: 4
+  retries: 5
-  until: remove_all_crio_containers.rc == 0
+  until: remove_all_cri_containers.rc == 0
   delay: 5
   tags:
     - crio
-  when: container_manager == 'crio'
+    - containerd
+  when: container_manager in ["crio", "containerd"]
 
-- name: reset | remove all cri-o pods
+- name: reset | remove all cri pods
-  shell: "crictl pods -q | xargs -r crictl rmp"
+  shell: "crictl pods -q | xargs -r crictl -t 60s rmp"
-  register: remove_all_crio_containers
+  register: remove_all_cri_containers
-  retries: 4
+  retries: 5
-  until: remove_all_crio_containers.rc == 0
+  until: remove_all_cri_containers.rc == 0
   delay: 5
   tags:
     - crio
-  when: container_manager == 'crio'
+    - containerd
+  when: container_manager in ["crio", "containerd"]
+
+- name: reset | stop etcd services
+  service:
+    name: "{{ item }}"
+    state: stopped
+  with_items:
+    - etcd
+    - etcd-events
+  failed_when: false
+  tags:
+    - services
+
+- name: reset | remove etcd services
+  file:
+    path: "/etc/systemd/system/{{ item }}.service"
+    state: absent
+  with_items:
+    - etcd
+    - etcd-events
+  register: services_removed
+  tags:
+    - services
+
 - name: reset | gather mounted kubelet dirs
   shell: mount | grep /var/lib/kubelet/ | awk '{print $3}' | tac
   args: