use cri-o from upstream instead of kubic/OBS (#9374)

* [cri-o] use cri-o from upstream instead of kubic/OBS

* [cri-o] add proper molecule coverage

* [skopeo] download skopeo from upstream build

* [cri-o] clean up legacy deployments

* disable cri-o per-distribution variables

roles/container-engine/cri-o/defaults/main.yml

@@ -1,7 +1,7 @@
 ---
 
 crio_cgroup_manager: "{{ kubelet_cgroup_driver | default('systemd') }}"
-crio_conmon: "/usr/bin/conmon"
+crio_conmon: "{{ bin_dir }}/conmon"
 crio_enable_metrics: false
 crio_log_level: "info"
 crio_metrics_port: "9090"
@@ -37,17 +37,10 @@ crio_stream_port: "10010"
 
 crio_required_version: "{{ kube_version | regex_replace('^v(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)$', '\\g<major>.\\g<minor>') }}"
 
-crio_kubernetes_version_matrix:
-  "1.24": "1.24"
-  "1.23": "1.23"
-  "1.22": "1.22"
-
-crio_version: "{{ crio_kubernetes_version_matrix[crio_required_version] | default('1.24') }}"
-
 # The crio_runtimes variable defines a list of OCI compatible runtimes.
 crio_runtimes:
   - name: runc
-    path: /usr/bin/runc
+    path: "{{ bin_dir }}/runc"
     type: oci
     root: /run/runc
 
@@ -65,7 +58,7 @@ kata_runtimes:
 # crun is a fast and low-memory footprint OCI Container Runtime fully written in C.
 crun_runtime:
   name: crun
-  path: /usr/bin/crun
+  path: "{{ bin_dir }}/crun"
   type: oci
   root: /run/crun
 
@@ -76,20 +69,10 @@ youki_runtime:
   type: oci
   root: /run/youki
 
-# When this is true, CRI-O package repositories are added. Set this to false when using an
+# TODO(cristicalin): remove this after 2.21
-# environment with preconfigured CRI-O package repositories.
-crio_add_repos: true
-
-# Allow crio offline installation
 crio_download_base: "download.opensuse.org/repositories/devel:kubic:libcontainers:stable"
-
-# Allow crio offline installation
 crio_download_crio: "http://{{ crio_download_base }}:/cri-o:/"
 
-# skopeo need for save/load images when download_run_once=true
-skopeo_packages:
-  - "skopeo"
-
 # Configure the cri-o pids limit, increase this for heavily multi-threaded workloads
 # see https://github.com/cri-o/cri-o/issues/1921
 crio_pids_limit: 1024
@@ -102,3 +85,19 @@ crio_subuid_start: 2130706432
 crio_subuid_length: 16777216
 crio_subgid_start: 2130706432
 crio_subgid_length: 16777216
+
+# cri-o binary files
+crio_bin_files:
+  - conmon
+  - crio
+  - crio-status
+  - pinns
+
+# cri-o manual files
+crio_man_files:
+  5:
+    - crio.conf
+    - crio.conf.d
+  8:
+    - crio
+    - crio-status

roles/container-engine/cri-o/meta/main.yml

@@ -1,3 +1,5 @@
 ---
 dependencies:
   - role: container-engine/crictl
+  - role: container-engine/runc
+  - role: container-engine/skopeo

roles/container-engine/cri-o/molecule/default/files/10-mynet.conf

@@ -0,0 +1,17 @@
+{
+  "cniVersion": "0.2.0",
+  "name": "mynet",
+  "type": "bridge",
+  "bridge": "cni0",
+  "isGateway": true,
+  "ipMasq": true,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "172.19.0.0/24",
+    "routes": [
+      {
+        "dst": "0.0.0.0/0"
+      }
+    ]
+  }
+}

roles/container-engine/cri-o/molecule/default/files/container.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "runc1"
+  },
+  "image": {
+    "image": "quay.io/kubespray/hello-world:latest"
+  },
+  "log_path": "runc1.0.log",
+  "linux": {}
+}

roles/container-engine/cri-o/molecule/default/files/sandbox.json

@@ -0,0 +1,10 @@
+{
+  "metadata": {
+    "name": "runc1",
+    "namespace": "default",
+    "attempt": 1,
+    "uid": "hdishd83djaidwnduwk28bcsb"
+  },
+  "linux": {},
+  "log_directory": "/tmp"
+}

roles/container-engine/cri-o/molecule/default/molecule.yml

@@ -7,24 +7,38 @@ lint: |
   set -e
   yamllint -c ../../../.yamllint .
 platforms:
-  - name: ubuntu2004
+  - name: ubuntu20
     box: generic/ubuntu2004
     cpus: 2
     memory: 1024
     groups:
       - kube_control_plane
+      - kube_node
+      - k8s_cluster
   - name: almalinux8
     box: almalinux/8
     cpus: 2
     memory: 1024
     groups:
       - kube_control_plane
+      - kube_node
+      - k8s_cluster
   - name: fedora
-    box: fedora/35-cloud-base
+    box: fedora/36-cloud-base
     cpus: 2
     memory: 1024
     groups:
       - kube_control_plane
+      - kube_node
+      - k8s_cluster
+  - name: debian10
+    box: generic/debian10
+    cpus: 2
+    memory: 1024
+    groups:
+      - kube_control_plane
+      - kube_node
+      - k8s_cluster
 provisioner:
   name: ansible
   env:

roles/container-engine/cri-o/molecule/default/prepare.yml

@@ -2,5 +2,51 @@
 - name: Prepare
   hosts: all
   gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
   roles:
+    - role: kubespray-defaults
     - role: bootstrap-os
+    - role: kubernetes/preinstall
+    - role: adduser
+      user: "{{ addusers.kube }}"
+  tasks:
+    - include_tasks: "../../../../download/tasks/download_file.yml"
+      vars:
+        download: "{{ download_defaults | combine(downloads.cni) }}"
+
+- name: Prepare CNI
+  hosts: all
+  gather_facts: False
+  become: true
+  vars:
+    ignore_assert_errors: true
+    kube_network_plugin: cni
+  roles:
+    - role: kubespray-defaults
+    - role: network_plugin/cni
+  tasks:
+    - name: Copy test container files
+      copy:
+        src: "{{ item }}"
+        dest: "/tmp/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - container.json
+        - sandbox.json
+    - name: Create /etc/cni/net.d directory
+      file:
+        path: /etc/cni/net.d
+        state: directory
+        owner: "{{ kube_owner }}"
+        mode: 0755
+    - name: Setup CNI
+      copy:
+        src: "{{ item }}"
+        dest: "/etc/cni/net.d/{{ item }}"
+        owner: root
+        mode: 0644
+      with_items:
+        - 10-mynet.conf

roles/container-engine/cri-o/molecule/default/tests/test_default.py

@@ -19,3 +19,17 @@ def test_run(host):
         cmd = host.command(crictl + " --runtime-endpoint " + path + " version")
     assert cmd.rc == 0
     assert "RuntimeName:  cri-o" in cmd.stdout
+
+def test_run_pod(host):
+    runtime = "runc"
+
+    run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime)
+    with host.sudo():
+        cmd = host.command(run_command)
+    assert cmd.rc == 0
+
+    with host.sudo():
+      log_f = host.file("/tmp/runc1.0.log")
+
+      assert log_f.exists
+      assert b"Hello from Docker" in log_f.content

roles/container-engine/cri-o/tasks/cleanup.yaml

@@ -0,0 +1,119 @@
+---
+# TODO(cristicalin): drop this file after 2.21
+- name: CRI-O kubic repo name for debian os family
+  set_fact:
+    crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x','')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}"
+  when: ansible_os_family == "Debian"
+
+- name: Remove legacy CRI-O kubic apt repo key
+  apt_key:
+    url: "https://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/Release.key"
+    state: absent
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Remove legacy CRI-O kubic apt repo
+  apt_repository:
+    repo: "deb http://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: absent
+    filename: devel-kubic-libcontainers-stable
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Remove legacy CRI-O kubic cri-o apt repo
+  apt_repository:
+    repo: "deb {{ crio_download_crio }}{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /"
+    state: absent
+    filename: devel-kubic-libcontainers-stable-cri-o
+  when: crio_kubic_debian_repo_name is defined
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    description: Stable Releases of Upstream github.com/containers packages (CentOS_$releasever)
+    baseurl: http://{{ crio_download_base }}/CentOS_{{ ansible_distribution_major_version }}/
+    state: absent
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution not in ["Amazon", "Fedora"]
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    description: "CRI-O {{ crio_version }} (CentOS_$releasever)"
+    baseurl: "{{ crio_download_crio }}{{ crio_version }}/CentOS_{{ ansible_distribution_major_version }}/"
+    state: absent
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution not in ["Amazon", "Fedora"]
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    description: Stable Releases of Upstream github.com/containers packages
+    baseurl: http://{{ crio_download_base }}/Fedora_{{ ansible_distribution_major_version }}/
+    state: absent
+  when:
+    - ansible_distribution in ["Fedora"]
+    - not is_ostree
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    description: "CRI-O {{ crio_version }}"
+    baseurl: "{{ crio_download_crio }}{{ crio_version }}/Fedora_{{ ansible_distribution_major_version }}/"
+    state: absent
+  when:
+    - ansible_distribution in ["Fedora"]
+    - not is_ostree
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: devel_kubic_libcontainers_stable
+    description: Stable Releases of Upstream github.com/containers packages
+    baseurl: http://{{ crio_download_base }}/CentOS_7/
+    state: absent
+  when: ansible_distribution in ["Amazon"]
+
+- name: Remove legacy CRI-O kubic yum repo
+  yum_repository:
+    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
+    description: "CRI-O {{ crio_version }}"
+    baseurl: "{{ crio_download_crio }}{{ crio_version }}/CentOS_7/"
+    state: absent
+  when: ansible_distribution in ["Amazon"]
+
+- name: Disable modular repos for CRI-O
+  ini_file:
+    path: "/etc/yum.repos.d/{{ item.repo }}.repo"
+    section: "{{ item.section }}"
+    option: enabled
+    value: 0
+    mode: 0644
+  become: true
+  when: is_ostree
+  loop:
+    - repo: "fedora-updates-modular"
+      section: "updates-modular"
+    - repo: "fedora-modular"
+      section: "fedora-modular"
+
+# Disable any older module version if we enabled them before
+- name: Disable CRI-O ex module
+  command: "rpm-ostree ex module disable cri-o:{{ item }}"
+  become: true
+  when:
+    - is_ostree
+    - ostree_version is defined and ostree_version.stdout is version('2021.9', '>=')
+  with_items:
+    - 1.22
+    - 1.23
+    - 1.24
+
+- name: cri-o | remove installed packages
+  package:
+    name: "{{ item }}"
+    state: absent
+  when: not is_ostree
+  with_items:
+    - cri-o
+    - cri-o-runc
+    - oci-systemd-hook

roles/container-engine/cri-o/tasks/crio_repo.yml

@@ -1,179 +0,0 @@
----
-- block:
-    - name: Add Debian Backports apt repo
-      apt_repository:
-        repo: "deb http://deb.debian.org/debian {{ ansible_distribution_release }}-backports main"
-        state: present
-        filename: debian-backports
-
-    - name: Set libseccomp2 pin priority to apt_preferences on Debian buster
-      copy:
-        content: |
-          Package: libseccomp2
-          Pin: release a={{ ansible_distribution_release }}-backports
-          Pin-Priority: 1001
-        dest: "/etc/apt/preferences.d/libseccomp2"
-        owner: "root"
-        mode: 0644
-  when:
-    - ansible_distribution == "Debian"
-    - ansible_distribution_version == "10"
-
-- name: CRI-O kubic repo name for debian os family
-  set_fact:
-    crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x','')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}"
-  when: ansible_os_family == "Debian"
-
-- name: Add CRI-O kubic apt repo key
-  apt_key:
-    url: "https://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/Release.key"
-    state: present
-  when: crio_kubic_debian_repo_name is defined
-  register: apt_key_download
-  until: apt_key_download is succeeded
-  retries: 4
-  delay: "{{ retry_stagger | d(3) }}"
-  environment: "{{ proxy_env }}"
-
-- name: Add CRI-O kubic apt repo
-  apt_repository:
-    repo: "deb http://{{ crio_download_base }}/{{ crio_kubic_debian_repo_name }}/ /"
-    state: present
-    filename: devel-kubic-libcontainers-stable
-  when: crio_kubic_debian_repo_name is defined
-
-- name: Add CRI-O kubic cri-o apt repo
-  apt_repository:
-    repo: "deb {{ crio_download_crio }}{{ crio_version }}/{{ crio_kubic_debian_repo_name }}/ /"
-    state: present
-    filename: devel-kubic-libcontainers-stable-cri-o
-  when: crio_kubic_debian_repo_name is defined
-
-- name: Check that amzn2-extras.repo exists
-  stat:
-    path: /etc/yum.repos.d/amzn2-extras.repo
-  register: amzn2_extras_file_stat
-  when: ansible_distribution in ["Amazon"]
-
-- name: Find docker repo in amzn2-extras.repo file
-  lineinfile:
-    dest: /etc/yum.repos.d/amzn2-extras.repo
-    line: "[amzn2extra-docker]"
-  check_mode: yes
-  register: amzn2_extras_docker_repo
-  when:
-    - ansible_distribution in ["Amazon"]
-    - amzn2_extras_file_stat.stat.exists
-
-- name: Remove docker repository
-  ini_file:
-    dest: /etc/yum.repos.d/amzn2-extras.repo
-    section: amzn2extra-docker
-    option: enabled
-    value: "0"
-    backup: yes
-    mode: 0644
-  when:
-    - ansible_distribution in ["Amazon"]
-    - amzn2_extras_file_stat.stat.exists
-    - not amzn2_extras_docker_repo.changed
-
-- name: Add container-selinux yum repo
-  yum_repository:
-    name: copr:copr.fedorainfracloud.org:lsm5:container-selinux
-    file: _copr_lsm5-container-selinux.repo
-    description: Copr repo for container-selinux owned by lsm5
-    baseurl: https://download.copr.fedorainfracloud.org/results/lsm5/container-selinux/epel-7-$basearch/
-    gpgcheck: yes
-    gpgkey: https://download.copr.fedorainfracloud.org/results/lsm5/container-selinux/pubkey.gpg
-    skip_if_unavailable: yes
-    enabled: yes
-    repo_gpgcheck: no
-  when: ansible_distribution in ["Amazon"]
-
-- name: Add CRI-O kubic yum repo
-  yum_repository:
-    name: devel_kubic_libcontainers_stable
-    description: Stable Releases of Upstream github.com/containers packages (CentOS_$releasever)
-    baseurl: http://{{ crio_download_base }}/CentOS_{{ ansible_distribution_major_version }}/
-    gpgcheck: yes
-    gpgkey: http://{{ crio_download_base }}/CentOS_{{ ansible_distribution_major_version }}/repodata/repomd.xml.key
-    keepcache: "0"
-  when:
-    - ansible_os_family == "RedHat"
-    - ansible_distribution not in ["Amazon", "Fedora"]
-
-- name: Add CRI-O kubic yum repo
-  yum_repository:
-    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
-    description: "CRI-O {{ crio_version }} (CentOS_$releasever)"
-    baseurl: "{{ crio_download_crio }}{{ crio_version }}/CentOS_{{ ansible_distribution_major_version }}/"
-    gpgcheck: yes
-    gpgkey: "{{ crio_download_crio }}{{ crio_version }}/CentOS_{{ ansible_distribution_major_version }}/repodata/repomd.xml.key"
-  when:
-    - ansible_os_family == "RedHat"
-    - ansible_distribution not in ["Amazon", "Fedora"]
-
-- name: Add CRI-O kubic yum repo
-  yum_repository:
-    name: devel_kubic_libcontainers_stable
-    description: Stable Releases of Upstream github.com/containers packages
-    baseurl: http://{{ crio_download_base }}/Fedora_{{ ansible_distribution_major_version }}/
-    gpgcheck: yes
-    gpgkey: http://{{ crio_download_base }}/Fedora_{{ ansible_distribution_major_version }}/repodata/repomd.xml.key
-    keepcache: "0"
-  when:
-    - ansible_distribution in ["Fedora"]
-    - not is_ostree
-
-- name: Add CRI-O kubic yum repo
-  yum_repository:
-    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
-    description: "CRI-O {{ crio_version }}"
-    baseurl: "{{ crio_download_crio }}{{ crio_version }}/Fedora_{{ ansible_distribution_major_version }}/"
-    gpgcheck: yes
-    gpgkey: "{{ crio_download_crio }}{{ crio_version }}/Fedora_{{ ansible_distribution_major_version }}/repodata/repomd.xml.key"
-  when:
-    - ansible_distribution in ["Fedora"]
-    - not is_ostree
-
-- name: Add CRI-O kubic yum repo
-  yum_repository:
-    name: devel_kubic_libcontainers_stable
-    description: Stable Releases of Upstream github.com/containers packages
-    baseurl: http://{{ crio_download_base }}/CentOS_7/
-    gpgcheck: yes
-    gpgkey: http://{{ crio_download_base }}/CentOS_7/repodata/repomd.xml.key
-    keepcache: "0"
-  when: ansible_distribution in ["Amazon"]
-
-- name: Add CRI-O kubic yum repo
-  yum_repository:
-    name: "devel_kubic_libcontainers_stable_cri-o_{{ crio_version }}"
-    description: "CRI-O {{ crio_version }}"
-    baseurl: "{{ crio_download_crio }}{{ crio_version }}/CentOS_7/"
-    gpgcheck: yes
-    gpgkey: "{{ crio_download_crio }}{{ crio_version }}/CentOS_7/repodata/repomd.xml.key"
-  when: ansible_distribution in ["Amazon"]
-
-- name: Enable modular repos for CRI-O
-  ini_file:
-    path: "/etc/yum.repos.d/{{ item.repo }}.repo"
-    section: "{{ item.section }}"
-    option: enabled
-    value: 1
-    mode: 0644
-  become: true
-  when: is_ostree
-  loop:
-    - repo: "fedora-updates-modular"
-      section: "updates-modular"
-    - repo: "fedora-modular"
-      section: "fedora-modular"
-
-- name: Enable CRI-O ex module
-  command: "rpm-ostree ex module enable cri-o:{{ crio_version }}"
-  become: true
-  when:
-    - is_ostree
-    - ostree_version is defined and ostree_version.stdout is version('2021.9', '>=')

roles/container-engine/cri-o/tasks/main.yaml

@@ -1,5 +1,5 @@
 ---
-- name: check if fedora coreos
+- name: cri-o | check if fedora coreos
   stat:
     path: /run/ostree-booted
     get_attributes: no
@@ -7,57 +7,48 @@
     get_mime: no
   register: ostree
 
-- name: set is_ostree
+- name: cri-o | set is_ostree
   set_fact:
     is_ostree: "{{ ostree.stat.exists }}"
 
-- name: get ostree version
+- name: cri-o | get ostree version
   shell: "set -o pipefail && rpm-ostree --version | awk -F\\' '/Version/{print $2}'"
   args:
     executable: /bin/bash
   register: ostree_version
   when: is_ostree
 
-- name: gather os specific variables
+- name: cri-o | Download cri-o
-  include_vars: "{{ item }}"
+  include_tasks: "../../../download/tasks/download_file.yml"
-  with_first_found:
+  vars:
-    - files:
+    download: "{{ download_defaults | combine(downloads.crio) }}"
-        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
-        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml"
-        - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
-        - "{{ ansible_distribution|lower }}.yml"
-        - "{{ ansible_os_family|lower }}-{{ ansible_architecture }}.yml"
-        - "{{ ansible_os_family|lower }}.yml"
-        - defaults.yml
-      paths:
-        - ../vars
-      skip: true
-  tags:
-    - facts
 
-- name: import crio repo
+- name: cri-o | special handling for amazon linux
-  import_tasks: "crio_repo.yml"
+  import_tasks: "setup-amazon.yaml"
-  when: crio_add_repos
+  when: ansible_distribution in ["Amazon"]
 
-- name: Build a list of crio runtimes with Katacontainers runtimes
+- name: cri-o | clean up reglacy repos
+  import_tasks: "cleanup.yaml"
+
+- name: cri-o | build a list of crio runtimes with Katacontainers runtimes
   set_fact:
     crio_runtimes: "{{ crio_runtimes + kata_runtimes  }}"
   when:
     - kata_containers_enabled
 
-- name: Build a list of crio runtimes with crun runtime
+- name: cri-o | build a list of crio runtimes with crun runtime
   set_fact:
     crio_runtimes: "{{ crio_runtimes + [crun_runtime] }}"
   when:
     - crun_enabled
 
-- name: Build a list of crio runtimes with youki runtime
+- name: cri-o | build a list of crio runtimes with youki runtime
   set_fact:
     crio_runtimes: "{{ crio_runtimes + [youki_runtime] }}"
   when:
     - youki_enabled
 
-- name: Make sure needed folders exist in the system
+- name: cri-o | make sure needed folders exist in the system
   with_items:
     - /etc/crio
     - /etc/containers
@@ -67,98 +58,47 @@
     state: directory
     mode: 0755
 
-- name: Install cri-o config
+- name: cri-o | install cri-o config
   template:
     src: crio.conf.j2
     dest: /etc/crio/crio.conf
     mode: 0644
   register: config_install
 
-- name: Install config.json
+- name: cri-o | install config.json
   template:
     src: config.json.j2
     dest: /etc/crio/config.json
     mode: 0644
   register: reg_auth_install
 
-- name: Add skopeo pkg to install
+- name: cri-o | copy binaries
-  set_fact:
+  copy:
-    crio_packages: "{{ crio_packages + skopeo_packages }}"
+    src: "{{ local_release_dir }}/cri-o/bin/{{ item }}"
-  when:
+    dest: "{{ bin_dir }}/{{ item }}"
-    - not skip_downloads|default(false)
+    mode: 0755
-    - download_run_once
+    remote_src: true
-
-- name: Add libseccomp2 package from Debian Backports to install
-  set_fact:
-    crio_packages: "{{ crio_debian_buster_backports_packages + crio_packages }}"
-  when:
-    - ansible_distribution == "Debian"
-    - ansible_distribution_version == "10"
-
-- name: Remove dpkg hold
-  dpkg_selections:
-    name: "{{ item | split ('=') | first }}"
-    selection: install
-  when:
-    - ansible_pkg_mgr == 'apt'
-  changed_when: false
   with_items:
-    - "{{ crio_packages }}"
+    - "{{ crio_bin_files }}"
+  notify: restart crio
 
-- name: Install cri-o packages
+- name: cri-o | copy service file
-  package:
+  copy:
-    name: "{{ item }}"
+    src: "{{ local_release_dir }}/cri-o/contrib/crio.service"
-    state: present
+    dest: /etc/systemd/system/crio.service
-  when: not is_ostree
+    mode: 0755
-  with_items: "{{ crio_packages }}"
+    remote_src: true
-  register: package_install
+  notify: restart crio
-  until: package_install is succeeded
-  retries: 4
-  delay: "{{ retry_stagger | d(3) }}"
 
-# This is required to ensure any apt upgrade will not break kubernetes
+- name: cri-o | copy default policy
-- name: Tell Debian hosts not to change the cri-o version with apt upgrade
+  copy:
-  dpkg_selections:
+    src: "{{ local_release_dir }}/cri-o/contrib/policy.json"
-    name: "{{ item | split ('=') | first }}"
+    dest: /etc/containers/policy.json
-    selection: hold
+    mode: 0755
-  when:
+    remote_src: true
-    - ansible_pkg_mgr == 'apt'
+  notify: restart crio
-  changed_when: false
-  with_items:
-    - "{{ crio_packages }}"
 
-- name: Check if already installed
+- name: cri-o | copy mounts.conf
-  stat:
-    path: "/bin/crio"
-    get_attributes: no
-    get_checksum: no
-    get_mime: no
-  register: need_bootstrap_crio
-  when: is_ostree
-
-- name: Install cri-o packages with ostree
-  command: "rpm-ostree install {{ crio_packages|join(' ') }}"
-  when:
-    - is_ostree
-    - not need_bootstrap_crio.stat.exists
-  become: true
-
-- name: Reboot immediately for updated ostree
-  reboot:
-  become: true
-  when:
-    - is_ostree
-    - not need_bootstrap_crio.stat.exists
-
-- name: Remove example CNI configs
-  file:
-    path: "/etc/cni/net.d/{{ item }}"
-    state: absent
-  loop:
-    - 100-crio-bridge.conf
-    - 200-loopback.conf
-
-- name: Copy mounts.conf
   copy:
     src: mounts.conf
     dest: /etc/containers/mounts.conf
@@ -167,15 +107,28 @@
     - ansible_os_family == 'RedHat'
   notify: restart crio
 
-- name: Create directory for oci hooks
+- name: cri-o | create directory for oci hooks
   file:
     path: /etc/containers/oci/hooks.d
     state: directory
     owner: root
     mode: 0755
 
+- name: cri-o | set overlay driver
+  ini_file:
+    dest: /etc/containers/storage.conf
+    section: storage
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+    mode: 0644
+  with_items:
+    - option: driver
+      value: '"overlay"'
+    - option: graphroot
+      value: '"/var/lib/containers/storage"'
+
 # metacopy=on is available since 4.19 and was backported to RHEL 4.18 kernel
-- name: Set metacopy mount options correctly
+- name: cri-o | set metacopy mount options correctly
   ini_file:
     dest: /etc/containers/storage.conf
     section: storage.options.overlay
@@ -183,14 +136,14 @@
     value: '{{ ''"nodev"'' if ansible_kernel is version_compare(("4.18" if ansible_os_family == "RedHat" else "4.19"), "<") else ''"nodev,metacopy=on"'' }}'
     mode: 0644
 
-- name: Create directory registries configs
+- name: cri-o | create directory registries configs
   file:
     path: /etc/containers/registries.conf.d
     state: directory
     owner: root
     mode: 0755
 
-- name: Write registries configs
+- name: cri-o | write registries configs
   template:
     src: registry.conf.j2
     dest: "/etc/containers/registries.conf.d/10-{{ item.prefix | default(item.location) | regex_replace(':', '_') }}.conf"
@@ -198,14 +151,14 @@
   loop: "{{ crio_registries }}"
   notify: restart crio
 
-- name: Configure unqualified registry settings
+- name: cri-o | configure unqualified registry settings
   template:
     src: unqualified.conf.j2
     dest: "/etc/containers/registries.conf.d/01-unqualified.conf"
     mode: 0644
   notify: restart crio
 
-- name: Write cri-o proxy drop-in
+- name: cri-o | write cri-o proxy drop-in
   template:
     src: http-proxy.conf.j2
     dest: /etc/systemd/system/crio.service.d/http-proxy.conf
@@ -213,7 +166,7 @@
   notify: restart crio
   when: http_proxy is defined or https_proxy is defined
 
-- name: Configure the uid/gid space for user namespaces
+- name: cri-o | configure the uid/gid space for user namespaces
   lineinfile:
     path: '{{ item.path }}'
     line: '{{ item.entry }}'
@@ -227,7 +180,7 @@
   loop_control:
     label: '{{ item.path }}'
 
-- name: Ensure crio service is started and enabled
+- name: cri-o | ensure crio service is started and enabled
   service:
     name: crio
     daemon_reload: true
@@ -235,18 +188,17 @@
     state: started
   register: service_start
 
-- name: Trigger service restart only when needed
+- name: cri-o | trigger service restart only when needed
   service:  # noqa 503
     name: crio
     state: restarted
   when:
     - config_install.changed
     - reg_auth_install.changed
-    - not package_install.changed
     - not service_start.changed
 
-- name: Verify that crio is running
+- name: cri-o | verify that crio is running
-  command: "crio-status info"
+  command: "{{ bin_dir }}/crio-status info"
   register: get_crio_info
   until: get_crio_info is succeeded
   changed_when: false

roles/container-engine/cri-o/tasks/setup-amazon.yaml

@@ -0,0 +1,38 @@
+---
+- name: Check that amzn2-extras.repo exists
+  stat:
+    path: /etc/yum.repos.d/amzn2-extras.repo
+  register: amzn2_extras_file_stat
+
+- name: Find docker repo in amzn2-extras.repo file
+  lineinfile:
+    dest: /etc/yum.repos.d/amzn2-extras.repo
+    line: "[amzn2extra-docker]"
+  check_mode: yes
+  register: amzn2_extras_docker_repo
+  when:
+    - amzn2_extras_file_stat.stat.exists
+
+- name: Remove docker repository
+  ini_file:
+    dest: /etc/yum.repos.d/amzn2-extras.repo
+    section: amzn2extra-docker
+    option: enabled
+    value: "0"
+    backup: yes
+    mode: 0644
+  when:
+    - amzn2_extras_file_stat.stat.exists
+    - not amzn2_extras_docker_repo.changed
+
+- name: Add container-selinux yum repo
+  yum_repository:
+    name: copr:copr.fedorainfracloud.org:lsm5:container-selinux
+    file: _copr_lsm5-container-selinux.repo
+    description: Copr repo for container-selinux owned by lsm5
+    baseurl: https://download.copr.fedorainfracloud.org/results/lsm5/container-selinux/epel-7-$basearch/
+    gpgcheck: yes
+    gpgkey: https://download.copr.fedorainfracloud.org/results/lsm5/container-selinux/pubkey.gpg
+    skip_if_unavailable: yes
+    enabled: yes
+    repo_gpgcheck: no

roles/container-engine/cri-o/vars/almalinux-8.yml

@@ -1 +0,0 @@
-centos-8.yml

roles/container-engine/cri-o/vars/amazon.yml

@@ -1,15 +0,0 @@
----
-
-crio_storage_driver: "overlay"
-
-crio_versioned_pkg:
-  "1.24":
-    - "cri-o-1.24.*"
-  "1.23":
-    - "cri-o-1.23.*"
-  "1.22":
-    - "cri-o-1.22.*"
-
-default_crio_packages: "{{ crio_versioned_pkg[crio_version] }}"
-
-crio_packages: "{{ centos_crio_packages | default(default_crio_packages) }}"

roles/container-engine/cri-o/vars/centos-7.yml

@@ -1,12 +0,0 @@
----
-crio_versioned_pkg:
-  "1.24":
-    - "cri-o-1.24.*"
-  "1.23":
-    - "cri-o-1.23.*"
-  "1.22":
-    - "cri-o-1.22.*"
-
-default_crio_packages: "{{ crio_versioned_pkg[crio_version] }}"
-
-crio_packages: "{{ centos_crio_packages | default(default_crio_packages) }}"

roles/container-engine/cri-o/vars/centos-8.yml

@@ -1,12 +0,0 @@
----
-crio_versioned_pkg:
-  "1.24":
-    - "cri-o-1.24.*"
-  "1.23":
-    - "cri-o-1.23.*"
-  "1.22":
-    - "cri-o-1.22.*"
-
-default_crio_packages: "{{ crio_versioned_pkg[crio_version] }}"
-
-crio_packages: "{{ centos_crio_packages | default(default_crio_packages) }}"

roles/container-engine/cri-o/vars/clearlinux.yml

@@ -1,6 +0,0 @@
----
-crio_packages:
-  - containers-basic
-
-crio_conmon: /usr/libexec/crio/conmon
-crio_seccomp_profile: /usr/share/defaults/crio/seccomp.json

roles/container-engine/cri-o/vars/debian.yml

@@ -1,25 +0,0 @@
----
-crio_versioned_pkg:
-  "1.24":
-    - "cri-o=1.24*"
-    - cri-o-runc
-  "1.23":
-    - "cri-o=1.23*"
-    - cri-o-runc
-  "1.22":
-    - "cri-o=1.22*"
-    - cri-o-runc
-
-crio_debian_buster_backports_packages:
-  - "libseccomp2"
-
-default_crio_packages: "{{ crio_versioned_pkg[crio_version] }}"
-
-crio_packages: "{{ debian_crio_packages | default(default_crio_packages) }}"
-
-# The crio_runtimes variable defines a list of OCI compatible runtimes.
-crio_runtimes:
-  - name: runc
-    path: /usr/sbin/runc
-    type: oci
-    root: /run/runc

roles/container-engine/cri-o/vars/fedora-36.yml

@@ -1,5 +0,0 @@
----
-crio_packages:
-  - cri-o
-
-crio_version: 1.24

roles/container-engine/cri-o/vars/fedora.yml

@@ -1,9 +0,0 @@
----
-crio_packages:
-  - cri-o
-
-crio_kubernetes_version_matrix:
-  "1.24": "1.23"
-  "1.23": "1.23"
-  "1.22": "1.22"
-crio_version: "{{ crio_kubernetes_version_matrix[crio_required_version] | default('1.23') }}"

roles/container-engine/cri-o/vars/oraclelinux-8.yml

@@ -1 +0,0 @@
-centos-8.yml

roles/container-engine/cri-o/vars/redhat.yml

@@ -1,4 +0,0 @@
----
-crio_packages:
-  - cri-o
-  - oci-systemd-hook

roles/container-engine/cri-o/vars/rocky-8.yml

@@ -1 +0,0 @@
-centos-8.yml

roles/container-engine/cri-o/vars/ubuntu.yml

@@ -1,22 +0,0 @@
----
-crio_versioned_pkg:
-  "1.24":
-    - "cri-o=1.24*"
-    - cri-o-runc
-  "1.23":
-    - "cri-o=1.23*"
-    - cri-o-runc
-  "1.22":
-    - "cri-o=1.22*"
-    - cri-o-runc
-
-default_crio_packages: "{{ crio_versioned_pkg[crio_version] }}"
-
-crio_packages: "{{ ubuntu_crio_packages | default(default_crio_packages) }}"
-
-# The crio_runtimes variable defines a list of OCI compatible runtimes.
-crio_runtimes:
-  - name: runc
-    path: /usr/sbin/runc
-    type: oci
-    root: /run/runc

roles/container-engine/skopeo/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+- name: skopeo | check if fedora coreos
+  stat:
+    path: /run/ostree-booted
+    get_attributes: no
+    get_checksum: no
+    get_mime: no
+  register: ostree
+
+- name: skopeo | set is_ostree
+  set_fact:
+    is_ostree: "{{ ostree.stat.exists }}"
+
+- name: skopeo | Uninstall skopeo package managed by package manager
+  package:
+    name: skopeo
+    state: absent
+  when:
+    - not (is_ostree or (ansible_distribution == "Flatcar Container Linux by Kinvolk") or (ansible_distribution == "Flatcar"))
+  ignore_errors: true  # noqa ignore-errors
+
+- name: skopeo | Download skopeo binary
+  include_tasks: "../../../download/tasks/download_file.yml"
+  vars:
+    download: "{{ download_defaults | combine(downloads.skopeo) }}"
+
+- name: Copy skopeo binary from download dir
+  copy:
+    src: "{{ downloads.skopeo.dest }}"
+    dest: "{{ bin_dir }}/skopeo"
+    mode: 0755
+    remote_src: true

roles/download/defaults/main.yml

@@ -126,6 +126,7 @@ multus_version: "v3.8-{{ image_arch }}"
 helm_version: "v3.9.4"
 nerdctl_version: "0.22.2"
 krew_version: "v0.4.3"
+skopeo_version: v1.10.0
 
 # Get kubernetes major version (i.e. 1.17.4 => 1.17)
 kube_major_version: "{{ kube_version | regex_replace('^v([0-9])+\\.([0-9]+)\\.[0-9]+', 'v\\1.\\2') }}"
@@ -142,6 +143,12 @@ crictl_supported_versions:
   v1.23: "v1.23.0"
 crictl_version: "{{ crictl_supported_versions[kube_major_version] }}"
 
+crio_supported_versions:
+  v1.25: v1.25.1
+  v1.24: v1.24.3
+  v1.23: v1.23.2
+crio_version: "{{ crio_supported_versions[kube_major_version] }}"
+
 # Download URLs
 kubelet_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubelet"
 kubectl_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/{{ image_arch }}/kubectl"
@@ -152,6 +159,7 @@ calicoctl_download_url: "https://github.com/projectcalico/calico/releases/downlo
 calicoctl_alternate_download_url: "https://github.com/projectcalico/calicoctl/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
 calico_crds_download_url: "https://github.com/projectcalico/calico/archive/{{ calico_version }}.tar.gz"
 crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
+crio_download_url: "https://storage.googleapis.com/cri-o/artifacts/cri-o.{{ image_arch }}.{{ crio_version }}.tar.gz"
 helm_download_url: "https://get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
 runc_download_url: "https://github.com/opencontainers/runc/releases/download/{{ runc_version }}/runc.{{ image_arch }}"
 crun_download_url: "https://github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}"
@@ -164,6 +172,7 @@ nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v
 krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew-{{ host_os }}_{{ image_arch }}.tar.gz"
 containerd_download_url: "https://github.com/containerd/containerd/releases/download/v{{ containerd_version }}/containerd-{{ containerd_version }}-linux-{{ image_arch }}.tar.gz"
 cri_dockerd_download_url: "https://github.com/Mirantis/cri-dockerd/releases/download/v{{ cri_dockerd_version }}/cri-dockerd-{{ cri_dockerd_version }}.{{ image_arch }}.tgz"
+skopeo_download_url: "https://github.com/lework/skopeo-binary/releases/download/{{ skopeo_version }}/skopeo-linux-{{ image_arch }}"
 
 crictl_checksums:
   arm:
@@ -183,6 +192,24 @@ crictl_checksums:
     v1.24.0: 586c263678c6d8d543976607ea1732115e622d44993e2bcbed29832370d3a754
     v1.23.0: 53db9e605a3042ea77bbf42a01a4e248dea8839bcab544c491745874f73aeee7
 
+crio_archive_checksums:
+  arm:
+    v1.25.1: 0
+    v1.24.3: 0
+    v1.23.2: 0
+  arm64:
+    v1.25.1: add26675dc993b292024d007fd69980d8d1e75c675851d0cb687fe1dfd1f3008
+    v1.24.3: d8040602e03c90e4482b4ce97b63c2cf1301cd2afb0aa722342f40f3537a1a1f
+    v1.23.2: a866ccc3a062ac29906a619b9045a5e23b11fa9249f8802f8be0849491d01fbd
+  amd64:
+    v1.25.1: 49f98a38805740c40266a5bf3badc28e4ca725ccf923327c75c00fccc241f562
+    v1.24.3: 43f6e3a7ad6ae8cf05ed0f1e493578c28abf6a798aedb8ee9643ff7c25a68ca3
+    v1.23.2: 5c766dbf366a80f8b5dbc7a06d566f43e7cb0675186c50062df01f3b3cb5e526
+  ppc64le:
+    v1.25.1: 0
+    v1.24.3: 0
+    v1.23.2: 0
+
 # Checksum
 # Kubernetes versions above Kubespray's current target version are untested and should be used with caution.
 kubelet_checksums:
@@ -810,6 +837,16 @@ containerd_archive_checksums:
     1.6.7: 0db5cb6d5dd4f3b7369c6945d2ec29a9c10b106643948e3224e53885f56863a9
     1.6.8: f18769721f614828f6b778030c72dc6969ce2108f2363ddc85f6c7a147df0fb8
 
+skopeo_binary_checksums:
+  arm:
+    v1.10.0: 0
+  arm64:
+    v1.10.0: 3bfc344d4940df29358f8056de7b8dd488b88a5d777b3106748ba66851fa2c58
+  amd64:
+    v1.10.0: 20fbd1bac1d33768c3671e4fe9d90c5233d7e13a40e4935b4b24ebc083390604
+  ppc64l3:
+    v1.10.0: 0
+
 etcd_binary_checksum: "{{ etcd_binary_checksums[image_arch][etcd_version] }}"
 cni_binary_checksum: "{{ cni_binary_checksums[image_arch][cni_version] }}"
 kubelet_binary_checksum: "{{ kubelet_checksums[image_arch][kube_version] }}"
@@ -818,6 +855,7 @@ kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}"
 calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}"
 calico_crds_archive_checksum: "{{ calico_crds_archive_checksums[calico_version] }}"
 crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}"
+crio_archive_checksum: "{{ crio_archive_checksums[image_arch][crio_version] }}"
 cri_dockerd_archive_checksum: "{{ cri_dockerd_archive_checksums[image_arch][cri_dockerd_version] }}"
 helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}"
 runc_binary_checksum: "{{ runc_checksums[image_arch][runc_version] }}"
@@ -829,6 +867,7 @@ gvisor_containerd_shim_binary_checksum: "{{ gvisor_containerd_shim_binary_checks
 nerdctl_archive_checksum: "{{ nerdctl_archive_checksums[image_arch][nerdctl_version] }}"
 krew_archive_checksum: "{{ krew_archive_checksums[host_os][image_arch][krew_version] }}"
 containerd_archive_checksum: "{{ containerd_archive_checksums[image_arch][containerd_version] }}"
+skopeo_binary_checksum: "{{ skopeo_binary_checksums[image_arch][skopeo_version] }}"
 
 # Containers
 # In some cases, we need a way to set --registry-mirror or --insecure-registry for docker,
@@ -1111,6 +1150,19 @@ downloads:
     groups:
     - k8s_cluster
 
+  crio:
+    file: true
+    enabled: "{{ container_manager == 'crio' }}"
+    version: "{{ crio_version }}"
+    dest: "{{ local_release_dir }}/cri-o.{{ image_arch }}.{{ crio_version }}tar.gz"
+    sha256: "{{ crio_archive_checksum }}"
+    url: "{{ crio_download_url }}"
+    unarchive: true
+    owner: "root"
+    mode: "0755"
+    groups:
+    - k8s_cluster
+
   cri_dockerd:
     file: true
     enabled: "{{ container_manager == 'docker' }}"
@@ -1230,6 +1282,19 @@ downloads:
     groups:
     - k8s_cluster
 
+  skopeo:
+    file: true
+    enabled: "{{ container_manager == 'crio' }}"
+    version: "{{ skopeo_version }}"
+    dest: "{{ local_release_dir }}/skopeo"
+    sha256: "{{ skopeo_binary_checksum }}"
+    url: "{{ skopeo_download_url }}"
+    unarchive: false
+    owner: "root"
+    mode: "0755"
+    groups:
+    - kube_control_plane
+
   cilium:
     enabled: "{{ kube_network_plugin == 'cilium' or cilium_deploy_additionally | default(false) | bool }}"
     container: true

roles/download/tasks/set_container_facts.yml

@@ -35,8 +35,8 @@
 
 - name: Set image save/load command for crio
   set_fact:
-    image_save_command: "skopeo copy containers-storage:{{ image_reponame }} docker-archive:{{ image_path_final }}"
+    image_save_command: "{{ bin_dir }}/skopeo copy containers-storage:{{ image_reponame }} docker-archive:{{ image_path_final }} 2>/dev/null"
-    image_load_command: "skopeo copy docker-archive:{{ image_path_final }} containers-storage:{{ image_reponame }}"
+    image_load_command: "{{ bin_dir }}/skopeo copy docker-archive:{{ image_path_final }} containers-storage:{{ image_reponame }} 2>/dev/null"
   when: container_manager == 'crio'
 
 - name: Set image save/load command for docker on localhost
@@ -51,5 +51,5 @@
 
 - name: Set image save/load command for crio on localhost
   set_fact:
-    image_save_command_on_localhost: "skopeo copy containers-storage:{{ image_reponame }} docker-archive:{{ image_path_final }}"
+    image_save_command_on_localhost: "{{ bin_dir }}/skopeo copy containers-storage:{{ image_reponame }} docker-archive:{{ image_path_final }} 2>/dev/null"
   when: container_manager_on_localhost == 'crio'

roles/kubernetes/node/tasks/facts.yml

@@ -13,7 +13,7 @@
 
 - block:
   - name: look up crio cgroup driver
-    shell: "crio-status info | grep 'cgroup driver' | awk -F': ' '{ print $2; }'"
+    shell: "{{ bin_dir }}/crio-status info | grep 'cgroup driver' | awk -F': ' '{ print $2; }'"
     register: crio_cgroup_driver_result
     changed_when: false
 

roles/kubernetes/preinstall/tasks/0070-system-packages.yml

@@ -9,6 +9,28 @@
     - ansible_pkg_mgr == 'zypper'
   tags: bootstrap-os
 
+- block:
+    - name: Add Debian Backports apt repo
+      apt_repository:
+        repo: "deb http://deb.debian.org/debian {{ ansible_distribution_release }}-backports main"
+        state: present
+        filename: debian-backports
+
+    - name: Set libseccomp2 pin priority to apt_preferences on Debian buster
+      copy:
+        content: |
+          Package: libseccomp2
+          Pin: release a={{ ansible_distribution_release }}-backports
+          Pin-Priority: 1001
+        dest: "/etc/apt/preferences.d/libseccomp2"
+        owner: "root"
+        mode: 0644
+  when:
+    - ansible_distribution == "Debian"
+    - ansible_distribution_version == "10"
+  tags:
+    - bootstrap-os
+
 - name: Update package management cache (APT)
   apt:
     update_cache: yes

roles/kubernetes/preinstall/vars/fedora.yml

@@ -1,5 +1,6 @@
 ---
 required_pkgs:
+  - iptables
   - libselinux-python3
   - device-mapper-libs
   - conntrack