containerd support (#4664)

* Add limited containerd support

Containerd support for Ubuntu + Calico

* Added CRI-O support for ubuntu

* containerd support.

* Reset  containerd support.

* fix lint.

* implemented feedback

* Change task name cri xx instead of cri-o in reset task and timeout condition.

* set crictl to fixed version

* Use docker-ce's container.io package for containerd.

* Add check containerd is installable or not.

* Avoid stop docker when use containerd and optimize retry for reset.

* Add config.toml.

* Fixed containerd for kubelet.env.

* Merge PR #4629

* Remove unused ubuntu variable for containerd

* Polish code for containerd and cri-o

* Refactoring cri socket configuration.

* Configurable conmon.

* Remove unused crictl/runc download

* Now crictl and runc is downloaded by common crictl.yml.

* fixed yamllint error

* Fixed brokenfiles by conflict.

* Remove commented line in config.toml

* Remove readded v1.12.x version

* Fixed broken set_docker_image_facts

* Fix yamllint errors.

* Remove unused apt source

* Fix crictl could not be installed

* Add containerd config from skolekonov's PR #4601
This commit is contained in:
okamototk 2019-06-30 06:09:20 +09:00 committed by Kubernetes Prow Robot
parent 216631bf02
commit 4c8b93e5b9
37 changed files with 610 additions and 214 deletions

View file

@ -145,7 +145,7 @@ skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipad
dns_domain: "{{ cluster_name }}" dns_domain: "{{ cluster_name }}"
## Container runtime ## Container runtime
## docker for docker and crio for cri-o. ## docker for docker, crio for cri-o and containerd for containerd.
container_manager: docker container_manager: docker
## Settings for containerized control plane (etcd/kubelet/secrets) ## Settings for containerized control plane (etcd/kubelet/secrets)

View file

@ -0,0 +1,12 @@
---
kubelet_cgroup_driver: systemd
containerd_config:
grpc:
max_recv_message_size: 16777216
max_send_message_size: 16777216
debug:
level: ""
registries:
"docker.io": "https://registry-1.docker.io"
max_container_log_line_size: -1

View file

@ -0,0 +1,24 @@
---
- name: restart containerd
command: /bin/true
notify:
- Containerd | reload containerd
- Containerd | pause while containerd restarts
- Containerd | wait for containerd
- name: Containerd | reload containerd
service:
name: containerd
state: restarted
- name: Containerd | pause while containerd restarts
pause:
seconds: 5
prompt: "Waiting for containerd restart"
- name: Containerd | wait for containerd
command: "{{ containerd_bin_dir }}/ctr images ls -q"
register: containerd_ready
retries: 10
delay: 5
until: containerd_ready.rc == 0

View file

@ -0,0 +1,26 @@
---
- name: crictl | Download crictl
include_tasks: "roles/download/tasks/download_file.yml"
vars:
download: "{{ download_defaults | combine(downloads.crictl) }}"
- name: Install crictl config
template:
src: ../templates/crictl.yaml.j2
dest: /etc/crictl.yaml
owner: bin
mode: 0644
- name: Copy crictl binary from download dir
synchronize:
src: "{{ local_release_dir }}/crictl"
dest: "{{ bin_dir }}/crictl"
compress: no
perms: yes
owner: no
group: no
delegate_to: "{{ inventory_hostname }}"
- name: Install crictl completion
shell: /usr/local/bin/crictl completion >/etc/bash_completion.d/crictl
ignore_errors: True

View file

@ -0,0 +1,50 @@
---
- name: Fail containerd setup if distribution is not supported
fail:
msg: "{{ ansible_distribution }} is not supported by containerd."
when:
- not ansible_distribution in ["CentOS","RedHat", "Ubuntu", "Debian"]
- name: Install Docker
include_role:
name: container-engine/docker
- name: Install config.toml
template:
src: config.toml.j2
dest: /etc/containerd/config.toml
owner: bin
mode: 0644
- name: Stop and disabled Docker
systemd:
name: docker
state: stopped
enabled: no
- name: Restart containerd
systemd:
name: containerd
state: restarted
- name: Install crictl config
template:
src: crictl.yaml.j2
dest: /etc/crictl.yaml
owner: bin
mode: 0644
- name: Install crictl completion
shell: /usr/local/bin/crictl completion >/etc/bash_completion.d/crictl
ignore_errors: True
when: ansible_distribution in ["CentOS","RedHat", "Ubuntu", "Debian"]
- name: Enable containerd
systemd:
name: containerd.service
state: started
enabled: yes
daemon-reload: yes
- name: flush handlers so we can wait for containerd to come up
meta: flush_handlers

View file

@ -0,0 +1,40 @@
# Kubernetes doesn't use containerd restart manager.
disabled_plugins = ["restart"]
[debug]
level = "{{ containerd_config.debug.level | default("") }}"
{% if 'grpc' in containerd_config %}
[grpc]
{% for param, value in containerd_config.grpc.items() %}
{{ param }} = {{ value }}
{% endfor %}
{% endif %}
[plugins.linux]
shim = "/usr/bin/containerd-shim"
runtime = "/usr/sbin/runc"
[plugins.cri]
stream_server_address = "127.0.0.1"
max_container_log_line_size = {{ containerd_config.max_container_log_line_size }}
sandbox_image = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
[plugins.cri.cni]
bin_dir = "/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
conf_template = ""
[plugins.cri.containerd.untrusted_workload_runtime]
runtime_type = ""
runtime_engine = ""
runtime_root = ""
{% if 'registries' in containerd_config %}
[plugins.cri.registry]
[plugins.cri.registry.mirrors]
{% for registry, addr in containerd_config.registries.items() %}
[plugins.cri.registry.mirrors."{{ registry }}"]
endpoint = ["{{ addr }}"]
{% endfor %}
{% endif %}

View file

@ -0,0 +1,4 @@
runtime-endpoint: unix://{{ cri_socket }}
image-endpoint: unix://{{ cri_socket }}
timeout: 30
debug: false

View file

@ -24,6 +24,12 @@
gpgcheck: no gpgcheck: no
when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic
- name: Add CRI-O PPA
apt_repository:
repo: ppa:projectatomic/ppa
state: present
when: ansible_distribution in ["Ubuntu"]
- name: Make sure needed folders exist in the system - name: Make sure needed folders exist in the system
with_items: with_items:
- /etc/crio - /etc/crio

View file

@ -64,7 +64,7 @@ file_locking = true
# This is a mandatory setting as this runtime will be the default one # This is a mandatory setting as this runtime will be the default one
# and will also be used for untrusted container workloads if # and will also be used for untrusted container workloads if
# runtime_untrusted_workload is not set. # runtime_untrusted_workload is not set.
{% if ansible_os_family == "ClearLinux" or ansible_os_family == "RedHat" %} {% if ansible_os_family == "ClearLinux" or ansible_os_family == "RedHat" or ansible_distribution == "Ubuntu" %}
runtime = "/usr/bin/runc" runtime = "/usr/bin/runc"
{% else %} {% else %}
runtime = "/usr/sbin/runc" runtime = "/usr/sbin/runc"
@ -96,7 +96,7 @@ default_workload_trust = "trusted"
no_pivot = false no_pivot = false
# conmon is the path to conmon binary, used for managing the runtime. # conmon is the path to conmon binary, used for managing the runtime.
conmon = "/usr/libexec/crio/conmon" conmon = "{{ crio_conmon }}"
# conmon_env is the environment variable list for conmon process, # conmon_env is the environment variable list for conmon process,
# used for passing necessary environment variable to conmon or runtime. # used for passing necessary environment variable to conmon or runtime.

View file

@ -3,3 +3,4 @@ crio_packages:
- containers-basic - containers-basic
crio_service: crio crio_service: crio
crio_conmon: /usr/libexec/crio/conmon

View file

@ -4,3 +4,4 @@ crio_packages:
- cri-tools - cri-tools
crio_service: cri-o crio_service: cri-o
crio_conmon: /usr/libexec/crio/conmon

View file

@ -5,3 +5,4 @@ crio_packages:
- oci-systemd-hook - oci-systemd-hook
crio_service: crio crio_service: crio
crio_conmon: /usr/libexec/crio/conmon

View file

@ -0,0 +1,6 @@
---
crio_packages:
- "cri-o-{{ kube_version | regex_replace('^v(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)$', '\\g<major>.\\g<minor>') }}"
crio_service: crio
crio_conmon: /usr/lib/crio/bin/conmon

View file

@ -7,9 +7,23 @@ dependencies:
- container-engine - container-engine
- crio - crio
- role: container-engine/containerd
when:
- container_manager == 'containerd'
tags:
- container-engine
- containerd
- role: container-engine/docker - role: container-engine/docker
when: when:
- container_manager == 'docker' - container_manager == 'docker'
tags: tags:
- container-engine - container-engine
- docker - docker
- role: container-engine/containerd
when:
- container_manager == 'containerd'
tags:
- container-engine
- containerd

View file

@ -74,7 +74,9 @@ pod_infra_version: 3.1
contiv_version: 1.2.1 contiv_version: 1.2.1
cilium_version: "v1.3.0" cilium_version: "v1.3.0"
kube_router_version: "v0.2.5" kube_router_version: "v0.2.5"
multus_version: "v3.2.1" multus_version: "v3.1.autoconf"
crictl_version: "v1.14.0"
# Download URLs # Download URLs
kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm" kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm"
@ -82,6 +84,18 @@ hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/relea
etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz" etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
cni_download_url: "https://github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz" cni_download_url: "https://github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
calicoctl_download_url: "https://github.com/projectcalico/calicoctl/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}" calicoctl_download_url: "https://github.com/projectcalico/calicoctl/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
crictl_checksums:
arm:
v1.14.0: 9910cecfd6558239ba015323066c7233d8371af359b9ddd0b2a35d5223bcf945
v1.13.0: 2e478ebed85f9d70d49fd8f1d1089c8fba6e37d3461aeef91813f1ab0f0df586
arm64:
v1.14.0: f76b3d00a272c8d210e9a45f77d07d3770bee310d99c4fd9a72d6f55278882e5
v1.13.0: 68949c0cb5a37e7604c145d189cf1e109c08c93d9c710ba663db026b9c6f2746
amd64:
v1.14.0: 483c90a9fe679590df4332ba807991c49232e8cd326c307c575ecef7fe22327b
v1.13.0: 9bdbea7a2b382494aff2ff014da328a042c5aba9096a7772e57fdf487e5a1d51
# Checksums # Checksums
hyperkube_checksums: hyperkube_checksums:
@ -164,6 +178,9 @@ kubeadm_checksums:
v1.13.2: 7cb0ce57c1e6e2d85e05de3780a2f35a191fe93f89cfc5816b424efcf39834b9 v1.13.2: 7cb0ce57c1e6e2d85e05de3780a2f35a191fe93f89cfc5816b424efcf39834b9
v1.13.1: 438173bfa0b7014ecae994c5b9e1f27e1328ab971a3fdb06a393a8095a176ba0 v1.13.1: 438173bfa0b7014ecae994c5b9e1f27e1328ab971a3fdb06a393a8095a176ba0
v1.13.0: f5366206416dc4cfc840a7add2289957b56ccc479cc1b74f7397a4df995d6b06 v1.13.0: f5366206416dc4cfc840a7add2289957b56ccc479cc1b74f7397a4df995d6b06
crictl_binary_checksums:
amd64:
v1.14.0: 483c90a9fe679590df4332ba807991c49232e8cd326c307c575ecef7fe22327b
etcd_binary_checksums: etcd_binary_checksums:
# Etcd does not have arm32 builds at the moment, having some dummy value is # Etcd does not have arm32 builds at the moment, having some dummy value is
@ -194,6 +211,7 @@ cni_binary_checksum: "{{ cni_binary_checksums[image_arch] }}"
hyperkube_binary_checksum: "{{ hyperkube_checksums[image_arch][kube_version] }}" hyperkube_binary_checksum: "{{ hyperkube_checksums[image_arch][kube_version] }}"
kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}" kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}"
calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}" calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}"
crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}"
# Containers # Containers
# In some cases, we need a way to set --registry-mirror or --insecure-registry for docker, # In some cases, we need a way to set --registry-mirror or --insecure-registry for docker,
@ -304,6 +322,9 @@ addon_resizer_image_tag: "{{ addon_resizer_version }}"
dashboard_image_repo: "gcr.io/google_containers/kubernetes-dashboard-{{ image_arch }}" dashboard_image_repo: "gcr.io/google_containers/kubernetes-dashboard-{{ image_arch }}"
dashboard_image_tag: "v1.10.1" dashboard_image_tag: "v1.10.1"
image_pull_command: "{{ docker_bin_dir }}/docker pull"
image_info_command: "{{ docker_bin_dir }}/docker images -q | xargs {{ docker_bin_dir }}/docker inspect -f \"{{ '{{' }} if .RepoTags {{ '}}' }}{{ '{{' }} (index .RepoTags 0) {{ '}}' }}{{ '{{' }} end {{ '}}' }}{{ '{{' }} if .RepoDigests {{ '}}' }},{{ '{{' }} (index .RepoDigests 0) {{ '}}' }}{{ '{{' }} end {{ '}}' }}\" | tr '\n' ','"
downloads: downloads:
netcheck_server: netcheck_server:
enabled: "{{ deploy_netchecker }}" enabled: "{{ deploy_netchecker }}"
@ -378,6 +399,19 @@ downloads:
groups: groups:
- k8s-cluster - k8s-cluster
crictl:
file: true
enabled: "{{ container_manager in ['crio', 'cri', 'containerd'] }}"
version: "{{ crictl_version }}"
dest: "{{local_release_dir}}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz"
sha256: "{{ crictl_binary_checksum }}"
url: "{{ crictl_download_url }}"
unarchive: true
owner: "root"
mode: "0755"
groups:
- k8s-cluster
cilium: cilium:
enabled: "{{ kube_network_plugin == 'cilium' }}" enabled: "{{ kube_network_plugin == 'cilium' }}"
container: true container: true

View file

@ -1,129 +1,137 @@
--- ---
- name: container_download | Make download decision if pull is required by tag or sha256
include_tasks: set_docker_image_facts.yml
when:
- download.enabled
- download.container
tags:
- facts
- block: - block:
- name: download_container | Set a few facts - name: download_container | Set a few facts
import_tasks: set_container_facts.yml import_tasks: set_container_facts.yml
run_once: "{{ download_run_once }}" run_once: "{{ download_run_once }}"
tags: tags:
- facts - facts
- name: download_container | Determine if image is in cache - name: download_container | Determine if image is in cache
stat: stat:
path: "{{ image_path_cached }}" path: "{{ image_path_cached }}"
delegate_to: localhost delegate_to: localhost
delegate_facts: no delegate_facts: no
register: cache_image register: cache_image
changed_when: false changed_when: false
become: false become: false
when: when:
- download_force_cache - download_force_cache
- name: download_container | Set fact indicating if image is in cache - name: download_container | Set fact indicating if image is in cache
set_fact: set_fact:
image_is_cached: "{{ cache_image.stat.exists | default(false) }}" image_is_cached: "{{ cache_image.stat.exists | default(false) }}"
tags: tags:
- facts - facts
when: when:
- download_force_cache - download_force_cache
- name: download_container | Upload image to node if it is cached - name: download_container | Upload image to node if it is cached
synchronize: synchronize:
src: "{{ image_path_cached }}" src: "{{ image_path_cached }}"
dest: "{{ image_path_final }}" dest: "{{ image_path_final }}"
use_ssh_args: "{{ has_bastion | default(false) }}" use_ssh_args: "{{ has_bastion | default(false) }}"
mode: push mode: push
delegate_facts: no delegate_facts: no
register: upload_image register: upload_image
failed_when: not upload_image failed_when: not upload_image
run_once: "{{ download_run_once }}" run_once: "{{ download_run_once }}"
until: upload_image is succeeded until: upload_image is succeeded
retries: 4 retries: 4
delay: "{{ retry_stagger | random + 3 }}" delay: "{{ retry_stagger | random + 3 }}"
when: when:
- download_force_cache - download_force_cache
- image_is_cached - image_is_cached
- not download_localhost - not download_localhost
- ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
- name: download_container | Load image into docker - name: download_container | Load image into docker
shell: "{{ docker_bin_dir }}/docker load < {{ image_path_cached if download_localhost else image_path_final }}" shell: "{{ docker_bin_dir }}/docker load < {{ image_path_cached if download_localhost else image_path_final }}"
delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}" delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
run_once: "{{ download_run_once }}" run_once: "{{ download_run_once }}"
register: container_load_status register: container_load_status
failed_when: container_load_status | failed failed_when: container_load_status | failed
become: "{{ user_can_become_root | default(false) or not (download_run_once and download_localhost) }}" become: "{{ user_can_become_root | default(false) or not (download_run_once and download_localhost) }}"
when: when:
- download_force_cache - download_force_cache
- image_is_cached - image_is_cached
- ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
- name: download_container | Prepare container download - name: download_container | Prepare container download
import_tasks: check_pull_required.yml import_tasks: check_pull_required.yml
run_once: "{{ download_run_once }}" run_once: "{{ download_run_once }}"
when: when:
- not download_always_pull - not download_always_pull
- debug: - debug:
msg: "XXX Pull required is: {{ pull_required }}" msg: "XXX Pull required is: {{ pull_required }}"
# NOTE: Pre-loading docker images will not prevent 'docker pull' from re-downloading the layers in that image # NOTE: Pre-loading docker images will not prevent 'docker pull' from re-downloading the layers in that image
# if a pull is forced. This is a known issue with docker. See https://github.com/moby/moby/issues/23684 # if a pull is forced. This is a known issue with docker. See https://github.com/moby/moby/issues/23684
- name: download_container | Download image if required - name: download_container | Download image if required
command: "{{ docker_bin_dir }}/docker pull {{ image_reponame }}" command: "{{ image_pull_command }} {{ image_reponame }}"
delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}" delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
delegate_facts: yes delegate_facts: yes
run_once: "{{ download_run_once }}" run_once: "{{ download_run_once }}"
register: pull_task_result register: pull_task_result
until: pull_task_result is succeeded until: pull_task_result is succeeded
delay: "{{ retry_stagger | random + 3 }}" delay: "{{ retry_stagger | random + 3 }}"
retries: 4 retries: 4
become: "{{ user_can_become_root | default(false) or not download_localhost }}" become: "{{ user_can_become_root | default(false) or not download_localhost }}"
when: when:
- pull_required | default(download_always_pull) - pull_required | default(download_always_pull)
# NOTE: image_changed is only valid if a pull is was needed or forced. # NOTE: image_changed is only valid if a pull is was needed or forced.
- name: download_container | Check if image changed - name: download_container | Check if image changed
set_fact: set_fact:
image_changed: "{{ true if pull_task_result.stdout is defined and not 'up to date' in pull_task_result.stdout else false }}" image_changed: "{{ true if pull_task_result.stdout is defined and not 'up to date' in pull_task_result.stdout else false }}"
run_once: true run_once: true
when: when:
- download_force_cache - download_force_cache
tags: tags:
- facts - facts
- name: download_container | Save and compress image - name: download_container | Save and compress image
shell: "{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_cached if download_localhost else image_path_final }}" shell: "{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_cached if download_localhost else image_path_final }}"
delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}" delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}"
delegate_facts: no delegate_facts: no
register: container_save_status register: container_save_status
failed_when: container_save_status.stderr failed_when: container_save_status.stderr
run_once: true run_once: true
become: "{{ user_can_become_root | default(false) or not download_localhost }}" become: "{{ user_can_become_root | default(false) or not download_localhost }}"
when: when:
- download_force_cache - download_force_cache
- not image_is_cached or (image_changed | default(true)) - not image_is_cached or (image_changed | default(true))
- ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
- name: download_container | Copy image to ansible host cache - name: download_container | Copy image to ansible host cache
synchronize: synchronize:
src: "{{ image_path_final }}" src: "{{ image_path_final }}"
dest: "{{ image_path_cached }}" dest: "{{ image_path_cached }}"
use_ssh_args: "{{ has_bastion | default(false) }}" use_ssh_args: "{{ has_bastion | default(false) }}"
mode: pull mode: pull
delegate_facts: no delegate_facts: no
run_once: true run_once: true
when: when:
- download_force_cache - download_force_cache
- not download_localhost - not download_localhost
- not image_is_cached or (image_changed | default(true)) - not image_is_cached or (image_changed | default(true))
- ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
- name: download_container | Remove container image from cache - name: download_container | Remove container image from cache
file: file:
state: absent state: absent
path: "{{ image_path_final }}" path: "{{ image_path_final }}"
when: when:
- not download_keep_remote_cache - not download_keep_remote_cache
- ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
tags: tags:
- download - download

View file

@ -0,0 +1,42 @@
---
# Use the same format for Containerd images as for Docker images
# ctr doesn't have inspect command
- name: Set image info command for containerd
set_fact:
image_info_command: "{{ containerd_bin_dir }}/ctr images ls | tail -n +2 | awk -F '[ :]+' '{print $1\":\"$2\",\"$1\":\"$4\"@\"$5}' | tr '\n' ','"
when: container_manager == 'containerd'
- name: Register docker images info
shell: "{{ image_info_command }}"
no_log: true
register: docker_images
failed_when: false
changed_when: false
check_mode: no
when: download_container
- name: container_download | Create dest directory for saved/loaded container images
file:
path: "{{ local_release_dir }}/containers"
state: directory
recurse: yes
mode: 0755
owner: "{{ ansible_ssh_user|default(ansible_user_id) }}"
when: download_container
- name: container_download | create local directory for saved/loaded container images
file:
path: "{{ local_release_dir }}/containers"
state: directory
recurse: yes
delegate_to: localhost
delegate_facts: false
become: false
run_once: true
when:
- download_run_once
- download_delegate == 'localhost'
- download_container
tags:
- localhost

View file

@ -7,6 +7,25 @@
- download - download
- upload - upload
- name: Use cri-o for cri connection
set_fact:
cri_socket: /var/run/crio/crio.sock
when: container_manager == 'crio'
- name: Use containerd for cri connetion
set_fact:
cri_socket: /var/run/containerd/containerd.sock
when: container_manager == 'containerd'
- name: Use docker for cri connetion
set_fact:
cri_socket: /var/run/dockershim.sock
when: container_manager == 'docker'
- include_tasks: ../../container-engine/containerd/tasks/crictl.yml
when:
- container_manager in ['containerd', 'crio']
- name: download | Get kubeadm binary and list of required images - name: download | Get kubeadm binary and list of required images
import_tasks: prep_kubeadm_images.yml import_tasks: prep_kubeadm_images.yml
when: when:

View file

@ -0,0 +1,54 @@
---
- name: Set if containers should be pulled by digest
set_fact:
pull_by_digest: >-
{%- if download.sha256 is defined and download.sha256 -%}true{%- else -%}false{%- endif -%}
- name: Set pull_args
set_fact:
pull_args: >-
{%- if pull_by_digest %}{{ download.repo }}@sha256:{{ download.sha256 }}{%- else -%}{{ download.repo }}:{{ download.tag }}{%- endif -%}
- name: Set image pull command for containerd
set_fact:
image_pull_command: "{{ bin_dir }}/crictl pull"
when: container_manager in ['crio' ,'containerd']
- name: Register docker images info
shell: "{{ image_info_command }}"
no_log: true
register: docker_images
failed_when: false
changed_when: false
check_mode: no
when:
- not download_always_pull
- group_names | intersect(download.groups) | length
- name: Set if pull is required per container
set_fact:
pull_required: >-
{%- if pull_args in docker_images.stdout.split(',') %}false{%- else -%}true{%- endif -%}
when:
- not download_always_pull
- group_names | intersect(download.groups) | length
- name: Does any host require container pull?
vars:
hosts_pull_required: "{{ hostvars.values() | map(attribute='pull_required') | select('defined') | list }}"
set_fact:
any_pull_required: "{{ True in hosts_pull_required }}"
run_once: true
changed_when: false
when: not download_always_pull
- name: Check the local digest sha256 corresponds to the given image tag
assert:
that: "{{ download.repo }}:{{ download.tag }} in docker_images.stdout.split(',')"
when:
- group_names | intersect(download.groups) | length
- not download_always_pull
- not pull_required
- pull_by_digest
tags:
- asserts

View file

@ -6,11 +6,7 @@ apiVersion: kubeadm.k8s.io/v1beta1
{% endif %} {% endif %}
kind: InitConfiguration kind: InitConfiguration
nodeRegistration: nodeRegistration:
{% if container_manager == 'crio' %} criSocket: {{ cri_socket }}
criSocket: /var/run/crio/crio.sock
{% else %}
criSocket: /var/run/dockershim.sock
{% endif %}
--- ---
{% endif %} {% endif %}
{% if kube_version is version('v1.11.0', '<') %} {% if kube_version is version('v1.11.0', '<') %}
@ -37,9 +33,5 @@ etcd:
{% endfor %} {% endfor %}
{% if kube_version is version('v1.12.0', '<') %} {% if kube_version is version('v1.12.0', '<') %}
nodeRegistration: nodeRegistration:
{% if container_manager == 'crio' %} criSocket: {{ cri_socket }}
criSocket: /var/run/crio/crio.sock
{% else %}
criSocket: /var/run/dockershim.sock
{% endif %}
{% endif %} {% endif %}

View file

@ -16,8 +16,4 @@ discoveryTokenAPIServers:
discoveryTokenUnsafeSkipCAVerification: true discoveryTokenUnsafeSkipCAVerification: true
nodeRegistration: nodeRegistration:
name: {{ kube_override_hostname }} name: {{ kube_override_hostname }}
{% if container_manager == 'crio' %} criSocket: {{ cri_socket }}
criSocket: /var/run/crio/crio.sock
{% else %}
criSocket: /var/run/dockershim.sock
{% endif %}

View file

@ -16,8 +16,4 @@ discoveryTokenAPIServers:
discoveryTokenUnsafeSkipCAVerification: true discoveryTokenUnsafeSkipCAVerification: true
nodeRegistration: nodeRegistration:
name: {{ kube_override_hostname }} name: {{ kube_override_hostname }}
{% if container_manager == 'crio' %} criSocket: {{ cri_socket }}
criSocket: /var/run/crio/crio.sock
{% else %}
criSocket: /var/run/dockershim.sock
{% endif %}

View file

@ -16,8 +16,4 @@ discovery:
caCertPath: {{ kube_cert_dir }}/ca.crt caCertPath: {{ kube_cert_dir }}/ca.crt
nodeRegistration: nodeRegistration:
name: {{ kube_override_hostname }} name: {{ kube_override_hostname }}
{% if container_manager == 'crio' %} criSocket: {{ cri_socket }}
criSocket: /var/run/crio/crio.sock
{% else %}
criSocket: /var/run/dockershim.sock
{% endif %}

View file

@ -228,11 +228,7 @@ nodeRegistration:
{% else %} {% else %}
taints: {} taints: {}
{% endif %} {% endif %}
{% if container_manager == 'crio' %} criSocket: {{ cri_socket }}
criSocket: /var/run/crio/crio.sock
{% else %}
criSocket: /var/run/dockershim.sock
{% endif %}
{% if dynamic_kubelet_configuration %} {% if dynamic_kubelet_configuration %}
featureGates: featureGates:
DynamicKubeletConfig: true DynamicKubeletConfig: true

View file

@ -14,11 +14,7 @@ nodeRegistration:
{% else %} {% else %}
taints: {} taints: {}
{% endif %} {% endif %}
{% if container_manager == 'crio' %} criSocket: {{ cri_socket }}
criSocket: /var/run/crio/crio.sock
{% else %}
criSocket: /var/run/dockershim.sock
{% endif %}
--- ---
apiVersion: kubeadm.k8s.io/v1alpha3 apiVersion: kubeadm.k8s.io/v1alpha3
kind: ClusterConfiguration kind: ClusterConfiguration

View file

@ -14,11 +14,7 @@ nodeRegistration:
{% else %} {% else %}
taints: [] taints: []
{% endif %} {% endif %}
{% if container_manager == 'crio' %} criSocket: {{ cri_socket }}
criSocket: /var/run/crio/crio.sock
{% else %}
criSocket: /var/run/dockershim.sock
{% endif %}
--- ---
apiVersion: kubeadm.k8s.io/v1beta1 apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration kind: ClusterConfiguration

View file

@ -17,8 +17,4 @@ controlPlane:
bindPort: {{ kube_apiserver_port }} bindPort: {{ kube_apiserver_port }}
nodeRegistration: nodeRegistration:
name: {{ kube_override_hostname|default(inventory_hostname) }} name: {{ kube_override_hostname|default(inventory_hostname) }}
{% if container_manager == 'crio' %} criSocket: {{ cri_socket }}
criSocket: /var/run/crio/crio.sock
{% else %}
criSocket: /var/run/dockershim.sock
{% endif %}

View file

@ -3,12 +3,23 @@
shell: "docker info | grep 'Cgroup Driver' | awk -F': ' '{ print $2; }'" shell: "docker info | grep 'Cgroup Driver' | awk -F': ' '{ print $2; }'"
register: docker_cgroup_driver_result register: docker_cgroup_driver_result
changed_when: false changed_when: false
when: container_manager in ['crio', 'docker', 'rkt']
- name: set facts - name: set standalone_kubelet fact
set_fact: set_fact:
standalone_kubelet: >- standalone_kubelet: >-
{%- if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] -%}true{%- else -%}false{%- endif -%} {%- if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] -%}true{%- else -%}false{%- endif -%}
- name: set kubelet_cgroup_driver_detected fact for containerd
set_fact:
kubelet_cgroup_driver_detected: >-
{%- if containerd_use_systemd_cgroup -%}systemd{%- else -%}cgroupfs{%- endif -%}
when: container_manager == 'containerd'
- name: set kubelet_cgroup_driver_detected fact for other engines
set_fact:
kubelet_cgroup_driver_detected: "{{ docker_cgroup_driver_result.stdout }}" kubelet_cgroup_driver_detected: "{{ docker_cgroup_driver_result.stdout }}"
when: container_manager in ['crio', 'docker', 'rkt']
- name: os specific vars - name: os specific vars
include_vars: "{{ item }}" include_vars: "{{ item }}"

View file

@ -1,12 +1,22 @@
--- ---
- name: "Pre-upgrade | check if kubelet container exists" - name: "Pre-upgrade | check if kubelet container exists"
shell: docker ps -af name=kubelet | grep kubelet shell: >-
{% if container_manager in ['crio', 'docker', 'rkt'] %}
docker ps -af name=kubelet | grep kubelet
{% elif container_manager == 'containerd' %}
crictl ps --all --name kubelet | grep kubelet
{% endif %}
failed_when: false failed_when: false
changed_when: false changed_when: false
register: kubelet_container_check register: kubelet_container_check
- name: "Pre-upgrade | copy /var/lib/cni from kubelet" - name: "Pre-upgrade | copy /var/lib/cni from kubelet"
command: docker cp kubelet:/var/lib/cni /var/lib/cni command: >-
{% if container_manager in ['crio', 'docker', 'rkt'] %}
docker cp kubelet:/var/lib/cni /var/lib/cni
{% elif container_manager == 'containerd' %}
ctr run --rm --mount type=bind,src=/var/lib/cni,dst=/cnilibdir,options=rbind:rw kubelet kubelet-tmp sh -c 'cp /var/lib/cni/* /cnilibdir/'
{% endif %}
args: args:
creates: "/var/lib/cni" creates: "/var/lib/cni"
failed_when: false failed_when: false
@ -19,7 +29,12 @@
when: kubelet_container_check.rc == 0 when: kubelet_container_check.rc == 0
- name: "Pre-upgrade | ensure kubelet container is removed if using host deployment" - name: "Pre-upgrade | ensure kubelet container is removed if using host deployment"
command: docker rm -fv kubelet shell: >-
{% if container_manager in ['crio', 'docker', 'rkt'] %}
docker rm -fv kubelet
{% elif container_manager == 'containerd' %}
crictl stop kubelet && crictl rm kubelet
{% endif %}
failed_when: false failed_when: false
changed_when: false changed_when: false
register: remove_kubelet_container register: remove_kubelet_container

View file

@ -43,9 +43,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
{% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %} {% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %}
--docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
{% endif %} {% endif %}
{% if container_manager == 'crio' %} {% if container_manager != 'docker' %}
--container-runtime=remote \ --container-runtime=remote \
--container-runtime-endpoint=/var/run/crio/crio.sock \ --container-runtime-endpoint={{ cri_socket }} \
{% endif %} {% endif %}
--anonymous-auth=false \ --anonymous-auth=false \
--read-only-port={{ kube_read_only_port }} \ --read-only-port={{ kube_read_only_port }} \

View file

@ -22,9 +22,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
{% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %} {% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %}
--docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
{% endif %} {% endif %}
{% if container_manager == 'crio' %} {% if container_manager != 'docker' %}
--container-runtime=remote \ --container-runtime=remote \
--container-runtime-endpoint=/var/run/crio/crio.sock \ --container-runtime-endpoint={{ cri_socket }} \
{% endif %} {% endif %}
{% if kube_version is version('v1.8', '<') %} {% if kube_version is version('v1.8', '<') %}
--experimental-fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \ --experimental-fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \

View file

@ -46,6 +46,7 @@ haproxy_config_dir: "/etc/haproxy"
# Directory where the binaries will be installed # Directory where the binaries will be installed
bin_dir: /usr/local/bin bin_dir: /usr/local/bin
docker_bin_dir: /usr/bin docker_bin_dir: /usr/bin
containerd_bin_dir: /usr/bin
etcd_data_dir: /var/lib/etcd etcd_data_dir: /var/lib/etcd
# Where the binaries will be downloaded. # Where the binaries will be downloaded.
# Note: ensure that you've enough disk space (about 1G) # Note: ensure that you've enough disk space (about 1G)
@ -258,6 +259,9 @@ docker_options: >-
# Experimental kubeadm etcd deployment mode. Available only for new deployment # Experimental kubeadm etcd deployment mode. Available only for new deployment
etcd_kubeadm_enabled: false etcd_kubeadm_enabled: false
# Containerd options
containerd_use_systemd_cgroup: false
# Settings for containerized control plane (etcd/kubelet/secrets) # Settings for containerized control plane (etcd/kubelet/secrets)
# deployment type for legacy etcd mode # deployment type for legacy etcd mode
etcd_deployment_type: docker etcd_deployment_type: docker

View file

@ -42,9 +42,19 @@
- name: Calico-rr | Write calico-rr systemd init file - name: Calico-rr | Write calico-rr systemd init file
template: template:
src: calico-rr.service.j2 src: calico-rr-docker.service.j2
dest: /etc/systemd/system/calico-rr.service dest: /etc/systemd/system/calico-rr.service
notify: restart calico-rr notify: restart calico-rr
when:
- container_manager in ['crio', 'docker', 'rkt']
- name: Calico-rr | Write calico-rr systemd init file
template:
src: calico-rr-containerd.service.j2
dest: /etc/systemd/system/calico-rr.service
notify: restart calico-rr
when:
- container_manager == 'containerd'
- name: Calico-rr | Configure route reflector - name: Calico-rr | Configure route reflector
command: |- command: |-

View file

@ -0,0 +1,27 @@
[Unit]
Description=calico-rr
After=containerd.service
Requires=containerd.service
[Service]
EnvironmentFile=/etc/calico/calico-rr.env
ExecStartPre=-{{ containerd_bin_dir }}/ctr t delete -f calico-rr
ExecStart={{ containerd_bin_dir }}/ctr run --net-host --privileged \
--env IP=${IP} \
--env IP6=${IP6} \
--env ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \
--env ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \
--env ETCD_CERT_FILE=${ETCD_CERT_FILE} \
--env ETCD_KEY_FILE=${ETCD_KEY_FILE} \
--mount type=bind,src=/var/log/calico-rr,dst=/var/log/calico,options=rbind:rw \
--mount type=bind,src={{ calico_cert_dir }},dst={{ calico_cert_dir }},options=rbind:ro \
{{ calico_rr_image_repo }}:{{ calico_rr_image_tag }} \
calico-rr
Restart=always
RestartSec=10s
ExecStop=-{{ containerd_bin_dir }}/ctr c rm calico-rr
[Install]
WantedBy=multi-user.target

View file

@ -151,14 +151,14 @@ spec:
mountPath: /host/opt/cni/bin mountPath: /host/opt/cni/bin
- name: etc-cni-netd - name: etc-cni-netd
mountPath: /host/etc/cni/net.d mountPath: /host/etc/cni/net.d
{% if container_manager == 'crio' %} {% if container_manager == 'docker' %}
- name: crio-socket
mountPath: /var/run/crio.sock
readOnly: true
{% else %}
- name: docker-socket - name: docker-socket
mountPath: /var/run/docker.sock mountPath: /var/run/docker.sock
readOnly: true readOnly: true
{% else %}
- name: "{{ container_manager }}-socket"
mountPath: {{ cri_socket }}
readOnly: true
{% endif %} {% endif %}
- name: etcd-config-path - name: etcd-config-path
mountPath: /var/lib/etcd-config mountPath: /var/lib/etcd-config
@ -182,16 +182,16 @@ spec:
- name: bpf-maps - name: bpf-maps
hostPath: hostPath:
path: /sys/fs/bpf path: /sys/fs/bpf
{% if container_manager == 'crio' %} {% if container_manager == 'docker' %}
# To read crio events from the node
- name: crio-socket
hostPath:
path: /var/run/crio/crio.sock
{% else %}
# To read docker events from the node # To read docker events from the node
- name: docker-socket - name: docker-socket
hostPath: hostPath:
path: /var/run/docker.sock path: /var/run/docker.sock
{% else %}
# To read crio events from the node
- name: {{ container_manager }}-socket
hostPath:
path: {{ cri_socket }}
{% endif %} {% endif %}
# To install cilium cni plugin in the host # To install cilium cni plugin in the host
- name: cni-path - name: cni-path

View file

@ -14,8 +14,6 @@
with_items: with_items:
- kubelet - kubelet
- vault - vault
- etcd
- etcd-events
failed_when: false failed_when: false
tags: tags:
- services - services
@ -26,8 +24,6 @@
state: absent state: absent
with_items: with_items:
- kubelet - kubelet
- etcd
- etcd-events
- vault - vault
- calico-node - calico-node
register: services_removed register: services_removed
@ -57,6 +53,7 @@
retries: 4 retries: 4
until: remove_all_containers.rc == 0 until: remove_all_containers.rc == 0
delay: 5 delay: 5
when: container_manager == "docker"
tags: tags:
- docker - docker
@ -64,50 +61,76 @@
service: service:
name: docker name: docker
state: restarted state: restarted
when: docker_dropins_removed.changed when: docker_dropins_removed.changed and container_manager == "docker"
tags: tags:
- docker - docker
- name: reset | stop all cri-o containers - name: reset | stop all cri containers
shell: "crictl ps -aq | xargs -r crictl stop" shell: "crictl ps -aq | xargs -r crictl -t 60s stop"
register: remove_all_crio_containers register: remove_all_cri_containers
retries: 4 retries: 5
until: remove_all_crio_containers.rc == 0 until: remove_all_cri_containers.rc == 0
delay: 5 delay: 5
tags: tags:
- crio - crio
when: container_manager == 'crio' - containerd
when: container_manager in ["crio", "containerd"]
- name: reset | remove all cri containers
- name: reset | remove all cri-o containers shell: "crictl ps -aq | xargs -r crictl -t 60s rm"
shell: "crictl ps -aq | xargs -r crictl rm" register: remove_all_cri_containers
register: remove_all_crio_containers retries: 5
retries: 4 until: remove_all_cri_containers.rc == 0
until: remove_all_crio_containers.rc == 0
delay: 5 delay: 5
tags: tags:
- crio - crio
when: container_manager == 'crio' and deploy_container_engine|default(true) - containerd
when: container_manager in ["crio", "containerd"] and deploy_container_engine|default(true)
- name: reset | stop all cri-o pods - name: reset | stop all cri pods
shell: "crictl pods -q | xargs -r crictl stopp" shell: "crictl pods -q | xargs -r crictl -t 60s stopp"
register: remove_all_crio_containers register: remove_all_cri_containers
retries: 4 retries: 5
until: remove_all_crio_containers.rc == 0 until: remove_all_cri_containers.rc == 0
delay: 5 delay: 5
tags: tags:
- crio - crio
when: container_manager == 'crio' - containerd
when: container_manager in ["crio", "containerd"]
- name: reset | remove all cri-o pods - name: reset | remove all cri pods
shell: "crictl pods -q | xargs -r crictl rmp" shell: "crictl pods -q | xargs -r crictl -t 60s rmp"
register: remove_all_crio_containers register: remove_all_cri_containers
retries: 4 retries: 5
until: remove_all_crio_containers.rc == 0 until: remove_all_cri_containers.rc == 0
delay: 5 delay: 5
tags: tags:
- crio - crio
when: container_manager == 'crio' - containerd
when: container_manager in ["crio", "containerd"]
- name: reset | stop etcd services
service:
name: "{{ item }}"
state: stopped
with_items:
- etcd
- etcd-events
failed_when: false
tags:
- services
- name: reset | remove etcd services
file:
path: "/etc/systemd/system/{{ item }}.service"
state: absent
with_items:
- etcd
- etcd-events
register: services_removed
tags:
- services
- name: reset | gather mounted kubelet dirs - name: reset | gather mounted kubelet dirs
shell: mount | grep /var/lib/kubelet/ | awk '{print $3}' | tac shell: mount | grep /var/lib/kubelet/ | awk '{print $3}' | tac
args: args: