diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index b2bfdf020..c07d943e3 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -145,7 +145,7 @@ skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipad dns_domain: "{{ cluster_name }}" ## Container runtime -## docker for docker and crio for cri-o. +## docker for docker, crio for cri-o and containerd for containerd. container_manager: docker ## Settings for containerized control plane (etcd/kubelet/secrets) diff --git a/roles/container-engine/containerd/defaults/main.yml b/roles/container-engine/containerd/defaults/main.yml new file mode 100644 index 000000000..0daeccd1b --- /dev/null +++ b/roles/container-engine/containerd/defaults/main.yml @@ -0,0 +1,12 @@ +--- +kubelet_cgroup_driver: systemd + +containerd_config: + grpc: + max_recv_message_size: 16777216 + max_send_message_size: 16777216 + debug: + level: "" + registries: + "docker.io": "https://registry-1.docker.io" + max_container_log_line_size: -1 diff --git a/roles/container-engine/containerd/handlers/main.yml b/roles/container-engine/containerd/handlers/main.yml new file mode 100644 index 000000000..f7aba44db --- /dev/null +++ b/roles/container-engine/containerd/handlers/main.yml @@ -0,0 +1,24 @@ +--- +- name: restart containerd + command: /bin/true + notify: + - Containerd | reload containerd + - Containerd | pause while containerd restarts + - Containerd | wait for containerd + +- name: Containerd | reload containerd + service: + name: containerd + state: restarted + +- name: Containerd | pause while containerd restarts + pause: + seconds: 5 + prompt: "Waiting for containerd restart" + +- name: Containerd | wait for containerd + command: "{{ containerd_bin_dir }}/ctr images ls -q" + register: containerd_ready + retries: 10 + delay: 5 + until: containerd_ready.rc == 0 diff --git a/roles/container-engine/containerd/tasks/crictl.yml b/roles/container-engine/containerd/tasks/crictl.yml new file mode 100644 index 000000000..02678f66d --- /dev/null +++ b/roles/container-engine/containerd/tasks/crictl.yml @@ -0,0 +1,26 @@ +--- +- name: crictl | Download crictl + include_tasks: "roles/download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.crictl) }}" + +- name: Install crictl config + template: + src: ../templates/crictl.yaml.j2 + dest: /etc/crictl.yaml + owner: bin + mode: 0644 + +- name: Copy crictl binary from download dir + synchronize: + src: "{{ local_release_dir }}/crictl" + dest: "{{ bin_dir }}/crictl" + compress: no + perms: yes + owner: no + group: no + delegate_to: "{{ inventory_hostname }}" + +- name: Install crictl completion + shell: /usr/local/bin/crictl completion >/etc/bash_completion.d/crictl + ignore_errors: True diff --git a/roles/container-engine/containerd/tasks/main.yml b/roles/container-engine/containerd/tasks/main.yml new file mode 100644 index 000000000..c552cbebf --- /dev/null +++ b/roles/container-engine/containerd/tasks/main.yml @@ -0,0 +1,50 @@ +--- +- name: Fail containerd setup if distribution is not supported + fail: + msg: "{{ ansible_distribution }} is not supported by containerd." + when: + - not ansible_distribution in ["CentOS","RedHat", "Ubuntu", "Debian"] + +- name: Install Docker + include_role: + name: container-engine/docker + +- name: Install config.toml + template: + src: config.toml.j2 + dest: /etc/containerd/config.toml + owner: bin + mode: 0644 + +- name: Stop and disabled Docker + systemd: + name: docker + state: stopped + enabled: no + +- name: Restart containerd + systemd: + name: containerd + state: restarted + +- name: Install crictl config + template: + src: crictl.yaml.j2 + dest: /etc/crictl.yaml + owner: bin + mode: 0644 + +- name: Install crictl completion + shell: /usr/local/bin/crictl completion >/etc/bash_completion.d/crictl + ignore_errors: True + when: ansible_distribution in ["CentOS","RedHat", "Ubuntu", "Debian"] + +- name: Enable containerd + systemd: + name: containerd.service + state: started + enabled: yes + daemon-reload: yes + +- name: flush handlers so we can wait for containerd to come up + meta: flush_handlers diff --git a/roles/container-engine/containerd/templates/config.toml.j2 b/roles/container-engine/containerd/templates/config.toml.j2 new file mode 100644 index 000000000..6c3dccb0a --- /dev/null +++ b/roles/container-engine/containerd/templates/config.toml.j2 @@ -0,0 +1,40 @@ +# Kubernetes doesn't use containerd restart manager. +disabled_plugins = ["restart"] + +[debug] + level = "{{ containerd_config.debug.level | default("") }}" + +{% if 'grpc' in containerd_config %} +[grpc] +{% for param, value in containerd_config.grpc.items() %} + {{ param }} = {{ value }} +{% endfor %} +{% endif %} + +[plugins.linux] + shim = "/usr/bin/containerd-shim" + runtime = "/usr/sbin/runc" + +[plugins.cri] + stream_server_address = "127.0.0.1" + max_container_log_line_size = {{ containerd_config.max_container_log_line_size }} + sandbox_image = "{{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}" + +[plugins.cri.cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "" + +[plugins.cri.containerd.untrusted_workload_runtime] + runtime_type = "" + runtime_engine = "" + runtime_root = "" + +{% if 'registries' in containerd_config %} +[plugins.cri.registry] +[plugins.cri.registry.mirrors] +{% for registry, addr in containerd_config.registries.items() %} +[plugins.cri.registry.mirrors."{{ registry }}"] + endpoint = ["{{ addr }}"] +{% endfor %} +{% endif %} diff --git a/roles/container-engine/containerd/templates/crictl.yaml.j2 b/roles/container-engine/containerd/templates/crictl.yaml.j2 new file mode 100644 index 000000000..fbf691f8a --- /dev/null +++ b/roles/container-engine/containerd/templates/crictl.yaml.j2 @@ -0,0 +1,4 @@ +runtime-endpoint: unix://{{ cri_socket }} +image-endpoint: unix://{{ cri_socket }} +timeout: 30 +debug: false diff --git a/roles/container-engine/cri-o/tasks/main.yaml b/roles/container-engine/cri-o/tasks/main.yaml index a85392993..82767a8b9 100644 --- a/roles/container-engine/cri-o/tasks/main.yaml +++ b/roles/container-engine/cri-o/tasks/main.yaml @@ -24,6 +24,12 @@ gpgcheck: no when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic +- name: Add CRI-O PPA + apt_repository: + repo: ppa:projectatomic/ppa + state: present + when: ansible_distribution in ["Ubuntu"] + - name: Make sure needed folders exist in the system with_items: - /etc/crio diff --git a/roles/container-engine/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2 index 45d02696e..199e34895 100644 --- a/roles/container-engine/cri-o/templates/crio.conf.j2 +++ b/roles/container-engine/cri-o/templates/crio.conf.j2 @@ -64,7 +64,7 @@ file_locking = true # This is a mandatory setting as this runtime will be the default one # and will also be used for untrusted container workloads if # runtime_untrusted_workload is not set. -{% if ansible_os_family == "ClearLinux" or ansible_os_family == "RedHat" %} +{% if ansible_os_family == "ClearLinux" or ansible_os_family == "RedHat" or ansible_distribution == "Ubuntu" %} runtime = "/usr/bin/runc" {% else %} runtime = "/usr/sbin/runc" @@ -96,7 +96,7 @@ default_workload_trust = "trusted" no_pivot = false # conmon is the path to conmon binary, used for managing the runtime. -conmon = "/usr/libexec/crio/conmon" +conmon = "{{ crio_conmon }}" # conmon_env is the environment variable list for conmon process, # used for passing necessary environment variable to conmon or runtime. diff --git a/roles/container-engine/cri-o/vars/clearlinux.yml b/roles/container-engine/cri-o/vars/clearlinux.yml index 4afc55079..bcaed568c 100644 --- a/roles/container-engine/cri-o/vars/clearlinux.yml +++ b/roles/container-engine/cri-o/vars/clearlinux.yml @@ -3,3 +3,4 @@ crio_packages: - containers-basic crio_service: crio +crio_conmon: /usr/libexec/crio/conmon diff --git a/roles/container-engine/cri-o/vars/fedora.yml b/roles/container-engine/cri-o/vars/fedora.yml index 32a1b9079..00dd69ed4 100644 --- a/roles/container-engine/cri-o/vars/fedora.yml +++ b/roles/container-engine/cri-o/vars/fedora.yml @@ -4,3 +4,4 @@ crio_packages: - cri-tools crio_service: cri-o +crio_conmon: /usr/libexec/crio/conmon diff --git a/roles/container-engine/cri-o/vars/redhat.yml b/roles/container-engine/cri-o/vars/redhat.yml index 962dc9a0a..51416807b 100644 --- a/roles/container-engine/cri-o/vars/redhat.yml +++ b/roles/container-engine/cri-o/vars/redhat.yml @@ -4,4 +4,5 @@ crio_packages: - cri-tools - oci-systemd-hook -crio_service: crio \ No newline at end of file +crio_service: crio +crio_conmon: /usr/libexec/crio/conmon diff --git a/roles/container-engine/cri-o/vars/ubuntu.yml b/roles/container-engine/cri-o/vars/ubuntu.yml new file mode 100644 index 000000000..c02c638e1 --- /dev/null +++ b/roles/container-engine/cri-o/vars/ubuntu.yml @@ -0,0 +1,6 @@ +--- +crio_packages: + - "cri-o-{{ kube_version | regex_replace('^v(?P\\d+).(?P\\d+).(?P\\d+)$', '\\g.\\g') }}" + +crio_service: crio +crio_conmon: /usr/lib/crio/bin/conmon diff --git a/roles/container-engine/meta/main.yml b/roles/container-engine/meta/main.yml index 661d6c103..e58eb9692 100644 --- a/roles/container-engine/meta/main.yml +++ b/roles/container-engine/meta/main.yml @@ -7,9 +7,23 @@ dependencies: - container-engine - crio + - role: container-engine/containerd + when: + - container_manager == 'containerd' + tags: + - container-engine + - containerd + - role: container-engine/docker when: - container_manager == 'docker' tags: - container-engine - docker + + - role: container-engine/containerd + when: + - container_manager == 'containerd' + tags: + - container-engine + - containerd diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 95ab0dc69..939a3bde6 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -74,7 +74,9 @@ pod_infra_version: 3.1 contiv_version: 1.2.1 cilium_version: "v1.3.0" kube_router_version: "v0.2.5" -multus_version: "v3.2.1" +multus_version: "v3.1.autoconf" + +crictl_version: "v1.14.0" # Download URLs kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm" @@ -82,6 +84,18 @@ hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/relea etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz" cni_download_url: "https://github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz" calicoctl_download_url: "https://github.com/projectcalico/calicoctl/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}" +crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" + +crictl_checksums: + arm: + v1.14.0: 9910cecfd6558239ba015323066c7233d8371af359b9ddd0b2a35d5223bcf945 + v1.13.0: 2e478ebed85f9d70d49fd8f1d1089c8fba6e37d3461aeef91813f1ab0f0df586 + arm64: + v1.14.0: f76b3d00a272c8d210e9a45f77d07d3770bee310d99c4fd9a72d6f55278882e5 + v1.13.0: 68949c0cb5a37e7604c145d189cf1e109c08c93d9c710ba663db026b9c6f2746 + amd64: + v1.14.0: 483c90a9fe679590df4332ba807991c49232e8cd326c307c575ecef7fe22327b + v1.13.0: 9bdbea7a2b382494aff2ff014da328a042c5aba9096a7772e57fdf487e5a1d51 # Checksums hyperkube_checksums: @@ -164,6 +178,9 @@ kubeadm_checksums: v1.13.2: 7cb0ce57c1e6e2d85e05de3780a2f35a191fe93f89cfc5816b424efcf39834b9 v1.13.1: 438173bfa0b7014ecae994c5b9e1f27e1328ab971a3fdb06a393a8095a176ba0 v1.13.0: f5366206416dc4cfc840a7add2289957b56ccc479cc1b74f7397a4df995d6b06 +crictl_binary_checksums: + amd64: + v1.14.0: 483c90a9fe679590df4332ba807991c49232e8cd326c307c575ecef7fe22327b etcd_binary_checksums: # Etcd does not have arm32 builds at the moment, having some dummy value is @@ -194,6 +211,7 @@ cni_binary_checksum: "{{ cni_binary_checksums[image_arch] }}" hyperkube_binary_checksum: "{{ hyperkube_checksums[image_arch][kube_version] }}" kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}" calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}" +crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}" # Containers # In some cases, we need a way to set --registry-mirror or --insecure-registry for docker, @@ -304,6 +322,9 @@ addon_resizer_image_tag: "{{ addon_resizer_version }}" dashboard_image_repo: "gcr.io/google_containers/kubernetes-dashboard-{{ image_arch }}" dashboard_image_tag: "v1.10.1" +image_pull_command: "{{ docker_bin_dir }}/docker pull" +image_info_command: "{{ docker_bin_dir }}/docker images -q | xargs {{ docker_bin_dir }}/docker inspect -f \"{{ '{{' }} if .RepoTags {{ '}}' }}{{ '{{' }} (index .RepoTags 0) {{ '}}' }}{{ '{{' }} end {{ '}}' }}{{ '{{' }} if .RepoDigests {{ '}}' }},{{ '{{' }} (index .RepoDigests 0) {{ '}}' }}{{ '{{' }} end {{ '}}' }}\" | tr '\n' ','" + downloads: netcheck_server: enabled: "{{ deploy_netchecker }}" @@ -378,6 +399,19 @@ downloads: groups: - k8s-cluster + crictl: + file: true + enabled: "{{ container_manager in ['crio', 'cri', 'containerd'] }}" + version: "{{ crictl_version }}" + dest: "{{local_release_dir}}/crictl-{{ crictl_version }}-linux-{{ image_arch }}.tar.gz" + sha256: "{{ crictl_binary_checksum }}" + url: "{{ crictl_download_url }}" + unarchive: true + owner: "root" + mode: "0755" + groups: + - k8s-cluster + cilium: enabled: "{{ kube_network_plugin == 'cilium' }}" container: true diff --git a/roles/download/tasks/download_container.yml b/roles/download/tasks/download_container.yml index 6cd763ebe..8fe3aee22 100644 --- a/roles/download/tasks/download_container.yml +++ b/roles/download/tasks/download_container.yml @@ -1,129 +1,137 @@ --- +- name: container_download | Make download decision if pull is required by tag or sha256 + include_tasks: set_docker_image_facts.yml + when: + - download.enabled + - download.container + tags: + - facts + - block: - - name: download_container | Set a few facts - import_tasks: set_container_facts.yml - run_once: "{{ download_run_once }}" - tags: - - facts + - name: download_container | Set a few facts + import_tasks: set_container_facts.yml + run_once: "{{ download_run_once }}" + tags: + - facts - - name: download_container | Determine if image is in cache - stat: - path: "{{ image_path_cached }}" - delegate_to: localhost - delegate_facts: no - register: cache_image - changed_when: false - become: false - when: - - download_force_cache + - name: download_container | Determine if image is in cache + stat: + path: "{{ image_path_cached }}" + delegate_to: localhost + delegate_facts: no + register: cache_image + changed_when: false + become: false + when: + - download_force_cache - - name: download_container | Set fact indicating if image is in cache - set_fact: - image_is_cached: "{{ cache_image.stat.exists | default(false) }}" - tags: - - facts - when: - - download_force_cache + - name: download_container | Set fact indicating if image is in cache + set_fact: + image_is_cached: "{{ cache_image.stat.exists | default(false) }}" + tags: + - facts + when: + - download_force_cache - - name: download_container | Upload image to node if it is cached - synchronize: - src: "{{ image_path_cached }}" - dest: "{{ image_path_final }}" - use_ssh_args: "{{ has_bastion | default(false) }}" - mode: push - delegate_facts: no - register: upload_image - failed_when: not upload_image - run_once: "{{ download_run_once }}" - until: upload_image is succeeded - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - when: - - download_force_cache - - image_is_cached - - not download_localhost - - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] + - name: download_container | Upload image to node if it is cached + synchronize: + src: "{{ image_path_cached }}" + dest: "{{ image_path_final }}" + use_ssh_args: "{{ has_bastion | default(false) }}" + mode: push + delegate_facts: no + register: upload_image + failed_when: not upload_image + run_once: "{{ download_run_once }}" + until: upload_image is succeeded + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + when: + - download_force_cache + - image_is_cached + - not download_localhost + - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] - - name: download_container | Load image into docker - shell: "{{ docker_bin_dir }}/docker load < {{ image_path_cached if download_localhost else image_path_final }}" - delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}" - run_once: "{{ download_run_once }}" - register: container_load_status - failed_when: container_load_status | failed - become: "{{ user_can_become_root | default(false) or not (download_run_once and download_localhost) }}" - when: - - download_force_cache - - image_is_cached - - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] + - name: download_container | Load image into docker + shell: "{{ docker_bin_dir }}/docker load < {{ image_path_cached if download_localhost else image_path_final }}" + delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}" + run_once: "{{ download_run_once }}" + register: container_load_status + failed_when: container_load_status | failed + become: "{{ user_can_become_root | default(false) or not (download_run_once and download_localhost) }}" + when: + - download_force_cache + - image_is_cached + - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] - - name: download_container | Prepare container download - import_tasks: check_pull_required.yml - run_once: "{{ download_run_once }}" - when: - - not download_always_pull + - name: download_container | Prepare container download + import_tasks: check_pull_required.yml + run_once: "{{ download_run_once }}" + when: + - not download_always_pull - - debug: - msg: "XXX Pull required is: {{ pull_required }}" + - debug: + msg: "XXX Pull required is: {{ pull_required }}" - # NOTE: Pre-loading docker images will not prevent 'docker pull' from re-downloading the layers in that image - # if a pull is forced. This is a known issue with docker. See https://github.com/moby/moby/issues/23684 - - name: download_container | Download image if required - command: "{{ docker_bin_dir }}/docker pull {{ image_reponame }}" - delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}" - delegate_facts: yes - run_once: "{{ download_run_once }}" - register: pull_task_result - until: pull_task_result is succeeded - delay: "{{ retry_stagger | random + 3 }}" - retries: 4 - become: "{{ user_can_become_root | default(false) or not download_localhost }}" - when: - - pull_required | default(download_always_pull) + # NOTE: Pre-loading docker images will not prevent 'docker pull' from re-downloading the layers in that image + # if a pull is forced. This is a known issue with docker. See https://github.com/moby/moby/issues/23684 + - name: download_container | Download image if required + command: "{{ image_pull_command }} {{ image_reponame }}" + delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}" + delegate_facts: yes + run_once: "{{ download_run_once }}" + register: pull_task_result + until: pull_task_result is succeeded + delay: "{{ retry_stagger | random + 3 }}" + retries: 4 + become: "{{ user_can_become_root | default(false) or not download_localhost }}" + when: + - pull_required | default(download_always_pull) - # NOTE: image_changed is only valid if a pull is was needed or forced. - - name: download_container | Check if image changed - set_fact: - image_changed: "{{ true if pull_task_result.stdout is defined and not 'up to date' in pull_task_result.stdout else false }}" - run_once: true - when: - - download_force_cache - tags: - - facts + # NOTE: image_changed is only valid if a pull is was needed or forced. + - name: download_container | Check if image changed + set_fact: + image_changed: "{{ true if pull_task_result.stdout is defined and not 'up to date' in pull_task_result.stdout else false }}" + run_once: true + when: + - download_force_cache + tags: + - facts - - name: download_container | Save and compress image - shell: "{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_cached if download_localhost else image_path_final }}" - delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}" - delegate_facts: no - register: container_save_status - failed_when: container_save_status.stderr - run_once: true - become: "{{ user_can_become_root | default(false) or not download_localhost }}" - when: - - download_force_cache - - not image_is_cached or (image_changed | default(true)) - - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] + - name: download_container | Save and compress image + shell: "{{ docker_bin_dir }}/docker save {{ image_reponame }} | gzip -{{ download_compress }} > {{ image_path_cached if download_localhost else image_path_final }}" + delegate_to: "{{ download_delegate if download_run_once or inventory_hostname }}" + delegate_facts: no + register: container_save_status + failed_when: container_save_status.stderr + run_once: true + become: "{{ user_can_become_root | default(false) or not download_localhost }}" + when: + - download_force_cache + - not image_is_cached or (image_changed | default(true)) + - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] - - name: download_container | Copy image to ansible host cache - synchronize: - src: "{{ image_path_final }}" - dest: "{{ image_path_cached }}" - use_ssh_args: "{{ has_bastion | default(false) }}" - mode: pull - delegate_facts: no - run_once: true - when: - - download_force_cache - - not download_localhost - - not image_is_cached or (image_changed | default(true)) - - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] + - name: download_container | Copy image to ansible host cache + synchronize: + src: "{{ image_path_final }}" + dest: "{{ image_path_cached }}" + use_ssh_args: "{{ has_bastion | default(false) }}" + mode: pull + delegate_facts: no + run_once: true + when: + - download_force_cache + - not download_localhost + - not image_is_cached or (image_changed | default(true)) + - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] - - name: download_container | Remove container image from cache - file: - state: absent - path: "{{ image_path_final }}" - when: - - not download_keep_remote_cache - - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] + - name: download_container | Remove container image from cache + file: + state: absent + path: "{{ image_path_final }}" + when: + - not download_keep_remote_cache + - ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] tags: - - download + - download diff --git a/roles/download/tasks/download_prep.yml b/roles/download/tasks/download_prep.yml new file mode 100644 index 000000000..ace884c25 --- /dev/null +++ b/roles/download/tasks/download_prep.yml @@ -0,0 +1,42 @@ +--- +# Use the same format for Containerd images as for Docker images +# ctr doesn't have inspect command + +- name: Set image info command for containerd + set_fact: + image_info_command: "{{ containerd_bin_dir }}/ctr images ls | tail -n +2 | awk -F '[ :]+' '{print $1\":\"$2\",\"$1\":\"$4\"@\"$5}' | tr '\n' ','" + when: container_manager == 'containerd' + +- name: Register docker images info + shell: "{{ image_info_command }}" + no_log: true + register: docker_images + failed_when: false + changed_when: false + check_mode: no + when: download_container + +- name: container_download | Create dest directory for saved/loaded container images + file: + path: "{{ local_release_dir }}/containers" + state: directory + recurse: yes + mode: 0755 + owner: "{{ ansible_ssh_user|default(ansible_user_id) }}" + when: download_container + +- name: container_download | create local directory for saved/loaded container images + file: + path: "{{ local_release_dir }}/containers" + state: directory + recurse: yes + delegate_to: localhost + delegate_facts: false + become: false + run_once: true + when: + - download_run_once + - download_delegate == 'localhost' + - download_container + tags: + - localhost diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index aa2498043..ca702a13a 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -7,6 +7,25 @@ - download - upload +- name: Use cri-o for cri connection + set_fact: + cri_socket: /var/run/crio/crio.sock + when: container_manager == 'crio' + +- name: Use containerd for cri connetion + set_fact: + cri_socket: /var/run/containerd/containerd.sock + when: container_manager == 'containerd' + +- name: Use docker for cri connetion + set_fact: + cri_socket: /var/run/dockershim.sock + when: container_manager == 'docker' + +- include_tasks: ../../container-engine/containerd/tasks/crictl.yml + when: + - container_manager in ['containerd', 'crio'] + - name: download | Get kubeadm binary and list of required images import_tasks: prep_kubeadm_images.yml when: diff --git a/roles/download/tasks/set_docker_image_facts.yml b/roles/download/tasks/set_docker_image_facts.yml new file mode 100644 index 000000000..1e9ca433b --- /dev/null +++ b/roles/download/tasks/set_docker_image_facts.yml @@ -0,0 +1,54 @@ +--- +- name: Set if containers should be pulled by digest + set_fact: + pull_by_digest: >- + {%- if download.sha256 is defined and download.sha256 -%}true{%- else -%}false{%- endif -%} + +- name: Set pull_args + set_fact: + pull_args: >- + {%- if pull_by_digest %}{{ download.repo }}@sha256:{{ download.sha256 }}{%- else -%}{{ download.repo }}:{{ download.tag }}{%- endif -%} + +- name: Set image pull command for containerd + set_fact: + image_pull_command: "{{ bin_dir }}/crictl pull" + when: container_manager in ['crio' ,'containerd'] + +- name: Register docker images info + shell: "{{ image_info_command }}" + no_log: true + register: docker_images + failed_when: false + changed_when: false + check_mode: no + when: + - not download_always_pull + - group_names | intersect(download.groups) | length + +- name: Set if pull is required per container + set_fact: + pull_required: >- + {%- if pull_args in docker_images.stdout.split(',') %}false{%- else -%}true{%- endif -%} + when: + - not download_always_pull + - group_names | intersect(download.groups) | length + +- name: Does any host require container pull? + vars: + hosts_pull_required: "{{ hostvars.values() | map(attribute='pull_required') | select('defined') | list }}" + set_fact: + any_pull_required: "{{ True in hosts_pull_required }}" + run_once: true + changed_when: false + when: not download_always_pull + +- name: Check the local digest sha256 corresponds to the given image tag + assert: + that: "{{ download.repo }}:{{ download.tag }} in docker_images.stdout.split(',')" + when: + - group_names | intersect(download.groups) | length + - not download_always_pull + - not pull_required + - pull_by_digest + tags: + - asserts diff --git a/roles/download/templates/kubeadm-images.yaml.j2 b/roles/download/templates/kubeadm-images.yaml.j2 index 8a91577bd..eb80e15db 100644 --- a/roles/download/templates/kubeadm-images.yaml.j2 +++ b/roles/download/templates/kubeadm-images.yaml.j2 @@ -6,11 +6,7 @@ apiVersion: kubeadm.k8s.io/v1beta1 {% endif %} kind: InitConfiguration nodeRegistration: -{% if container_manager == 'crio' %} - criSocket: /var/run/crio/crio.sock -{% else %} - criSocket: /var/run/dockershim.sock -{% endif %} + criSocket: {{ cri_socket }} --- {% endif %} {% if kube_version is version('v1.11.0', '<') %} @@ -37,9 +33,5 @@ etcd: {% endfor %} {% if kube_version is version('v1.12.0', '<') %} nodeRegistration: -{% if container_manager == 'crio' %} - criSocket: /var/run/crio/crio.sock -{% else %} - criSocket: /var/run/dockershim.sock -{% endif %} + criSocket: {{ cri_socket }} {% endif %} diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 index 38d4733c7..243383667 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 @@ -16,8 +16,4 @@ discoveryTokenAPIServers: discoveryTokenUnsafeSkipCAVerification: true nodeRegistration: name: {{ kube_override_hostname }} -{% if container_manager == 'crio' %} - criSocket: /var/run/crio/crio.sock -{% else %} - criSocket: /var/run/dockershim.sock -{% endif %} + criSocket: {{ cri_socket }} diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 index 46e365a83..81efb98fc 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 @@ -16,8 +16,4 @@ discoveryTokenAPIServers: discoveryTokenUnsafeSkipCAVerification: true nodeRegistration: name: {{ kube_override_hostname }} -{% if container_manager == 'crio' %} - criSocket: /var/run/crio/crio.sock -{% else %} - criSocket: /var/run/dockershim.sock -{% endif %} + criSocket: {{ cri_socket }} diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 index d03c9c7af..93ea517d3 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 @@ -16,8 +16,4 @@ discovery: caCertPath: {{ kube_cert_dir }}/ca.crt nodeRegistration: name: {{ kube_override_hostname }} -{% if container_manager == 'crio' %} - criSocket: /var/run/crio/crio.sock -{% else %} - criSocket: /var/run/dockershim.sock -{% endif %} + criSocket: {{ cri_socket }} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index 68bf2fd5d..c46f75c20 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -228,11 +228,7 @@ nodeRegistration: {% else %} taints: {} {% endif %} -{% if container_manager == 'crio' %} - criSocket: /var/run/crio/crio.sock -{% else %} - criSocket: /var/run/dockershim.sock -{% endif %} + criSocket: {{ cri_socket }} {% if dynamic_kubelet_configuration %} featureGates: DynamicKubeletConfig: true diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 index 4658537f6..b83ffea0d 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 @@ -14,11 +14,7 @@ nodeRegistration: {% else %} taints: {} {% endif %} -{% if container_manager == 'crio' %} - criSocket: /var/run/crio/crio.sock -{% else %} - criSocket: /var/run/dockershim.sock -{% endif %} + criSocket: {{ cri_socket }} --- apiVersion: kubeadm.k8s.io/v1alpha3 kind: ClusterConfiguration diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 index 87c4f0b4f..b5ee0dd0d 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 @@ -14,11 +14,7 @@ nodeRegistration: {% else %} taints: [] {% endif %} -{% if container_manager == 'crio' %} - criSocket: /var/run/crio/crio.sock -{% else %} - criSocket: /var/run/dockershim.sock -{% endif %} + criSocket: {{ cri_socket }} --- apiVersion: kubeadm.k8s.io/v1beta1 kind: ClusterConfiguration diff --git a/roles/kubernetes/master/templates/kubeadm-controlplane.v1beta1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-controlplane.v1beta1.yaml.j2 index 31c054c8f..9d7952759 100644 --- a/roles/kubernetes/master/templates/kubeadm-controlplane.v1beta1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-controlplane.v1beta1.yaml.j2 @@ -17,8 +17,4 @@ controlPlane: bindPort: {{ kube_apiserver_port }} nodeRegistration: name: {{ kube_override_hostname|default(inventory_hostname) }} -{% if container_manager == 'crio' %} - criSocket: /var/run/crio/crio.sock -{% else %} - criSocket: /var/run/dockershim.sock -{% endif %} + criSocket: {{ cri_socket }} diff --git a/roles/kubernetes/node/tasks/facts.yml b/roles/kubernetes/node/tasks/facts.yml index 6f8539c0e..6558a8319 100644 --- a/roles/kubernetes/node/tasks/facts.yml +++ b/roles/kubernetes/node/tasks/facts.yml @@ -3,12 +3,23 @@ shell: "docker info | grep 'Cgroup Driver' | awk -F': ' '{ print $2; }'" register: docker_cgroup_driver_result changed_when: false + when: container_manager in ['crio', 'docker', 'rkt'] -- name: set facts +- name: set standalone_kubelet fact set_fact: standalone_kubelet: >- {%- if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] -%}true{%- else -%}false{%- endif -%} + +- name: set kubelet_cgroup_driver_detected fact for containerd + set_fact: + kubelet_cgroup_driver_detected: >- + {%- if containerd_use_systemd_cgroup -%}systemd{%- else -%}cgroupfs{%- endif -%} + when: container_manager == 'containerd' + +- name: set kubelet_cgroup_driver_detected fact for other engines + set_fact: kubelet_cgroup_driver_detected: "{{ docker_cgroup_driver_result.stdout }}" + when: container_manager in ['crio', 'docker', 'rkt'] - name: os specific vars include_vars: "{{ item }}" diff --git a/roles/kubernetes/node/tasks/pre_upgrade.yml b/roles/kubernetes/node/tasks/pre_upgrade.yml index 2191d6fbd..846e5e46b 100644 --- a/roles/kubernetes/node/tasks/pre_upgrade.yml +++ b/roles/kubernetes/node/tasks/pre_upgrade.yml @@ -1,12 +1,22 @@ --- - name: "Pre-upgrade | check if kubelet container exists" - shell: docker ps -af name=kubelet | grep kubelet + shell: >- + {% if container_manager in ['crio', 'docker', 'rkt'] %} + docker ps -af name=kubelet | grep kubelet + {% elif container_manager == 'containerd' %} + crictl ps --all --name kubelet | grep kubelet + {% endif %} failed_when: false changed_when: false register: kubelet_container_check - name: "Pre-upgrade | copy /var/lib/cni from kubelet" - command: docker cp kubelet:/var/lib/cni /var/lib/cni + command: >- + {% if container_manager in ['crio', 'docker', 'rkt'] %} + docker cp kubelet:/var/lib/cni /var/lib/cni + {% elif container_manager == 'containerd' %} + ctr run --rm --mount type=bind,src=/var/lib/cni,dst=/cnilibdir,options=rbind:rw kubelet kubelet-tmp sh -c 'cp /var/lib/cni/* /cnilibdir/' + {% endif %} args: creates: "/var/lib/cni" failed_when: false @@ -19,7 +29,12 @@ when: kubelet_container_check.rc == 0 - name: "Pre-upgrade | ensure kubelet container is removed if using host deployment" - command: docker rm -fv kubelet + shell: >- + {% if container_manager in ['crio', 'docker', 'rkt'] %} + docker rm -fv kubelet + {% elif container_manager == 'containerd' %} + crictl stop kubelet && crictl rm kubelet + {% endif %} failed_when: false changed_when: false register: remove_kubelet_container diff --git a/roles/kubernetes/node/templates/kubelet.env.j2 b/roles/kubernetes/node/templates/kubelet.env.j2 index a02949786..eb335e45a 100644 --- a/roles/kubernetes/node/templates/kubelet.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.env.j2 @@ -43,9 +43,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %} --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ {% endif %} -{% if container_manager == 'crio' %} +{% if container_manager != 'docker' %} --container-runtime=remote \ ---container-runtime-endpoint=/var/run/crio/crio.sock \ +--container-runtime-endpoint={{ cri_socket }} \ {% endif %} --anonymous-auth=false \ --read-only-port={{ kube_read_only_port }} \ diff --git a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 index 703c0dca0..b186065a7 100644 --- a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 +++ b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 @@ -22,9 +22,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %} --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ {% endif %} -{% if container_manager == 'crio' %} +{% if container_manager != 'docker' %} --container-runtime=remote \ ---container-runtime-endpoint=/var/run/crio/crio.sock \ +--container-runtime-endpoint={{ cri_socket }} \ {% endif %} {% if kube_version is version('v1.8', '<') %} --experimental-fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \ diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index d9848b64f..6865703bb 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -46,6 +46,7 @@ haproxy_config_dir: "/etc/haproxy" # Directory where the binaries will be installed bin_dir: /usr/local/bin docker_bin_dir: /usr/bin +containerd_bin_dir: /usr/bin etcd_data_dir: /var/lib/etcd # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) @@ -258,6 +259,9 @@ docker_options: >- # Experimental kubeadm etcd deployment mode. Available only for new deployment etcd_kubeadm_enabled: false +# Containerd options +containerd_use_systemd_cgroup: false + # Settings for containerized control plane (etcd/kubelet/secrets) # deployment type for legacy etcd mode etcd_deployment_type: docker diff --git a/roles/network_plugin/calico/rr/tasks/main.yml b/roles/network_plugin/calico/rr/tasks/main.yml index 41e8c85da..2d9ba6ba0 100644 --- a/roles/network_plugin/calico/rr/tasks/main.yml +++ b/roles/network_plugin/calico/rr/tasks/main.yml @@ -42,9 +42,19 @@ - name: Calico-rr | Write calico-rr systemd init file template: - src: calico-rr.service.j2 + src: calico-rr-docker.service.j2 dest: /etc/systemd/system/calico-rr.service notify: restart calico-rr + when: + - container_manager in ['crio', 'docker', 'rkt'] + +- name: Calico-rr | Write calico-rr systemd init file + template: + src: calico-rr-containerd.service.j2 + dest: /etc/systemd/system/calico-rr.service + notify: restart calico-rr + when: + - container_manager == 'containerd' - name: Calico-rr | Configure route reflector command: |- diff --git a/roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2 b/roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2 new file mode 100644 index 000000000..db719afd3 --- /dev/null +++ b/roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2 @@ -0,0 +1,27 @@ +[Unit] +Description=calico-rr +After=containerd.service +Requires=containerd.service + +[Service] +EnvironmentFile=/etc/calico/calico-rr.env +ExecStartPre=-{{ containerd_bin_dir }}/ctr t delete -f calico-rr +ExecStart={{ containerd_bin_dir }}/ctr run --net-host --privileged \ + --env IP=${IP} \ + --env IP6=${IP6} \ + --env ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \ + --env ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \ + --env ETCD_CERT_FILE=${ETCD_CERT_FILE} \ + --env ETCD_KEY_FILE=${ETCD_KEY_FILE} \ + --mount type=bind,src=/var/log/calico-rr,dst=/var/log/calico,options=rbind:rw \ + --mount type=bind,src={{ calico_cert_dir }},dst={{ calico_cert_dir }},options=rbind:ro \ + {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }} \ + calico-rr + +Restart=always +RestartSec=10s + +ExecStop=-{{ containerd_bin_dir }}/ctr c rm calico-rr + +[Install] +WantedBy=multi-user.target diff --git a/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 b/roles/network_plugin/calico/rr/templates/calico-rr-docker.service.j2 similarity index 100% rename from roles/network_plugin/calico/rr/templates/calico-rr.service.j2 rename to roles/network_plugin/calico/rr/templates/calico-rr-docker.service.j2 diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 index e0b54b982..55266c923 100755 --- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 @@ -151,14 +151,14 @@ spec: mountPath: /host/opt/cni/bin - name: etc-cni-netd mountPath: /host/etc/cni/net.d -{% if container_manager == 'crio' %} - - name: crio-socket - mountPath: /var/run/crio.sock - readOnly: true -{% else %} +{% if container_manager == 'docker' %} - name: docker-socket mountPath: /var/run/docker.sock readOnly: true +{% else %} + - name: "{{ container_manager }}-socket" + mountPath: {{ cri_socket }} + readOnly: true {% endif %} - name: etcd-config-path mountPath: /var/lib/etcd-config @@ -182,16 +182,16 @@ spec: - name: bpf-maps hostPath: path: /sys/fs/bpf -{% if container_manager == 'crio' %} - # To read crio events from the node - - name: crio-socket - hostPath: - path: /var/run/crio/crio.sock -{% else %} +{% if container_manager == 'docker' %} # To read docker events from the node - name: docker-socket hostPath: path: /var/run/docker.sock +{% else %} + # To read crio events from the node + - name: {{ container_manager }}-socket + hostPath: + path: {{ cri_socket }} {% endif %} # To install cilium cni plugin in the host - name: cni-path diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index 08b9c71a8..97812b3aa 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -14,8 +14,6 @@ with_items: - kubelet - vault - - etcd - - etcd-events failed_when: false tags: - services @@ -26,8 +24,6 @@ state: absent with_items: - kubelet - - etcd - - etcd-events - vault - calico-node register: services_removed @@ -57,6 +53,7 @@ retries: 4 until: remove_all_containers.rc == 0 delay: 5 + when: container_manager == "docker" tags: - docker @@ -64,50 +61,76 @@ service: name: docker state: restarted - when: docker_dropins_removed.changed + when: docker_dropins_removed.changed and container_manager == "docker" tags: - docker -- name: reset | stop all cri-o containers - shell: "crictl ps -aq | xargs -r crictl stop" - register: remove_all_crio_containers - retries: 4 - until: remove_all_crio_containers.rc == 0 +- name: reset | stop all cri containers + shell: "crictl ps -aq | xargs -r crictl -t 60s stop" + register: remove_all_cri_containers + retries: 5 + until: remove_all_cri_containers.rc == 0 delay: 5 tags: - crio - when: container_manager == 'crio' + - containerd + when: container_manager in ["crio", "containerd"] - -- name: reset | remove all cri-o containers - shell: "crictl ps -aq | xargs -r crictl rm" - register: remove_all_crio_containers - retries: 4 - until: remove_all_crio_containers.rc == 0 +- name: reset | remove all cri containers + shell: "crictl ps -aq | xargs -r crictl -t 60s rm" + register: remove_all_cri_containers + retries: 5 + until: remove_all_cri_containers.rc == 0 delay: 5 tags: - crio - when: container_manager == 'crio' and deploy_container_engine|default(true) + - containerd + when: container_manager in ["crio", "containerd"] and deploy_container_engine|default(true) -- name: reset | stop all cri-o pods - shell: "crictl pods -q | xargs -r crictl stopp" - register: remove_all_crio_containers - retries: 4 - until: remove_all_crio_containers.rc == 0 +- name: reset | stop all cri pods + shell: "crictl pods -q | xargs -r crictl -t 60s stopp" + register: remove_all_cri_containers + retries: 5 + until: remove_all_cri_containers.rc == 0 delay: 5 tags: - crio - when: container_manager == 'crio' + - containerd + when: container_manager in ["crio", "containerd"] -- name: reset | remove all cri-o pods - shell: "crictl pods -q | xargs -r crictl rmp" - register: remove_all_crio_containers - retries: 4 - until: remove_all_crio_containers.rc == 0 +- name: reset | remove all cri pods + shell: "crictl pods -q | xargs -r crictl -t 60s rmp" + register: remove_all_cri_containers + retries: 5 + until: remove_all_cri_containers.rc == 0 delay: 5 tags: - crio - when: container_manager == 'crio' + - containerd + when: container_manager in ["crio", "containerd"] + +- name: reset | stop etcd services + service: + name: "{{ item }}" + state: stopped + with_items: + - etcd + - etcd-events + failed_when: false + tags: + - services + +- name: reset | remove etcd services + file: + path: "/etc/systemd/system/{{ item }}.service" + state: absent + with_items: + - etcd + - etcd-events + register: services_removed + tags: + - services + - name: reset | gather mounted kubelet dirs shell: mount | grep /var/lib/kubelet/ | awk '{print $3}' | tac args: