Enable delegating all downloads (binaries, images, kubeadm images) (#4420)
* Download to delegate and sync files when download_run_once * Fail on error after saving container image * Do not set changed status when downloaded container was up to date * Only sync containers when they are actually required Previously, non-required images (pull_required=false as image existed on target host) were synced to the target hosts. This failed as the image was not downloaded to the download_delegate and hence was not available for syncing. * Sync containers when only missing on some hosts * Consider images with multiple repo tags * Enable kubeadm images pull/syncing with download_delegate * Use kubeadm images list to pull/sync 'kubeadm config images pull' is replaced by collecting the images list with 'kubeadm config images list' and using the commonly used method of pull/syncing the images. * Ensure containers are downloaded and synced for all hosts * Fix download/syncing when download_delegate is a kubernetes host
This commit is contained in:
Timoses
Kubernetes Prow Robot
parent
e814da1eec
commit
d6fd0d2aca
11 changed files with 188 additions and 50 deletions
|
|
@ -12,14 +12,14 @@ Kubespray supports several download/upload modes. The default is:
|
|||
|
||||
There is also a "pull once, push many" mode as well:
|
||||
|
||||
* Override the ``download_run_once: True`` to download container images only once
|
||||
* Override the ``download_run_once: True`` to download container images and binaries only once
|
||||
then push to cluster nodes in batches. The default delegate node
|
||||
for pushing images is the first `kube-master`.
|
||||
for pushing is the first `kube-master`.
|
||||
* If your ansible runner node (aka the admin node) have password-less sudo and
|
||||
docker enabled, you may want to define the ``download_localhost: True``, which
|
||||
makes that node a delegate for pushing images while running the deployment with
|
||||
ansible. This maybe the case if cluster nodes cannot access each over via ssh
|
||||
or you want to use local docker images as a cache for multiple clusters.
|
||||
makes that node a delegate for pushing while running the deployment with
|
||||
ansible. This may be the case if cluster nodes cannot access each other via ssh
|
||||
or you want to use local docker images and binaries as a cache for multiple clusters.
|
||||
|
||||
Container images and binary files are described by the vars like ``foo_version``,
|
||||
``foo_download_url``, ``foo_checksum`` for binaries and ``foo_image_repo``,
|
||||
|
|
|
|||
|
|
@ -321,7 +321,7 @@ downloads:
|
|||
enabled: true
|
||||
file: true
|
||||
version: "{{ kubeadm_version }}"
|
||||
dest: "{{local_release_dir}}/kubeadm"
|
||||
dest: "{{ local_release_dir }}/kubeadm-{{ kubeadm_version }}-{{ image_arch }}"
|
||||
sha256: "{{ kubeadm_binary_checksum }}"
|
||||
url: "{{ kubeadm_download_url }}"
|
||||
unarchive: false
|
||||
|
|
@ -334,7 +334,7 @@ downloads:
|
|||
enabled: true
|
||||
file: true
|
||||
version: "{{ kube_version }}"
|
||||
dest: "{{ local_release_dir }}/hyperkube"
|
||||
dest: "{{ local_release_dir }}/hyperkube-{{ kube_version }}-{{ image_arch }}"
|
||||
sha256: "{{ hyperkube_binary_checksum }}"
|
||||
url: "{{ hyperkube_download_url }}"
|
||||
unarchive: false
|
||||
|
|
|
|||
|
|
@ -1,9 +1,6 @@
|
|||
---
|
||||
- name: container_download | Make download decision if pull is required by tag or sha256
|
||||
include_tasks: set_docker_image_facts.yml
|
||||
delegate_to: "{{ download_delegate if download_run_once or omit }}"
|
||||
delegate_facts: yes
|
||||
run_once: "{{ download_run_once }}"
|
||||
when:
|
||||
- download.enabled
|
||||
- download.container
|
||||
|
|
@ -18,11 +15,12 @@
|
|||
until: pull_task_result is succeeded
|
||||
retries: 4
|
||||
delay: "{{ retry_stagger | random + 3 }}"
|
||||
changed_when: not 'up to date' in pull_task_result.stdout
|
||||
when:
|
||||
- download_run_once
|
||||
- download.enabled
|
||||
- download.container
|
||||
- pull_required|default(download_always_pull)
|
||||
- any_pull_required | default(download_always_pull)
|
||||
delegate_to: "{{ download_delegate }}"
|
||||
delegate_facts: yes
|
||||
run_once: yes
|
||||
|
|
@ -33,6 +31,7 @@
|
|||
until: pull_task_result is succeeded
|
||||
retries: 4
|
||||
delay: "{{ retry_stagger | random + 3 }}"
|
||||
changed_when: not 'up to date' in pull_task_result.stdout
|
||||
when:
|
||||
- not download_run_once
|
||||
- download.enabled
|
||||
|
|
|
|||
|
|
@ -15,7 +15,33 @@
|
|||
- download.file
|
||||
- group_names | intersect(download.groups) | length
|
||||
|
||||
- name: file_download | Download item
|
||||
# As in 'download_container.yml':
|
||||
# In Ansible 2.4 omitting download delegate is broken. Move back
|
||||
# to one task in the future.
|
||||
- name: file_download | Download item (delegate)
|
||||
get_url:
|
||||
url: "{{download.url}}"
|
||||
dest: "{{download.dest}}"
|
||||
sha256sum: "{{download.sha256 | default(omit)}}"
|
||||
owner: "{{ download.owner|default(omit) }}"
|
||||
mode: "{{ download.mode|default(omit) }}"
|
||||
validate_certs: "{{ download_validate_certs }}"
|
||||
url_username: "{{ download.username|default(omit) }}"
|
||||
url_password: "{{ download.password|default(omit) }}"
|
||||
force_basic_auth: "{{ download.force_basic_auth|default(omit) }}"
|
||||
register: get_url_result
|
||||
until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg"
|
||||
retries: 4
|
||||
delay: "{{ retry_stagger | default(5) }}"
|
||||
delegate_to: "{{ download_delegate }}"
|
||||
when:
|
||||
- download_run_once
|
||||
- download.enabled
|
||||
- download.file
|
||||
- group_names | intersect(download.groups) | length
|
||||
run_once: yes
|
||||
|
||||
- name: file_download | Download item (all)
|
||||
get_url:
|
||||
url: "{{download.url}}"
|
||||
dest: "{{download.dest}}"
|
||||
|
|
@ -31,6 +57,7 @@
|
|||
retries: 4
|
||||
delay: "{{ retry_stagger | default(5) }}"
|
||||
when:
|
||||
- not download_run_once
|
||||
- download.enabled
|
||||
- download.file
|
||||
- group_names | intersect(download.groups) | length
|
||||
|
|
|
|||
|
|
@ -1,4 +1,22 @@
|
|||
---
|
||||
- name: kubeadm | Download kubeadm
|
||||
include_tasks: "download_file.yml"
|
||||
vars:
|
||||
download: "{{ download_defaults | combine(downloads.kubeadm) }}"
|
||||
when:
|
||||
- not skip_downloads|default(false)
|
||||
- downloads.kubeadm.enabled
|
||||
|
||||
- name: kubeadm | Sync kubeadm
|
||||
include_tasks: "sync_file.yml"
|
||||
vars:
|
||||
download: "{{ download_defaults | combine(downloads.kubeadm) }}"
|
||||
when:
|
||||
- not skip_downloads|default(false)
|
||||
- downloads.kubeadm.enabled
|
||||
- download_run_once
|
||||
- group_names | intersect(download.groups) | length
|
||||
|
||||
- name: kubeadm | Create kubeadm config
|
||||
template:
|
||||
src: "kubeadm-images.yaml.j2"
|
||||
|
|
@ -6,7 +24,7 @@
|
|||
|
||||
- name: kubeadm | Copy kubeadm binary from download dir
|
||||
synchronize:
|
||||
src: "{{ local_release_dir }}/kubeadm"
|
||||
src: "{{ local_release_dir }}/kubeadm-{{ kubeadm_version }}-{{ image_arch }}"
|
||||
dest: "{{ bin_dir }}/kubeadm"
|
||||
compress: no
|
||||
perms: yes
|
||||
|
|
@ -22,3 +40,33 @@
|
|||
|
||||
- name: container_download | download images for kubeadm config images
|
||||
command: "{{ bin_dir }}/kubeadm config images pull --config={{ kube_config_dir }}/kubeadm-images.yaml"
|
||||
when: not download_run_once
|
||||
|
||||
- name: container_download | fetch list of kubeadm config images
|
||||
command: "{{ bin_dir }}/kubeadm config images list --config={{ kube_config_dir }}/kubeadm-images.yaml"
|
||||
register: result
|
||||
run_once: true
|
||||
when: download_run_once
|
||||
changed_when: false
|
||||
|
||||
- vars:
|
||||
kubeadm_images_list: "{{ result.stdout_lines }}"
|
||||
set_fact:
|
||||
kubeadm_image:
|
||||
key: "kubeadm_{{ (item | regex_replace('^(?:.*\\/)*','')).split(':')[0] }}"
|
||||
value:
|
||||
enabled: true
|
||||
container: true
|
||||
repo: "{{ item.split(':')[0] }}"
|
||||
tag: "{{ item.split(':')[1] }}"
|
||||
groups:
|
||||
- k8s-cluster
|
||||
loop: "{{ kubeadm_images_list | flatten(levels=1) }}"
|
||||
run_once: true
|
||||
when: download_run_once
|
||||
register: result_images
|
||||
|
||||
- set_fact:
|
||||
kubeadm_images: "{{ result_images.results | map(attribute='ansible_facts.kubeadm_image') | list | items2dict }}"
|
||||
run_once: true
|
||||
when: download_run_once
|
||||
|
|
|
|||
|
|
@ -3,31 +3,36 @@
|
|||
when:
|
||||
- not skip_downloads|default(false)
|
||||
|
||||
- name: "Download items"
|
||||
include_tasks: "{{ include_file }}"
|
||||
vars:
|
||||
download: "{{ download_defaults | combine(item.value) }}"
|
||||
include_file: "download_{% if download.container %}container{% else %}file{% endif %}.yml"
|
||||
with_dict: "{{ downloads }}"
|
||||
when:
|
||||
- not skip_downloads|default(false)
|
||||
- item.value.enabled
|
||||
- (not (item.value.container|default(False))) or (item.value.container and download_container)
|
||||
|
||||
- name: "Sync container"
|
||||
include_tasks: sync_container.yml
|
||||
vars:
|
||||
download: "{{ download_defaults | combine(item.value) }}"
|
||||
with_dict: "{{ downloads }}"
|
||||
when:
|
||||
- not skip_downloads|default(false)
|
||||
- item.value.enabled
|
||||
- item.value.container | default(false)
|
||||
- download_run_once
|
||||
- group_names | intersect(download.groups) | length
|
||||
|
||||
- include_tasks: kubeadm_images.yml
|
||||
when:
|
||||
- kube_version is version('v1.11.0', '>=')
|
||||
- not skip_downloads|default(false)
|
||||
- inventory_hostname in groups['kube-master']
|
||||
|
||||
- set_fact:
|
||||
kubeadm_images: {}
|
||||
when:
|
||||
- kubeadm_images is not defined
|
||||
|
||||
- name: "Download items"
|
||||
include_tasks: "{{ include_file }}"
|
||||
vars:
|
||||
download: "{{ download_defaults | combine(item.value) }}"
|
||||
include_file: "download_{% if download.container %}container{% else %}file{% endif %}.yml"
|
||||
with_dict: "{{ downloads | combine(kubeadm_images) }}"
|
||||
when:
|
||||
- not skip_downloads|default(false)
|
||||
- item.value.enabled
|
||||
- (not (item.value.container|default(False))) or (item.value.container and download_container)
|
||||
|
||||
- name: "Sync items"
|
||||
include_tasks: "{{ include_file }}"
|
||||
vars:
|
||||
download: "{{ download_defaults | combine(item.value) }}"
|
||||
include_file: "sync_{% if download.container %}container{% else %}file{% endif %}.yml"
|
||||
with_dict: "{{ downloads | combine(kubeadm_images) }}"
|
||||
when:
|
||||
- not skip_downloads|default(false)
|
||||
- item.value.enabled
|
||||
- download_run_once
|
||||
- group_names | intersect(download.groups) | length
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
|
||||
- name: Register docker images info
|
||||
shell: >-
|
||||
{{ docker_bin_dir }}/docker images -q | xargs {{ docker_bin_dir }}/docker inspect -f "{{ '{{' }} if .RepoTags {{ '}}' }}{{ '{{' }} (index .RepoTags 0) {{ '}}' }}{{ '{{' }} end {{ '}}' }}{{ '{{' }} if .RepoDigests {{ '}}' }},{{ '{{' }} (index .RepoDigests 0) {{ '}}' }}{{ '{{' }} end {{ '}}' }}" | tr '\n' ','
|
||||
{{ docker_bin_dir }}/docker images -q | xargs -r {{ docker_bin_dir }}/docker inspect -f "{{ '{{' }} if .RepoTags {{ '}}' }}{{ '{{' }} (index .RepoTags) {{ '}}' }}{{ '{{' }} end {{ '}}' }}{{ '{{' }} if .RepoDigests {{ '}}' }},{{ '{{' }} (index .RepoDigests) {{ '}}' }}{{ '{{' }} end {{ '}}' }}" | sed -e 's/^ *\[//g' -e 's/\] *$//g' -e 's/ /\n/g' | tr '\n' ','
|
||||
no_log: true
|
||||
register: docker_images
|
||||
failed_when: false
|
||||
|
|
@ -22,6 +22,15 @@
|
|||
{%- if pull_args in docker_images.stdout.split(',') %}false{%- else -%}true{%- endif -%}
|
||||
when: not download_always_pull
|
||||
|
||||
- name: Does any host require container pull?
|
||||
vars:
|
||||
hosts_pull_required: "{{ hostvars.values() | map(attribute='pull_required') | select('defined') | list }}"
|
||||
set_fact:
|
||||
any_pull_required: "{{ True in hosts_pull_required }}"
|
||||
run_once: true
|
||||
changed_when: false
|
||||
when: not download_always_pull
|
||||
|
||||
- name: Check the local digest sha256 corresponds to the given image tag
|
||||
assert:
|
||||
that: "{{download.repo}}:{{download.tag}} in docker_images.stdout.split(',')"
|
||||
|
|
|
|||
|
|
@ -1,9 +1,6 @@
|
|||
---
|
||||
- name: container_download | Make download decision if pull is required by tag or sha256
|
||||
include: set_docker_image_facts.yml
|
||||
delegate_to: "{{ download_delegate if download_run_once or omit }}"
|
||||
delegate_facts: no
|
||||
run_once: "{{ download_run_once }}"
|
||||
when:
|
||||
- download.enabled
|
||||
- download.container
|
||||
|
|
@ -54,6 +51,7 @@
|
|||
- download.enabled
|
||||
- download.container
|
||||
- download_run_once
|
||||
- any_pull_required | default(download_always_pull)
|
||||
tags:
|
||||
- facts
|
||||
|
||||
|
|
@ -62,11 +60,12 @@
|
|||
delegate_to: "{{ download_delegate }}"
|
||||
delegate_facts: no
|
||||
register: saved
|
||||
run_once: true
|
||||
failed_when: saved.stderr != ""
|
||||
when:
|
||||
- download.enabled
|
||||
- download.container
|
||||
- download_run_once
|
||||
- any_pull_required | default(download_always_pull)
|
||||
- (ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] or download_delegate == "localhost")
|
||||
- (container_changed or not img.stat.exists)
|
||||
|
||||
|
|
@ -82,6 +81,7 @@
|
|||
- download.enabled
|
||||
- download.container
|
||||
- download_run_once
|
||||
- any_pull_required | default(download_always_pull)
|
||||
- ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
|
||||
- inventory_hostname == download_delegate
|
||||
- download_delegate != "localhost"
|
||||
|
|
@ -94,9 +94,6 @@
|
|||
use_ssh_args: "{{ has_bastion | default(false) }}"
|
||||
mode: pull
|
||||
private_key: "{{ ansible_ssh_private_key_file }}"
|
||||
delegate_to: localhost
|
||||
delegate_facts: no
|
||||
run_once: true
|
||||
become: false
|
||||
when:
|
||||
- download.enabled
|
||||
|
|
@ -113,8 +110,6 @@
|
|||
dest: "{{ fname }}"
|
||||
use_ssh_args: "{{ has_bastion | default(false) }}"
|
||||
mode: push
|
||||
delegate_to: localhost
|
||||
delegate_facts: no
|
||||
become: true
|
||||
register: get_task
|
||||
until: get_task is succeeded
|
||||
|
|
@ -124,6 +119,7 @@
|
|||
- download.enabled
|
||||
- download.container
|
||||
- download_run_once
|
||||
- pull_required|default(download_always_pull)
|
||||
- (ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] and
|
||||
inventory_hostname != download_delegate or
|
||||
download_delegate == "localhost")
|
||||
|
|
@ -137,6 +133,7 @@
|
|||
- download.enabled
|
||||
- download.container
|
||||
- download_run_once
|
||||
- pull_required|default(download_always_pull)
|
||||
- (ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] and
|
||||
inventory_hostname != download_delegate or download_delegate == "localhost")
|
||||
tags:
|
||||
|
|
|
|||
53
roles/download/tasks/sync_file.yml
Normal file
53
roles/download/tasks/sync_file.yml
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
---
|
||||
- name: file_download | create local download destination directory
|
||||
file:
|
||||
path: "{{download.dest|dirname}}"
|
||||
state: directory
|
||||
recurse: yes
|
||||
mode: 0755
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
run_once: true
|
||||
when:
|
||||
- download_delegate != "localhost"
|
||||
- download_run_once
|
||||
- download.enabled
|
||||
- download.file
|
||||
|
||||
- name: file_download | copy file to ansible host
|
||||
synchronize:
|
||||
src: "{{ download.dest }}"
|
||||
dest: "{{ download.dest }}"
|
||||
use_ssh_args: "{{ has_bastion | default(false) }}"
|
||||
mode: pull
|
||||
run_once: true
|
||||
become: false
|
||||
when:
|
||||
- download.enabled
|
||||
- download.file
|
||||
- download_run_once
|
||||
- ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"]
|
||||
- inventory_hostname == download_delegate
|
||||
- download_delegate != "localhost"
|
||||
|
||||
- name: file_download | upload file to nodes
|
||||
synchronize:
|
||||
src: "{{ download.dest }}"
|
||||
dest: "{{ download.dest }}"
|
||||
use_ssh_args: "{{ has_bastion | default(false) }}"
|
||||
mode: push
|
||||
become: true
|
||||
register: get_task
|
||||
until: get_task is succeeded
|
||||
retries: 4
|
||||
delay: "{{ retry_stagger | random + 3 }}"
|
||||
when:
|
||||
- download.enabled
|
||||
- download.file
|
||||
- download_run_once
|
||||
- (ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] and
|
||||
inventory_hostname != download_delegate or
|
||||
download_delegate == "localhost")
|
||||
tags:
|
||||
- upload
|
||||
- upgrade
|
||||
|
|
@ -19,7 +19,7 @@
|
|||
|
||||
- name: Install | Copy kubectl binary from download dir
|
||||
synchronize:
|
||||
src: "{{ local_release_dir }}/hyperkube"
|
||||
src: "{{ local_release_dir }}/hyperkube-{{ kube_version }}-{{ image_arch }}"
|
||||
dest: "{{ bin_dir }}/kubectl"
|
||||
compress: no
|
||||
perms: yes
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
- name: install | Copy kubeadm binary from download dir
|
||||
synchronize:
|
||||
src: "{{ local_release_dir }}/kubeadm"
|
||||
src: "{{ local_release_dir }}/kubeadm-{{ kubeadm_version }}-{{ image_arch }}"
|
||||
dest: "{{ bin_dir }}/kubeadm"
|
||||
compress: no
|
||||
perms: yes
|
||||
|
|
@ -25,7 +25,7 @@
|
|||
|
||||
- name: install | Copy kubelet binary from download dir
|
||||
synchronize:
|
||||
src: "{{ local_release_dir }}/hyperkube"
|
||||
src: "{{ local_release_dir }}/hyperkube-{{ kube_version }}-{{ image_arch }}"
|
||||
dest: "{{ bin_dir }}/kubelet"
|
||||
compress: no
|
||||
perms: yes
|
||||
|
|
@ -48,7 +48,7 @@
|
|||
|
||||
- name: install | Copy hyperkube binary from download dir
|
||||
synchronize:
|
||||
src: "{{ local_release_dir }}/hyperkube"
|
||||
src: "{{ local_release_dir }}/hyperkube-{{ kube_version }}-{{ image_arch }}"
|
||||
dest: "{{ bin_dir }}/hyperkube"
|
||||
compress: no
|
||||
perms: yes
|
||||
|
|
|
|||
Loading…
Reference in a new issue