Add download localhost and enable for CI

* Add download_localhost for the download_run_once mode, which is
  use the ansible host (a travis node for CI case) to store and
  distribute containers across cluster nodes in inventory.
  Defaults to false.
* Rework download_run_once logic to fix idempotency of uploading
  containers.
* For Travis CI, enable docker images caching and run Travis
  workers with sudo enabled as a dependency
* For Travis CI, deploy with download_localhost and download_run_once
  enabled to shourten dev path drastically.
* Add compression for saved container images. Defaults to 'best'.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
Co-authored-by: Aleksandr Didenko <adidenko@mirantis.com>

.travis.yml

@@ -1,4 +1,7 @@
-sudo: false
+sudo: required
+
+services:
+  - docker
 
 git:
   depth: 5
@@ -6,6 +9,7 @@ git:
 env:
   global:
     GCE_USER=travis
+    LOG_LEVEL=-v
     SSH_USER=$GCE_USER
     TEST_ID=$TRAVIS_JOB_NUMBER
     CONTAINER_ENGINE=docker
@@ -114,6 +118,8 @@ before_install:
   # W/A https://github.com/ansible/ansible-modules-core/issues/5196#issuecomment-253766186
   - pip install --user apache-libcloud==0.20.1
   - pip install --user boto==2.9.0 -U
+  # Load cached docker images
+  - if [ -d /var/tmp/releases ]; then find /var/tmp/releases -type f -name "*.tar" | xargs -I {} sh -c "zcat {} | docker load"; fi
 
 cache:
   - directories:
@@ -153,6 +159,7 @@ script:
     -e bootstrap_os=${BOOTSTRAP_OS}
     -e ansible_python_interpreter=${PYPATH}
     -e download_run_once=true
+    -e download_localhost=true
     -e local_release_dir=/var/tmp/releases
     cluster.yml
 

contrib/terraform/openstack/group_vars/all.yml

@@ -1,5 +1,5 @@
 # Valid bootstrap options (required): ubuntu, coreos, none
-bootstrap_os: "none" 
+bootstrap_os: "none"
 
 # Directory where the binaries will be installed
 bin_dir: /usr/local/bin

roles/download/defaults/main.yml

@@ -1,8 +1,18 @@
 ---
 local_release_dir: /tmp
 
-# if this is set to true will only download files once
+# if this is set to true will only download files once. Doesn't work
+# on CoreOS unless the download_localhost is true and localhost
+# is running another OS type. Default compress level is 9 (best).
 download_run_once: False
+download_compress: 9
+
+# if this is set to true, uses the localhost for download_run_once mode
+# (requires docker and sudo to access docker). You may want this option for
+# local caching of docker images or for CoreOS cluster nodes.
+# Otherwise, uses the first node in the kube-master group to store images
+# in the download_run_once mode.
+download_localhost: False
 
 # Versions
 kube_version: v1.4.6

roles/download/tasks/main.yml

@@ -43,19 +43,36 @@
     msg: "{{ download.repo }}:{{ download.tag }}"
   when: "{{ download.enabled|bool and download.container|bool }}"
 
+- set_fact:
+    download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}"
+
 - name: Create dest directory for saved/loaded container images
-  file: path="{{local_release_dir}}/containers" state=directory recurse=yes mode=0755 owner="{{ansible_ssh_user}}"
+  file: path="{{local_release_dir}}/containers" state=directory recurse=yes mode=0755 owner={{ansible_ssh_user}}
   when: "{{ download.enabled|bool and download.container|bool }}"
 
+# This is required for the download_localhost delegate to work smooth with CoreOS cluster nodes
+- name: Hack python binary path for localhost
+  raw: sh -c "mkdir -p /opt/bin; ln -sf /usr/bin/python /opt/bin/python"
+  when: "{{ download_delegate == 'localhost' }}"
+  delegate_to: localhost
+  run_once: true
+
+- name: Download | create local directory for saved/loaded container images
+  file: path="{{local_release_dir}}/containers" state=directory recurse=yes
+  delegate_to: localhost
+  become: false
+  run_once: true
+  when: "{{ download_run_once|bool and download.enabled|bool and download.container|bool and download_delegate == 'localhost' }}"
+
 #NOTE(bogdando) this brings no docker-py deps for nodes
 - name: Download containers
   command: "/usr/bin/docker pull {{ download.repo }}:{{ download.tag }}"
   register: pull_task_result
-  until: pull_task_result.rc == 0
+  until: pull_task_result|success
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   when: "{{ download.enabled|bool and download.container|bool }}"
-  delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else inventory_hostname }}"
+  delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}"
   run_once: "{{ download_run_once|bool }}"
 
 - set_fact:
@@ -69,21 +86,25 @@
   set_fact:
     container_changed: "{{ not 'up to date' in pull_task_result.stdout }}"
   when: "{{ download.enabled|bool and download.container|bool }}"
-  delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else inventory_hostname }}"
+  delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}"
   run_once: "{{ download_run_once|bool }}"
 
-- name: Download | save container images
-  shell: docker save "{{ download.repo }}:{{ download.tag }}" > "{{ fname }}"
-  delegate_to: "{{groups['kube-master'][0]}}"
-  run_once: true
-  when: ansible_os_family != "CoreOS" and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool
-
-- name: Download | create local directory for saved/loaded container images
-  file: path="{{local_release_dir}}/containers" state=directory recurse=yes mode=0755
-  delegate_to: localhost
+- name: Stat saved container image
+  stat: path="{{fname}}"
+  register: img
+  changed_when: false
+  when: "{{ download.enabled|bool and download.container|bool and download_run_once|bool }}"
+  delegate_to: "{{ download_delegate }}"
   become: false
   run_once: true
-  when: "{{ download.enabled|bool and download.container|bool }}"
+
+- name: Download | save container images
+  shell: docker save "{{ download.repo }}:{{ download.tag }}" | gzip -{{ download_compress }} > "{{ fname }}"
+  delegate_to: "{{ download_delegate }}"
+  register: saved
+  run_once: true
+  become: false
+  when: (ansible_os_family != "CoreOS" or download_delegate == "localhost") and download_run_once|bool and download.enabled|bool and download.container|bool and (container_changed|bool or not img.stat.exists)
 
 - name: Download | copy container images to ansible host
   synchronize:
@@ -92,7 +113,7 @@
     mode: pull
   delegate_to: localhost
   become: false
-  when: ansible_os_family != "CoreOS" and inventory_hostname == groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool
+  when: ansible_os_family != "CoreOS" and inventory_hostname == groups['kube-master'][0] and download_delegate != "localhost" and download_run_once|bool and download.enabled|bool and download.container|bool and saved.changed
 
 - name: Download | upload container images to nodes
   synchronize:
@@ -105,8 +126,8 @@
   until: get_task|success
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
-  when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool
+  when: (ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] or download_delegate == "localhost") and download_run_once|bool and download.enabled|bool and download.container|bool
 
 - name: Download | load container images
   shell: docker load < "{{ fname }}"
-  when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool
+  when: (ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] or download_delegate == "localhost") and download_run_once|bool and download.enabled|bool and download.container|bool

scripts/collect-info.yaml

@@ -86,7 +86,7 @@
       with_items: "{{logs}}"
 
     - name: Pack results and logs
-      local_action: shell GZIP=-9 tar --remove-files -cvzf {{dir|default(".")}}/logs.tar.gz -C /tmp collect-info
+      local_action: raw GZIP=-9 tar --remove-files -cvzf {{dir|default(".")}}/logs.tar.gz -C /tmp collect-info
       run_once: true
 
     - name: Clean up collected command outputs

tests/cloud_playbooks/upload-logs-gcs.yml

@@ -8,12 +8,12 @@
 
   tasks:
     - name: Generate uniq bucket name prefix
-      shell: date +%Y%m%d
+      raw: date +%Y%m%d
       register: out
 
     - name: replace_test_id
       set_fact:
-        test_name: "kargo-ci-{{ out.stdout }}"
+        test_name: "kargo-ci-{{ out.stdout_lines[0] }}"
 
     - set_fact:
         file_name: "{{ostype}}-{{kube_network_plugin}}-{{commit}}-logs.tar.gz"