From ef43b2159739678e3d0cc2c3df5ceeae6914e2a1 Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Tue, 13 Sep 2016 16:48:07 +0300
Subject: [PATCH 01/12] Fix kubedns idempotency

Removed api-version from kube.py because it is deprecated.
Updating both kube.py because dnsmasq one is actually used.
Fixed name back to kubedns for checking its resource.
---
 roles/dnsmasq/library/kube.py                 | 13 -------------
 roles/kubernetes-apps/ansible/library/kube.py | 13 -------------
 roles/kubernetes-apps/ansible/tasks/main.yaml |  2 +-
 3 files changed, 1 insertion(+), 27 deletions(-)

diff --git a/roles/dnsmasq/library/kube.py b/roles/dnsmasq/library/kube.py
index aab92a733..2922c6212 100644
--- a/roles/dnsmasq/library/kube.py
+++ b/roles/dnsmasq/library/kube.py
@@ -44,12 +44,6 @@ options:
     default: null
     description:
       - The url for the API server that commands are executed against.
-  api_version:
-    required: false
-    choices: ['v1', 'v1beta3']
-    default: v1
-    description:
-      - The API version associated with cluster.
   force:
     required: false
     default: false
@@ -105,10 +99,6 @@ class KubeManager(object):
         if self.kubectl is None:
             self.kubectl =  module.get_bin_path('kubectl', True)
         self.base_cmd = [self.kubectl]
-        self.api_version = module.params.get('api_version')
-
-        if self.api_version:
-            self.base_cmd.append('--api-version=' + self.api_version)
 
         if module.params.get('server'):
             self.base_cmd.append('--server=' + module.params.get('server'))
@@ -164,8 +154,6 @@ class KubeManager(object):
             return []
 
         cmd = ['replace']
-        if self.api_version != 'v1':
-            cmd = ['update']
 
         if self.force:
             cmd.append('--force')
@@ -271,7 +259,6 @@ def main():
             label=dict(),
             server=dict(),
             kubectl=dict(),
-            api_version=dict(default='v1', choices=['v1', 'v1beta3']),
             force=dict(default=False, type='bool'),
             all=dict(default=False, type='bool'),
             log_level=dict(default=0, type='int'),
diff --git a/roles/kubernetes-apps/ansible/library/kube.py b/roles/kubernetes-apps/ansible/library/kube.py
index aab92a733..2922c6212 100644
--- a/roles/kubernetes-apps/ansible/library/kube.py
+++ b/roles/kubernetes-apps/ansible/library/kube.py
@@ -44,12 +44,6 @@ options:
     default: null
     description:
       - The url for the API server that commands are executed against.
-  api_version:
-    required: false
-    choices: ['v1', 'v1beta3']
-    default: v1
-    description:
-      - The API version associated with cluster.
   force:
     required: false
     default: false
@@ -105,10 +99,6 @@ class KubeManager(object):
         if self.kubectl is None:
             self.kubectl =  module.get_bin_path('kubectl', True)
         self.base_cmd = [self.kubectl]
-        self.api_version = module.params.get('api_version')
-
-        if self.api_version:
-            self.base_cmd.append('--api-version=' + self.api_version)
 
         if module.params.get('server'):
             self.base_cmd.append('--server=' + module.params.get('server'))
@@ -164,8 +154,6 @@ class KubeManager(object):
             return []
 
         cmd = ['replace']
-        if self.api_version != 'v1':
-            cmd = ['update']
 
         if self.force:
             cmd.append('--force')
@@ -271,7 +259,6 @@ def main():
             label=dict(),
             server=dict(),
             kubectl=dict(),
-            api_version=dict(default='v1', choices=['v1', 'v1beta3']),
             force=dict(default=False, type='bool'),
             all=dict(default=False, type='bool'),
             log_level=dict(default=0, type='int'),
diff --git a/roles/kubernetes-apps/ansible/tasks/main.yaml b/roles/kubernetes-apps/ansible/tasks/main.yaml
index d5694c669..aadd9c1a5 100644
--- a/roles/kubernetes-apps/ansible/tasks/main.yaml
+++ b/roles/kubernetes-apps/ansible/tasks/main.yaml
@@ -9,7 +9,7 @@
 
 - name: Kubernetes Apps | Start Resources
   kube:
-    name: KubeDNS
+    name: kubedns
     namespace: kube-system
     kubectl: "{{bin_dir}}/kubectl"
     resource: "{{item.item.type}}"

From 3ae29d763e43e5801cee0436f2c1ccb5879ab8f4 Mon Sep 17 00:00:00 2001
From: Adrian Ursu <development@advance-leap.com>
Date: Wed, 14 Sep 2016 10:34:47 +0100
Subject: [PATCH 02/12] Added IntelliJ IDEA Project folder to .gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 1f9359992..86dec6fb4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,4 @@
 *.retry
 inventory/vagrant_ansible_inventory
 temp
+.idea

From e3ebabc3b00260a63bfb2dcf6266f98315ada4e2 Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Wed, 14 Sep 2016 18:14:26 +0300
Subject: [PATCH 03/12] switch /etc/hosts to use blockinfile

---
 roles/kubernetes/preinstall/tasks/etchosts.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/roles/kubernetes/preinstall/tasks/etchosts.yml b/roles/kubernetes/preinstall/tasks/etchosts.yml
index dd8562b8c..6f21ffa8f 100644
--- a/roles/kubernetes/preinstall/tasks/etchosts.yml
+++ b/roles/kubernetes/preinstall/tasks/etchosts.yml
@@ -1,14 +1,15 @@
 ---
 - name: Hosts | populate inventory into hosts file
-  lineinfile:
+  blockinfile:
     dest: /etc/hosts
-    regexp: "^{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }}$"
-    line: "{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }}"
+    block: |-
+      {% for item in groups['all'] -%}
+      {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }}
+      {% endfor %}
     state: present
     create: yes
     backup: yes
-  when: hostvars[item].ansible_default_ipv4.address is defined
-  with_items: "{{ groups['all'] }}"
+    marker: "# Ansible inventory hosts {mark}"
 
 - name: Hosts | populate kubernetes loadbalancer address into hosts file
   lineinfile:

From 0db441b28fba42c5153542912e19f490289bbdb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= <lukaszoles@gmail.com>
Date: Wed, 14 Sep 2016 21:27:33 +0200
Subject: [PATCH 04/12] Add socat do required pkgs

It's required for port forwarding.
---
 roles/kubernetes/preinstall/defaults/main.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml
index f419a091c..343ba5707 100644
--- a/roles/kubernetes/preinstall/defaults/main.yml
+++ b/roles/kubernetes/preinstall/defaults/main.yml
@@ -28,6 +28,7 @@ common_required_pkgs:
   - curl
   - rsync
   - bash-completion
+  - socat
 
 # Set to true if your network does not support IPv6
 # This maybe necessary for pulling Docker images from

From b69d5f6e6e25c50666c8b9ce9c187f115a04a053 Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Thu, 15 Sep 2016 16:09:40 +0300
Subject: [PATCH 05/12] Fix logic handling for use_hyperkube_cni

---
 roles/network_plugin/calico/tasks/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml
index eef15816a..1ce6c79d3 100644
--- a/roles/network_plugin/calico/tasks/main.yml
+++ b/roles/network_plugin/calico/tasks/main.yml
@@ -36,12 +36,12 @@
 - name: Calico | Install calico cni bin
   command: rsync -piu "{{ local_release_dir }}/calico/bin/calico" "/opt/cni/bin/calico"
   changed_when: false
-  when: not use_hyperkube_cni
+  when: use_hyperkube_cni|bool == false
 
 - name: Calico | Install calico-ipam cni bin
   command: rsync -piu "{{ local_release_dir }}/calico/bin/calico" "/opt/cni/bin/calico-ipam"
   changed_when: false
-  when: not use_hyperkube_cni
+  when: use_hyperkube_cni|bool == false
 
 - name: Calico | Copy cni plugins from hyperkube
   command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/"

From 422428908a09003ab650357530677859eb11af5c Mon Sep 17 00:00:00 2001
From: Bogdan Dobrelya <bdobrelia@mirantis.com>
Date: Wed, 14 Sep 2016 14:30:57 +0200
Subject: [PATCH 06/12] Download containers and save all

Move version/repo vars to download role.
Add container to download params, which overrides url/source_url,
if enabled.
Fix networking plugins download depending on kube_network_plugin.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
---
 roles/download/defaults/main.yml              | 53 ++++++++++++++++++-
 roles/download/tasks/main.yml                 | 34 ++++++++++--
 roles/etcd/defaults/main.yml                  |  8 ---
 roles/etcd/meta/main.yml                      |  5 +-
 roles/kubernetes/master/defaults/main.yml     |  3 --
 roles/kubernetes/master/meta/main.yml         |  4 +-
 roles/kubernetes/node/defaults/main.yml       |  3 --
 roles/kubernetes/node/meta/main.yml           |  4 +-
 roles/network_plugin/calico/defaults/main.yml |  6 ---
 roles/network_plugin/calico/meta/main.yml     |  6 +++
 .../network_plugin/flannel/defaults/main.yml  |  7 ---
 roles/network_plugin/flannel/meta/main.yml    |  6 +++
 12 files changed, 98 insertions(+), 41 deletions(-)
 create mode 100644 roles/network_plugin/flannel/meta/main.yml

diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index 610fb5b7a..51f0b02fd 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -13,6 +13,8 @@ etcd_version: v3.0.6
 calico_version: v0.20.0
 calico_cni_version: v1.3.1
 weave_version: v1.6.1
+flannel_version: 0.5.5
+flannel_server_helper_version: 0.1
 
 # Download URL's
 etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd"
@@ -26,6 +28,22 @@ calico_cni_ipam_checksum: "3df6951a30749c279229e7e318e74ac4e41263996125be65257db
 weave_checksum: "9bf9d6e5a839e7bcbb28cc00c7acae9d09284faa3e7a3720ca9c2b9e93c68580"
 etcd_checksum: "385afd518f93e3005510b7aaa04d38ee4a39f06f5152cd33bb86d4f0c94c7485"
 
+# Containers
+# Possible values: host, docker
+etcd_deployment_type: "docker"
+etcd_image_repo: "quay.io/coreos/etcd"
+etcd_image_tag: "{{ etcd_version }}"
+flannel_server_helper_image_repo: "gcr.io/google_containers/flannel-server-helper"
+flannel_server_helper_image_tag: "{{ flannel_server_helper_version }}"
+flannel_image_repo: "quay.io/coreos/flannel"
+flannel_image_tag: "{{ flannel_version }}"
+calicoctl_image_repo: "calico/ctl"
+calicoctl_image_tag: "{{ calico_version }}"
+calico_node_image_repo: "calico/node"
+calico_node_image_tag: "{{ calico_version }}"
+hyperkube_image_repo: "quay.io/coreos/hyperkube"
+hyperkube_image_tag: "{{ kube_version }}_coreos.0"
+
 downloads:
   calico_cni_plugin:
     dest: calico/bin/calico
@@ -35,6 +53,7 @@ downloads:
     url: "{{ calico_cni_download_url }}"
     owner: "root"
     mode: "0755"
+    enabled: "{{ kube_network_plugin == 'calico' }}"
   calico_cni_plugin_ipam:
     dest: calico/bin/calico-ipam
     version: "{{calico_cni_version}}"
@@ -43,6 +62,7 @@ downloads:
     url: "{{ calico_cni_ipam_download_url }}"
     owner: "root"
     mode: "0755"
+    enabled: "{{ kube_network_plugin == 'calico' }}"
   weave:
     dest: weave/bin/weave
     version: "{{weave_version}}"
@@ -51,6 +71,7 @@ downloads:
     sha256: "{{ weave_checksum }}"
     owner: "root"
     mode: "0755"
+    enabled: "{{ kube_network_plugin == 'weave' }}"
   etcd:
     version: "{{etcd_version}}"
     dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
@@ -60,10 +81,38 @@ downloads:
     unarchive: true
     owner: "etcd"
     mode: "0755"
-  nothing:
-    enabled: false
+    container: "{{ etcd_deployment_type == 'docker' }}"
+    repo: "{{ etcd_image_repo }}"
+    tag: "{{ etcd_image_tag }}"
+  hyperkube:
+    container: true
+    repo: "{{ hyperkube_image_repo }}"
+    tag: "{{ hyperkube_image_tag }}"
+  flannel:
+    container: true
+    repo: "{{ flannel_image_repo }}"
+    tag: "{{ flannel_image_tag }}"
+    enabled: "{{ kube_network_plugin == 'flannel' }}"
+  flannel_server_helper:
+    container: true
+    repo: "{{ flannel_server_helper_image_repo }}"
+    tag: "{{ flannel_server_helper_image_tag }}"
+    enabled: "{{ kube_network_plugin == 'flannel' }}"
+  calicoctl:
+    container: true
+    repo: "{{ calicoctl_image_repo }}"
+    tag: "{{ calicoctl_image_tag }}"
+    enabled: "{{ kube_network_plugin == 'calico' }}"
+  calico_node:
+    container: true
+    repo: "{{ calico_node_image_repo }}"
+    tag: "{{ calico_node_image_tag }}"
+    enabled: "{{ kube_network_plugin == 'calico' }}"
 
 download:
+  container: "{{ file.container|default('false') }}"
+  repo: "{{ file.repo|default(None) }}"
+  tag: "{{ file.tag|default(None) }}"
   enabled: "{{ file.enabled|default('true') }}"
   dest: "{{ file.dest|default(None) }}"
   version: "{{ file.version|default(None) }}"
diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml
index 40c52f5ea..8d9c351a3 100644
--- a/roles/download/tasks/main.yml
+++ b/roles/download/tasks/main.yml
@@ -4,11 +4,11 @@
 - name: downloading...
   debug:
     msg: "{{ download.url }}"
-  when: "{{ download.enabled|bool }}"
+  when: "{{ download.enabled|bool and not download.container|bool }}"
 
 - name: Create dest directories
   file: path={{local_release_dir}}/{{download.dest|dirname}} state=directory recurse=yes
-  when: "{{ download.enabled|bool }}"
+  when: "{{ download.enabled|bool and not download.container|bool }}"
   run_once: "{{ download_run_once|bool }}"
 
 - name: Download items
@@ -18,7 +18,11 @@
     sha256sum: "{{download.sha256 | default(omit)}}"
     owner: "{{ download.owner|default(omit) }}"
     mode: "{{ download.mode|default(omit) }}"
-  when: "{{ download.enabled|bool }}"
+  register: get_url_result
+  until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg"
+  retries: 4
+  delay: "{{ 20 | random + 3 }}"
+  when: "{{ download.enabled|bool and not download.container|bool }}"
   run_once: "{{ download_run_once|bool }}"
 
 - name: Extract archives
@@ -28,7 +32,7 @@
     owner: "{{ download.owner|default(omit) }}"
     mode: "{{ download.mode|default(omit) }}"
     copy: no
-  when: "{{ download.enabled|bool }} and ({{download.unarchive is defined and download.unarchive == True}})"
+  when: "{{ download.enabled|bool and not download.container|bool and download.unarchive is defined and download.unarchive == True }}"
   run_once: "{{ download_run_once|bool }}"
 
 - name: Fix permissions
@@ -37,5 +41,25 @@
     path: "{{local_release_dir}}/{{download.dest}}"
     owner: "{{ download.owner|default(omit) }}"
     mode: "{{ download.mode|default(omit) }}"
-  when: "{{ download.enabled|bool }} and ({{download.unarchive is not defined or download.unarchive == False}})"
+  when: "{{ download.enabled|bool and not download.container|bool and (download.unarchive is not defined or download.unarchive == False) }}"
+  run_once: "{{ download_run_once|bool }}"
+
+- name: pulling...
+  debug:
+    msg: "{{ download.repo }}:{{ download.tag }}"
+  when: "{{ download.enabled|bool and download.container|bool }}"
+
+- name: Create dest directory for container images to be saved
+  file: path="{{local_release_dir}}/containers" state=directory recurse=yes
+  when: "{{ download.enabled|bool and download.container|bool }}"
+  run_once: "{{ download_run_once|bool }}"
+
+#NOTE(bogdando) this brings no docker-py deps for nodes
+- name: Download containers
+  command: "/usr/bin/docker pull {{ download.repo }}:{{ download.tag }}"
+  register: pull_task_result
+  until: pull_task_result.rc == 0
+  retries: 4
+  delay: "{{ 20 | random + 3 }}"
+  when: "{{ download.enabled|bool and download.container|bool }}"
   run_once: "{{ download_run_once|bool }}"
diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml
index 585f75a40..02234a2fe 100644
--- a/roles/etcd/defaults/main.yml
+++ b/roles/etcd/defaults/main.yml
@@ -1,10 +1,2 @@
 ---
-etcd_version: v3.0.6
 etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"
-
-# Possible values: host, docker
-etcd_deployment_type: "docker"
-
-
-etcd_image_repo: "quay.io/coreos/etcd"
-etcd_image_tag: "{{ etcd_version }}"
diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml
index 8e4cb5846..b55966a99 100644
--- a/roles/etcd/meta/main.yml
+++ b/roles/etcd/meta/main.yml
@@ -3,8 +3,7 @@ dependencies:
   - role: adduser
     user: "{{ addusers.etcd }}"
     when: ansible_os_family != 'CoreOS'
-  - role: download
-    file: "{{ downloads.etcd }}"
-    when: etcd_deployment_type == "host"
   - role: docker
     when: (ansible_os_family != "CoreOS" and etcd_deployment_type == "docker" or inventory_hostname in groups['k8s-cluster'])
+  - role: download
+    file: "{{ downloads.etcd }}"
diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml
index ac23c0ada..d0be14d64 100644
--- a/roles/kubernetes/master/defaults/main.yml
+++ b/roles/kubernetes/master/defaults/main.yml
@@ -10,6 +10,3 @@ kube_users_dir: "{{ kube_config_dir }}/users"
 # An experimental dev/test only dynamic volumes provisioner,
 # for PetSets. Works for kube>=v1.3 only.
 kube_hostpath_dynamic_provisioner: "false"
-
-hyperkube_image_repo: "quay.io/coreos/hyperkube"
-hyperkube_image_tag: "{{ kube_version }}_coreos.0"
diff --git a/roles/kubernetes/master/meta/main.yml b/roles/kubernetes/master/meta/main.yml
index 021c01de4..f4da42e39 100644
--- a/roles/kubernetes/master/meta/main.yml
+++ b/roles/kubernetes/master/meta/main.yml
@@ -1,4 +1,4 @@
 ---
 dependencies:
-  - role: download # For kube_version variable
-    file: "{{ downloads.nothing }}"
+  - role: download
+    file: "{{ downloads.hyperkube }}"
diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml
index 96b941a5e..7dc7d1183 100644
--- a/roles/kubernetes/node/defaults/main.yml
+++ b/roles/kubernetes/node/defaults/main.yml
@@ -8,9 +8,6 @@ kube_resolv_conf: "/etc/resolv.conf"
 
 kube_proxy_mode: iptables
 
-hyperkube_image_repo: "quay.io/coreos/hyperkube"
-hyperkube_image_tag: "{{ kube_version }}_coreos.0"
-
 # IP address of the DNS server.
 # Kubernetes will create a pod with several containers, serving as the DNS
 # server and expose it under this IP address. The IP address must be from
diff --git a/roles/kubernetes/node/meta/main.yml b/roles/kubernetes/node/meta/main.yml
index c65c68393..b9cbbd9ff 100644
--- a/roles/kubernetes/node/meta/main.yml
+++ b/roles/kubernetes/node/meta/main.yml
@@ -1,5 +1,5 @@
 ---
 dependencies:
-  - role: download #For kube_version
-    file: "{{ downloads.nothing }}"
+  - role: download
+    file: "{{ downloads.hyperkube }}"
   - role: kubernetes/secrets
diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml
index 3cdf5b492..b5b275e04 100644
--- a/roles/network_plugin/calico/defaults/main.yml
+++ b/roles/network_plugin/calico/defaults/main.yml
@@ -7,9 +7,3 @@ ipip: false
 
 # cloud_provider can only be set to 'gce' or 'aws'
 # cloud_provider:
-
-calicoctl_image_repo: calico/ctl
-calicoctl_image_tag: "{{ calico_version }}"
-
-calico_node_image_repo: calico/node
-calico_node_image_tag: "{{ calico_version }}"
diff --git a/roles/network_plugin/calico/meta/main.yml b/roles/network_plugin/calico/meta/main.yml
index 92ab5391b..c13e976d3 100644
--- a/roles/network_plugin/calico/meta/main.yml
+++ b/roles/network_plugin/calico/meta/main.yml
@@ -4,3 +4,9 @@ dependencies:
     file: "{{ downloads.calico_cni_plugin }}"
   - role: download
     file: "{{ downloads.calico_cni_plugin_ipam }}"
+  - role: download
+    file: "{{ downloads.calico_node }}"
+  - role: download
+    file: "{{ downloads.calicoctl }}"
+  - role: download
+    file: "{{ downloads.hyperkube }}"
diff --git a/roles/network_plugin/flannel/defaults/main.yml b/roles/network_plugin/flannel/defaults/main.yml
index 8ff48cdb0..ce00090ec 100644
--- a/roles/network_plugin/flannel/defaults/main.yml
+++ b/roles/network_plugin/flannel/defaults/main.yml
@@ -10,10 +10,3 @@ flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address
 # You can choose what type of flannel backend to use
 # please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md
 flannel_backend_type: "vxlan"
-
-
-flannel_server_helper_image_repo: "gcr.io/google_containers/flannel-server-helper"
-flannel_server_helper_image_tag: "0.1"
-
-flannel_image_repo: "quay.io/coreos/flannel"
-flannel_image_tag: "0.5.5"
diff --git a/roles/network_plugin/flannel/meta/main.yml b/roles/network_plugin/flannel/meta/main.yml
new file mode 100644
index 000000000..935d9c3bb
--- /dev/null
+++ b/roles/network_plugin/flannel/meta/main.yml
@@ -0,0 +1,6 @@
+---
+dependencies:
+  - role: download
+    file: "{{ downloads.flannel_server_helper }}"
+  - role: download
+    file: "{{ downloads.flannel }}"

From 9926395e5b4f3f35d00b33fd7a4d130977f70df7 Mon Sep 17 00:00:00 2001
From: Bogdan Dobrelya <bdobrelia@mirantis.com>
Date: Wed, 14 Sep 2016 18:20:10 +0200
Subject: [PATCH 07/12] Distribute downloaded artifacts

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
---
 roles/download/tasks/main.yml | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml
index 8d9c351a3..73622f06d 100644
--- a/roles/download/tasks/main.yml
+++ b/roles/download/tasks/main.yml
@@ -9,6 +9,7 @@
 - name: Create dest directories
   file: path={{local_release_dir}}/{{download.dest|dirname}} state=directory recurse=yes
   when: "{{ download.enabled|bool and not download.container|bool }}"
+  delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else omit }}"
   run_once: "{{ download_run_once|bool }}"
 
 - name: Download items
@@ -23,6 +24,7 @@
   retries: 4
   delay: "{{ 20 | random + 3 }}"
   when: "{{ download.enabled|bool and not download.container|bool }}"
+  delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else omit }}"
   run_once: "{{ download_run_once|bool }}"
 
 - name: Extract archives
@@ -33,6 +35,7 @@
     mode: "{{ download.mode|default(omit) }}"
     copy: no
   when: "{{ download.enabled|bool and not download.container|bool and download.unarchive is defined and download.unarchive == True }}"
+  delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else omit }}"
   run_once: "{{ download_run_once|bool }}"
 
 - name: Fix permissions
@@ -42,6 +45,7 @@
     owner: "{{ download.owner|default(omit) }}"
     mode: "{{ download.mode|default(omit) }}"
   when: "{{ download.enabled|bool and not download.container|bool and (download.unarchive is not defined or download.unarchive == False) }}"
+  delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else omit }}"
   run_once: "{{ download_run_once|bool }}"
 
 - name: pulling...
@@ -49,10 +53,9 @@
     msg: "{{ download.repo }}:{{ download.tag }}"
   when: "{{ download.enabled|bool and download.container|bool }}"
 
-- name: Create dest directory for container images to be saved
+- name: Create dest directory for saved/loaded container images
   file: path="{{local_release_dir}}/containers" state=directory recurse=yes
   when: "{{ download.enabled|bool and download.container|bool }}"
-  run_once: "{{ download_run_once|bool }}"
 
 #NOTE(bogdando) this brings no docker-py deps for nodes
 - name: Download containers
@@ -62,4 +65,29 @@
   retries: 4
   delay: "{{ 20 | random + 3 }}"
   when: "{{ download.enabled|bool and download.container|bool }}"
+  delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else omit }}"
   run_once: "{{ download_run_once|bool }}"
+
+- set_fact:
+    fname: "{{local_release_dir}}/containers/{{download.repo|regex_replace('/|\0|:', '_')}}:{{download.tag|regex_replace('/|\0|:', '_')}}.tar"
+
+- name: Download | save container images
+  shell: docker save "{{ download.repo }}:{{ download.tag }}" > "{{ fname }}"
+  delegate_to: "{{groups['kube-master'][0]}}"
+  run_once: true
+  when: ansible_os_family != "CoreOS" and download_run_once|bool
+
+- name: Download | get container images
+  synchronize:
+    src: "{{ fname }}"
+    dest: "{{local_release_dir}}/containers"
+    mode: push
+  register: get_task
+  until: get_task|success
+  retries: 4
+  delay: "{{ 20 | random + 3 }}"
+  when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool
+
+- name: Download | load container images
+  shell: docker load < "{{ fname }}"
+  when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool

From 390764c2b4e5777c74ef52ab5e4199ea34151e69 Mon Sep 17 00:00:00 2001
From: Bogdan Dobrelya <bdobrelia@mirantis.com>
Date: Thu, 15 Sep 2016 11:23:27 +0200
Subject: [PATCH 08/12] Add retry_stagger var for failed download/pushes.

* Add the retry_stagger var to tweak push and retry time strategies.
* Add large deployments related docs.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
---
 docs/large-deploymets.md                   | 19 +++++++++++++++++++
 inventory/group_vars/all.yml               |  2 ++
 roles/docker/tasks/main.yml                |  4 ++--
 roles/download/tasks/main.yml              |  6 +++---
 roles/etcd/tasks/install.yml               |  2 +-
 roles/kubernetes/master/tasks/main.yml     |  2 +-
 roles/kubernetes/preinstall/tasks/main.yml |  2 +-
 roles/network_plugin/calico/tasks/main.yml |  2 +-
 8 files changed, 30 insertions(+), 9 deletions(-)
 create mode 100644 docs/large-deploymets.md

diff --git a/docs/large-deploymets.md b/docs/large-deploymets.md
new file mode 100644
index 000000000..2a36c3ebc
--- /dev/null
+++ b/docs/large-deploymets.md
@@ -0,0 +1,19 @@
+Large deployments of K8s
+========================
+
+For a large scaled deployments, consider the following configuration changes:
+
+* Tune [ansible settings](http://docs.ansible.com/ansible/intro_configuration.html)
+  for `forks` and `timeout` vars to fit large numbers of nodes being deployed.
+
+* Override containers' `foo_image_repo` vars to point to intranet registry.
+
+* Override the ``download_run_once: true`` to download binaries and container
+  images only once then push to nodes in batches.
+
+* Adjust the `retry_stagger` global var as appropriate. It should provide sane
+  load on a delegate (the first K8s master node) then retrying failed
+  push or download operations.
+
+For example, when deploying 200 nodes, you may want to run ansible with
+``--forks=50``, ``--timeout=600`` and define the ``retry_stagger: 60``.
diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml
index 91fab4c06..2de01828c 100644
--- a/inventory/group_vars/all.yml
+++ b/inventory/group_vars/all.yml
@@ -7,6 +7,8 @@ bin_dir: /usr/local/bin
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
 local_release_dir: "/tmp/releases"
+# Random shifts for retrying failed ops like pushing/downloading
+retry_stagger: 5
 
 # Uncomment this line for CoreOS only.
 # Directory where python binary is installed
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index 826e16978..1d237f5e9 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -30,7 +30,7 @@
   register: keyserver_task_result
   until: keyserver_task_result|success
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{ docker_repo_key_info.repo_keys }}"
   when: ansible_os_family != "CoreOS"
 
@@ -58,7 +58,7 @@
   register: docker_task_result
   until: docker_task_result|success
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{ docker_package_info.pkgs }}"
   when: (ansible_os_family != "CoreOS") and (docker_package_info.pkgs|length > 0)
 
diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml
index 73622f06d..6329a1108 100644
--- a/roles/download/tasks/main.yml
+++ b/roles/download/tasks/main.yml
@@ -22,7 +22,7 @@
   register: get_url_result
   until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg"
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   when: "{{ download.enabled|bool and not download.container|bool }}"
   delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else omit }}"
   run_once: "{{ download_run_once|bool }}"
@@ -63,7 +63,7 @@
   register: pull_task_result
   until: pull_task_result.rc == 0
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   when: "{{ download.enabled|bool and download.container|bool }}"
   delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else omit }}"
   run_once: "{{ download_run_once|bool }}"
@@ -85,7 +85,7 @@
   register: get_task
   until: get_task|success
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool
 
 - name: Download | load container images
diff --git a/roles/etcd/tasks/install.yml b/roles/etcd/tasks/install.yml
index 959133c29..aa7f32ca3 100644
--- a/roles/etcd/tasks/install.yml
+++ b/roles/etcd/tasks/install.yml
@@ -20,7 +20,7 @@
   register: etcd_task_result
   until: etcd_task_result.rc == 0
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   changed_when: false
 
 #Plan B: looks nicer, but requires docker-py on all hosts:
diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml
index e8dfe08fc..ff6abcb13 100644
--- a/roles/kubernetes/master/tasks/main.yml
+++ b/roles/kubernetes/master/tasks/main.yml
@@ -12,7 +12,7 @@
   register: kube_task_result
   until: kube_task_result.rc == 0
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   changed_when: false
 
 - name: Write kube-apiserver manifest
diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml
index 8c22b73bf..8c2aecec5 100644
--- a/roles/kubernetes/preinstall/tasks/main.yml
+++ b/roles/kubernetes/preinstall/tasks/main.yml
@@ -104,7 +104,7 @@
   register: pkgs_task_result
   until: pkgs_task_result|success
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{required_pkgs | default([]) | union(common_required_pkgs|default([]))}}"
   when: ansible_os_family != "CoreOS"
 
diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml
index 1ce6c79d3..ff7bc32ae 100644
--- a/roles/network_plugin/calico/tasks/main.yml
+++ b/roles/network_plugin/calico/tasks/main.yml
@@ -48,7 +48,7 @@
   register: cni_task_result
   until: cni_task_result.rc == 0
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   changed_when: false
   when: use_hyperkube_cni
 

From 5ed3916f8239cb32ae5655b5aaffb6c55312ee8f Mon Sep 17 00:00:00 2001
From: Bogdan Dobrelya <bdobrelia@mirantis.com>
Date: Fri, 16 Sep 2016 12:45:40 +0200
Subject: [PATCH 09/12] Fix use_hyperkube_cni logic

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
---
 roles/network_plugin/calico/tasks/main.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml
index ff7bc32ae..69f91949a 100644
--- a/roles/network_plugin/calico/tasks/main.yml
+++ b/roles/network_plugin/calico/tasks/main.yml
@@ -36,12 +36,12 @@
 - name: Calico | Install calico cni bin
   command: rsync -piu "{{ local_release_dir }}/calico/bin/calico" "/opt/cni/bin/calico"
   changed_when: false
-  when: use_hyperkube_cni|bool == false
+  when: "{{ not use_hyperkube_cni|bool }}"
 
 - name: Calico | Install calico-ipam cni bin
   command: rsync -piu "{{ local_release_dir }}/calico/bin/calico" "/opt/cni/bin/calico-ipam"
   changed_when: false
-  when: use_hyperkube_cni|bool == false
+  when: "{{ not use_hyperkube_cni|bool }}"
 
 - name: Calico | Copy cni plugins from hyperkube
   command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/"
@@ -50,7 +50,7 @@
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   changed_when: false
-  when: use_hyperkube_cni
+  when: "{{ use_hyperkube_cni|bool }}"
 
 - name: Calico | wait for etcd
   uri: url=http://localhost:2379/health

From 341ea5a6eacf08dedcdaaad5deba64a28d3f252e Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Sun, 18 Sep 2016 19:57:36 +0400
Subject: [PATCH 10/12] always bind etcd_proxy to localhost

---
 roles/kubernetes/preinstall/tasks/set_facts.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml
index 527c29b32..e3f4757a7 100644
--- a/roles/kubernetes/preinstall/tasks/set_facts.yml
+++ b/roles/kubernetes/preinstall/tasks/set_facts.yml
@@ -25,7 +25,7 @@
 - set_fact: etcd_access_address="{{ access_ip | default(etcd_address) }}"
 - set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380"
 - set_fact: etcd_client_url="http://{{ etcd_access_address }}:2379"
-- set_fact: etcd_authority="{{ access_ip|default('127.0.0.1') }}:2379"
+- set_fact: etcd_authority="127.0.0.1:2379"
 - set_fact: etcd_endpoint="http://{{ etcd_authority }}"
 - set_fact:
     etcd_access_addresses: |-

From 13874f46106474288f9e9211298732bbae0b48ca Mon Sep 17 00:00:00 2001
From: Ivan Shvedunov <ishvedunov@mirantis.com>
Date: Fri, 16 Sep 2016 21:01:37 +0300
Subject: [PATCH 11/12] Fix reverse DNS lookups of service IPs.

This fixes "DNS should provide DNS for services [Conformance]"
e2e test in k8s.
---
 roles/dnsmasq/templates/01-kube-dns.conf.j2 | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/roles/dnsmasq/templates/01-kube-dns.conf.j2 b/roles/dnsmasq/templates/01-kube-dns.conf.j2
index 5ade101ce..a9ef6200e 100644
--- a/roles/dnsmasq/templates/01-kube-dns.conf.j2
+++ b/roles/dnsmasq/templates/01-kube-dns.conf.j2
@@ -4,8 +4,6 @@ listen-address=0.0.0.0
 
 addn-hosts=/etc/hosts
 
-bogus-priv
-
 #Set upstream dns servers
 {% if upstream_dns_servers is defined %}
 {% for srv in upstream_dns_servers %}
@@ -18,3 +16,6 @@ server={{ srv }}
 
 # Forward k8s domain to kube-dns
 server=/{{ dns_domain }}/{{ skydns_server }}
+
+# Forward reverse lookups for k8s service addresses to kube-dns
+rev-server={{ kube_service_addresses }},{{ skydns_server }}

From 42a5055d3c71fb167d7c92b307d5c52b3b5cc410 Mon Sep 17 00:00:00 2001
From: "Sean M. Collins" <sean@coreitpro.com>
Date: Mon, 19 Sep 2016 11:51:37 -0400
Subject: [PATCH 12/12] Rename large-deploymets.md to large-deployments.md

Filename was a typo
---
 docs/{large-deploymets.md => large-deployments.md} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename docs/{large-deploymets.md => large-deployments.md} (100%)

diff --git a/docs/large-deploymets.md b/docs/large-deployments.md
similarity index 100%
rename from docs/large-deploymets.md
rename to docs/large-deployments.md