From c11f981692f8c1c7d9f6c918ad7ed21a19adc528 Mon Sep 17 00:00:00 2001
From: Jan Jungnickel <jan@jungnickel.com>
Date: Mon, 7 Nov 2016 12:11:16 +0100
Subject: [PATCH 01/10] Initial support for vsphere as cloud provider

---
 inventory/group_vars/all.yml                  |  2 +-
 .../manifests/kube-apiserver.manifest.j2      |  2 +-
 roles/kubernetes/node/templates/kubelet.j2    |  2 +-
 roles/kubernetes/preinstall/defaults/main.yml | 10 +++++++
 roles/kubernetes/preinstall/tasks/main.yml    | 16 ++++++----
 .../tasks/vsphere-credential-check.yml        | 30 +++++++++++++++++++
 .../templates/vsphere-cloud-config.j2         |  9 ++++++
 7 files changed, 62 insertions(+), 9 deletions(-)
 create mode 100644 roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml
 create mode 100644 roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2

diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml
index 56a777e05..12ca18d9b 100644
--- a/inventory/group_vars/all.yml
+++ b/inventory/group_vars/all.yml
@@ -47,7 +47,7 @@
 
 ## There are some changes specific to the cloud providers
 ## for instance we need to encapsulate packets with some network plugins
-## If set the possible values are either 'gce', 'aws', 'azure' or 'openstack'
+## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', or 'vsphere'
 ## When openstack is used make sure to source in the openstack credentials
 ## like you would do when using nova-client before starting the playbook.
 #cloud_provider:
diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
index c05030697..c4882bd56 100644
--- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
@@ -51,7 +51,7 @@ spec:
 {% endif %}
     - --v={{ kube_log_level }}
     - --allow-privileged=true
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure"] %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %}
     - --cloud-provider={{ cloud_provider }}
     - --cloud-config={{ kube_config_dir }}/cloud_config
 {% elif cloud_provider is defined and cloud_provider == "aws" %}
diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2
index 8ec348a05..54d3b38aa 100644
--- a/roles/kubernetes/node/templates/kubelet.j2
+++ b/roles/kubernetes/node/templates/kubelet.j2
@@ -42,7 +42,7 @@ KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kuben
 {% endif %}
 # Should this cluster be allowed to run privileged docker containers
 KUBE_ALLOW_PRIV="--allow-privileged=true"
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure"] %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere" ] %}
 KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"
 {% elif cloud_provider is defined and cloud_provider == "aws" %}
 KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }}"
diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml
index 59076c204..517b91b72 100644
--- a/roles/kubernetes/preinstall/defaults/main.yml
+++ b/roles/kubernetes/preinstall/defaults/main.yml
@@ -32,3 +32,13 @@ openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')|default(lookup('env','OS_P
 # Container Linux by CoreOS cloud init config file to define /etc/resolv.conf content
 # for hostnet pods and infra needs
 resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf
+
+# For the vSphere integration kubelet will need credentials to access
+# the api. Per default this values will be
+# read from the environment.
+vsphere_username: "{{ lookup('env', VSPHERE_USERNAME ) }}"
+vsphere_password: "{{ lookup('env', VSPHERE_PASSWORD ) }}"
+vsphere_server: "{{ lookup('env', VSPHERE_SERVER ) }}"
+vsphere_datacenter: "{{ lookup('env', VSPHERE_DATACENTER ) }}"
+vsphere_datastore: "{{ lookup('env', VSPHERE_DATASTORE ) }}"
+vsphere_working_dir: "{{ lookup('env', VSPHERE_WORKING_DIR ) }}"
diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml
index 5b79c101d..8dd422353 100644
--- a/roles/kubernetes/preinstall/tasks/main.yml
+++ b/roles/kubernetes/preinstall/tasks/main.yml
@@ -64,8 +64,8 @@
 
 - name: check cloud_provider value
   fail:
-    msg: "If set the 'cloud_provider' var must be set either to 'generic', 'gce', 'aws', 'azure' or 'openstack'"
-  when: cloud_provider is defined and cloud_provider not in ['generic', 'gce', 'aws', 'openstack', 'azure']
+    msg: "If set the 'cloud_provider' var must be set either to 'generic', 'gce', 'aws', 'azure', 'vsphere',  or 'openstack'"
+  when: cloud_provider is defined and cloud_provider not in ['generic', 'gce', 'aws', 'openstack', 'azure', 'vsphere']
   tags: [cloud-provider, facts]
 
 - include: openstack-credential-check.yml
@@ -76,6 +76,10 @@
   when: cloud_provider is defined and cloud_provider == 'azure'
   tags: [cloud-provider, azure, facts]
 
+- include: vsphere-credential-check.yml
+  when: cloud_provider is defined and cloud_provider == 'vsphere'
+  tags: [cloud-provider, vsphere, facts]
+
 - name: Create cni directories
   file:
     path: "{{ item }}"
@@ -179,14 +183,14 @@
     state: present
   tags: bootstrap-os
 
-- name: Write openstack cloud-config
+- name: "Write {{ cloud_provider}} cloud-config"
   template:
-    src: openstack-cloud-config.j2
+    src: "{{ cloud_provider }}-cloud-config.j2"
     dest: "{{ kube_config_dir }}/cloud_config"
     group: "{{ kube_cert_group }}"
     mode: 0640
-  when: inventory_hostname in groups['k8s-cluster'] and cloud_provider is defined and cloud_provider == "openstack"
-  tags: [cloud-provider, openstack]
+  when: inventory_hostname in groups['k8s-cluster'] and cloud_provider is defined and cloud_provider in [ "openstack", "vsphere" ]
+  tags: [cloud-provider, openstack, vsphere]
 
 - name: Write azure cloud-config
   template:
diff --git a/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml b/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml
new file mode 100644
index 000000000..8f0b3ba50
--- /dev/null
+++ b/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml
@@ -0,0 +1,30 @@
+---
+- name: check vsphere_username value
+  fail:
+    msg: "vsphere_username is missing"
+  when: vsphere_username is not defined or vsphere_username == ""
+
+- name: check vsphere_password value
+  fail:
+    msg: "vsphere_password is missing"
+  when: vsphere_password is not defined or vsphere_password == ""
+
+- name: check vsphere_server value
+  fail:
+    msg: "vsphere_server is missing"
+  when: vsphere_server is not defined or vsphere_server == ""
+
+- name: check vsphere_datacenter value
+  fail:
+    msg: "vsphere_datacenter is missing"
+  when: vsphere_datacenter is not defined or vsphere_datacenter == ""
+
+- name: check vsphere_datastore value
+  fail:
+    msg: "vsphere_datastore is missing"
+  when: vsphere_datastore is not defined or vsphere_datastore == ""
+
+- name: check vsphere_working_dir value
+  fail:
+    msg: "vsphere_working_dir is missing"
+  when: vsphere_working_dir is not defined or vsphere_working_dir == ""
diff --git a/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2 b/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2
new file mode 100644
index 000000000..d350f973c
--- /dev/null
+++ b/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2
@@ -0,0 +1,9 @@
+[Global]
+user = {{ vsphere_username }}
+password = {{ vsphere_password }}
+server = {{ vsphere_server }}
+port = 443
+insecure-flag = true
+datacenter = {{ vsphere_datacenter }}
+datastore = {{ vsphere_datastore }}
+working-dir = {{ vsphere_working_dir }}

From 6a144213c93047fc6f08456f40abb5726e206e15 Mon Sep 17 00:00:00 2001
From: Brad Beam <brad.beam@b-rad.info>
Date: Thu, 16 Feb 2017 21:59:40 -0600
Subject: [PATCH 02/10] Updating vsphere cloud provider support

---
 .../kube-controller-manager.manifest.j2       |  6 +--
 roles/kubernetes/node/templates/kubelet.j2    |  2 +-
 roles/kubernetes/preinstall/defaults/main.yml | 26 ++++++----
 roles/kubernetes/preinstall/tasks/main.yml    | 33 +++----------
 .../tasks/vsphere-credential-check.yml        | 49 ++++++++-----------
 .../templates/vsphere-cloud-config.j2         | 21 ++++++--
 6 files changed, 64 insertions(+), 73 deletions(-)

diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
index 7bcd51cc4..6faf6dea5 100644
--- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
@@ -32,7 +32,7 @@ spec:
     - --node-monitor-period={{ kube_controller_node_monitor_period }}
     - --pod-eviction-timeout={{ kube_controller_pod_eviction_timeout }}
     - --v={{ kube_log_level }}
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure"] %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %}
     - --cloud-provider={{cloud_provider}}
     - --cloud-config={{ kube_config_dir }}/cloud_config
 {% elif cloud_provider is defined and cloud_provider == "aws" %}
@@ -54,7 +54,7 @@ spec:
     - mountPath: {{ kube_cert_dir }}
       name: ssl-certs-kubernetes
       readOnly: true
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure"] %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere" ] %}
     - mountPath: {{ kube_config_dir }}/cloud_config
       name: cloudconfig
       readOnly: true
@@ -63,7 +63,7 @@ spec:
   - hostPath:
       path: {{ kube_cert_dir }}
     name: ssl-certs-kubernetes
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure"] %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %}
   - hostPath:
       path: {{ kube_config_dir }}/cloud_config
     name: cloudconfig
diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2
index 54d3b38aa..10135c13f 100644
--- a/roles/kubernetes/node/templates/kubelet.j2
+++ b/roles/kubernetes/node/templates/kubelet.j2
@@ -42,7 +42,7 @@ KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kuben
 {% endif %}
 # Should this cluster be allowed to run privileged docker containers
 KUBE_ALLOW_PRIV="--allow-privileged=true"
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere" ] %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %}
 KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"
 {% elif cloud_provider is defined and cloud_provider == "aws" %}
 KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }}"
diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml
index 517b91b72..c775f748d 100644
--- a/roles/kubernetes/preinstall/defaults/main.yml
+++ b/roles/kubernetes/preinstall/defaults/main.yml
@@ -29,16 +29,22 @@ openstack_password: "{{ lookup('env','OS_PASSWORD')  }}"
 openstack_region: "{{ lookup('env','OS_REGION_NAME')  }}"
 openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')|default(lookup('env','OS_PROJECT_ID'),true)  }}"
 
+# For the vsphere integration, kubelet will need credentials to access
+# vsphere apis
+# Documentation regarting these values can be found 
+# https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/providers/vsphere/vsphere.go#L105
+vsphere_vcenter_ip: "{{ lookup('env', 'VSPHERE_VCENTER') }}"
+vsphere_vcenter_port: "{{ lookup('env', 'VSPHERE_VCENTER_PORT') }}"
+vsphere_user: "{{ lookup('env', 'VSPHERE_USER') }}"
+vsphere_password: "{{ lookup('env', 'VSPHERE_PASSWORD') }}"
+vsphere_datacenter: "{{ lookup('env', 'VSPHERE_DATACENTER') }}"
+vsphere_datastore: "{{ lookup('env', 'VSPHERE_DATASTORE') }}"
+vsphere_working_dir: "{{ lookup('env', 'VSPHERE_WORKING_DIR') }}"
+vsphere_insecure: "{{ lookup('env', 'VSPHERE_INSECURE') }}"
+vsphere_scsi_controller_type: pvscsi
+# vsphere_public_network is name of the network the VMs are joined to
+vsphere_public_network: "{{ lookup('env', 'VSPHERE_PUBLIC_NETWORK')|default('') }}"
+
 # Container Linux by CoreOS cloud init config file to define /etc/resolv.conf content
 # for hostnet pods and infra needs
 resolveconf_cloud_init_conf: /etc/resolveconf_cloud_init.conf
-
-# For the vSphere integration kubelet will need credentials to access
-# the api. Per default this values will be
-# read from the environment.
-vsphere_username: "{{ lookup('env', VSPHERE_USERNAME ) }}"
-vsphere_password: "{{ lookup('env', VSPHERE_PASSWORD ) }}"
-vsphere_server: "{{ lookup('env', VSPHERE_SERVER ) }}"
-vsphere_datacenter: "{{ lookup('env', VSPHERE_DATACENTER ) }}"
-vsphere_datastore: "{{ lookup('env', VSPHERE_DATASTORE ) }}"
-vsphere_working_dir: "{{ lookup('env', VSPHERE_WORKING_DIR ) }}"
diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml
index 8dd422353..c995ed21e 100644
--- a/roles/kubernetes/preinstall/tasks/main.yml
+++ b/roles/kubernetes/preinstall/tasks/main.yml
@@ -64,21 +64,13 @@
 
 - name: check cloud_provider value
   fail:
-    msg: "If set the 'cloud_provider' var must be set either to 'generic', 'gce', 'aws', 'azure', 'vsphere',  or 'openstack'"
-  when: cloud_provider is defined and cloud_provider not in ['generic', 'gce', 'aws', 'openstack', 'azure', 'vsphere']
+    msg: "If set the 'cloud_provider' var must be set either to 'generic', 'gce', 'aws', 'azure', 'openstack' or 'vsphere'"
+  when: cloud_provider is defined and cloud_provider not in ['generic', 'gce', 'aws', 'azure', 'openstack', 'vsphere']
   tags: [cloud-provider, facts]
 
-- include: openstack-credential-check.yml
-  when: cloud_provider is defined and cloud_provider == 'openstack'
-  tags: [cloud-provider, openstack, facts]
-
-- include: azure-credential-check.yml
-  when: cloud_provider is defined and cloud_provider == 'azure'
-  tags: [cloud-provider, azure, facts]
-
-- include: vsphere-credential-check.yml
-  when: cloud_provider is defined and cloud_provider == 'vsphere'
-  tags: [cloud-provider, vsphere, facts]
+- include: "{{ cloud_provider }}-credential-check.yml"
+  when: cloud_provider is defined and cloud_provider in [ 'openstack', 'azure', 'vsphere' ]
+  tags: [cloud-provider, "{{ cloud_provider }}", facts]
 
 - name: Create cni directories
   file:
@@ -183,23 +175,14 @@
     state: present
   tags: bootstrap-os
 
-- name: "Write {{ cloud_provider}} cloud-config"
+- name: Write cloud-config
   template:
     src: "{{ cloud_provider }}-cloud-config.j2"
     dest: "{{ kube_config_dir }}/cloud_config"
     group: "{{ kube_cert_group }}"
     mode: 0640
-  when: inventory_hostname in groups['k8s-cluster'] and cloud_provider is defined and cloud_provider in [ "openstack", "vsphere" ]
-  tags: [cloud-provider, openstack, vsphere]
-
-- name: Write azure cloud-config
-  template:
-    src: azure-cloud-config.j2
-    dest: "{{ kube_config_dir }}/cloud_config"
-    group: "{{ kube_cert_group }}"
-    mode: 0640
-  when: inventory_hostname in groups['k8s-cluster'] and cloud_provider is defined and cloud_provider == "azure"
-  tags: [cloud-provider, azure]
+  when: inventory_hostname in groups['k8s-cluster'] and cloud_provider is defined and cloud_provider in [ 'openstack', 'azure', 'vsphere' ]
+  tags: [cloud-provider, "{{ cloud_provider }}"]
 
 - include: etchosts.yml
   tags: [bootstrap-os, etchosts]
diff --git a/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml b/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml
index 8f0b3ba50..b91726d50 100644
--- a/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml
+++ b/roles/kubernetes/preinstall/tasks/vsphere-credential-check.yml
@@ -1,30 +1,21 @@
----
-- name: check vsphere_username value
+- name: check vsphere environment variables
   fail:
-    msg: "vsphere_username is missing"
-  when: vsphere_username is not defined or vsphere_username == ""
-
-- name: check vsphere_password value
-  fail:
-    msg: "vsphere_password is missing"
-  when: vsphere_password is not defined or vsphere_password == ""
-
-- name: check vsphere_server value
-  fail:
-    msg: "vsphere_server is missing"
-  when: vsphere_server is not defined or vsphere_server == ""
-
-- name: check vsphere_datacenter value
-  fail:
-    msg: "vsphere_datacenter is missing"
-  when: vsphere_datacenter is not defined or vsphere_datacenter == ""
-
-- name: check vsphere_datastore value
-  fail:
-    msg: "vsphere_datastore is missing"
-  when: vsphere_datastore is not defined or vsphere_datastore == ""
-
-- name: check vsphere_working_dir value
-  fail:
-    msg: "vsphere_working_dir is missing"
-  when: vsphere_working_dir is not defined or vsphere_working_dir == ""
+    msg: "{{ item.name }} is missing"
+  when: item.value is not defined or item.value == ''
+  with_items:
+    - name: vsphere_vcenter_ip
+      value: "{{ vsphere_vcenter_ip }}"
+    - name: vsphere_vcenter_port
+      value: "{{ vsphere_vcenter_port }}"
+    - name: vsphere_user
+      value: "{{ vsphere_user }}"
+    - name: vsphere_password
+      value: "{{ vsphere_password }}"
+    - name: vsphere_datacenter
+      value: "{{ vsphere_datacenter }}"
+    - name: vsphere_datastore
+      value: "{{ vsphere_datastore }}"
+    - name: vsphere_working_dir
+      value: "{{ vsphere_working_dir }}"
+    - name: vsphere_insecure
+      value: "{{ vsphere_insecure }}"
diff --git a/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2 b/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2
index d350f973c..c68ac0f55 100644
--- a/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2
+++ b/roles/kubernetes/preinstall/templates/vsphere-cloud-config.j2
@@ -1,9 +1,20 @@
 [Global]
-user = {{ vsphere_username }}
-password = {{ vsphere_password }}
-server = {{ vsphere_server }}
-port = 443
-insecure-flag = true
 datacenter = {{ vsphere_datacenter }}
 datastore = {{ vsphere_datastore }}
+insecure-flag = {{ vsphere_insecure }}
+password = {{ vsphere_password }}
+port = {{ vsphere_vcenter_port }}
+server = {{ vsphere_vcenter_ip }}
+user = {{ vsphere_user }}
 working-dir = {{ vsphere_working_dir }}
+{% if vsphere_vm_uuid is defined %}
+vm-uuid = {{ vsphere_vm_uuid }}
+{% endif %}
+
+[Disk]
+scsicontrollertype = {{ vsphere_scsi_controller_type }}
+
+{% if vsphere_public_network is defined and vsphere_public_network != ""  %}
+[Network]
+public-network = {{ vsphere_public_network }}
+{% endif %}

From 8fc5a844b30a83d27ed5e2de843d11311e834bb4 Mon Sep 17 00:00:00 2001
From: Vijay Katam <vijkatam@cisco.com>
Date: Fri, 24 Feb 2017 13:41:27 -0800
Subject: [PATCH 03/10] Add support for atomic host

Updates based on feedback

Simplify checks for file exists

remove invalid char

Review feedback. Use regular systemd file.

Add template for docker systemd atomic
---
 Vagrantfile                                   |  3 +-
 docs/atomic.md                                | 22 +++++++++++
 docs/vars.md                                  |  1 -
 roles/bootstrap-os/tasks/main.yml             | 10 ++++-
 roles/docker/tasks/main.yml                   |  8 ++--
 roles/docker/tasks/systemd.yml                |  9 ++++-
 roles/docker/templates/docker-dns.conf.j2     |  2 +-
 roles/docker/templates/docker-options.conf.j2 |  2 +-
 .../docker/templates/docker_atomic.service.j2 | 38 +++++++++++++++++++
 roles/etcd/meta/main.yml                      |  2 +-
 roles/kernel-upgrade/tasks/main.yml           |  2 +-
 .../manifests/kube-proxy.manifest.j2          |  4 ++
 roles/kubernetes/preinstall/meta/main.yml     |  1 +
 roles/kubernetes/preinstall/tasks/main.yml    |  6 +--
 .../kubernetes/preinstall/tasks/set_facts.yml | 12 ++++++
 15 files changed, 107 insertions(+), 15 deletions(-)
 create mode 100644 docs/atomic.md
 create mode 100644 roles/docker/templates/docker_atomic.service.j2

diff --git a/Vagrantfile b/Vagrantfile
index 8d3f2bbdd..b769199b1 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -23,6 +23,7 @@ $etcd_instances = $num_instances
 $kube_master_instances = $num_instances == 1 ? $num_instances : ($num_instances - 1)
 # All nodes are kube nodes
 $kube_node_instances = $num_instances
+$local_release_dir = "/vagrant/temp"
 
 host_vars = {}
 
@@ -97,7 +98,7 @@ Vagrant.configure("2") do |config|
         "ip": ip,
         "flannel_interface": ip,
         "flannel_backend_type": "host-gw",
-        "local_release_dir": "/vagrant/temp",
+        "local_release_dir" => $local_release_dir,
         "download_run_once": "False",
         # Override the default 'calico' with flannel.
         # inventory/group_vars/k8s-cluster.yml
diff --git a/docs/atomic.md b/docs/atomic.md
new file mode 100644
index 000000000..cb506a9f3
--- /dev/null
+++ b/docs/atomic.md
@@ -0,0 +1,22 @@
+Atomic host bootstrap
+=====================
+
+Atomic host testing has been done with the network plugin flannel. Change the inventory var `kube_network_plugin: flannel`.
+
+Note: Flannel is the only plugin that has currently been tested with atomic
+
+### Vagrant
+
+* For bootstrapping with Vagrant, use box centos/atomic-host 
+* Update VagrantFile variable `local_release_dir` to `/var/vagrant/temp`.
+* Update `vm_memory = 2048` and `vm_cpus = 2`
+* Networking on vagrant hosts has to be brought up manually once they are booted.
+
+    ```
+    vagrant ssh
+    sudo /sbin/ifup enp0s8
+    ```
+
+* For users of vagrant-libvirt download qcow2 format from https://wiki.centos.org/SpecialInterestGroup/Atomic/Download/
+
+Then you can proceed to [cluster deployment](#run-deployment)
\ No newline at end of file
diff --git a/docs/vars.md b/docs/vars.md
index b763f6a34..966b3ffc8 100644
--- a/docs/vars.md
+++ b/docs/vars.md
@@ -102,4 +102,3 @@ Stack](https://github.com/kubernetes-incubator/kargo/blob/master/docs/dns-stack.
 
 Kargo sets up two Kubernetes accounts by default: ``root`` and ``kube``. Their
 passwords default to changeme. You can set this by changing ``kube_api_pwd``.
-
diff --git a/roles/bootstrap-os/tasks/main.yml b/roles/bootstrap-os/tasks/main.yml
index 7f1355577..4adefb394 100644
--- a/roles/bootstrap-os/tasks/main.yml
+++ b/roles/bootstrap-os/tasks/main.yml
@@ -8,4 +8,12 @@
 - include: bootstrap-centos.yml
   when: bootstrap_os == "centos"
 
-- include: setup-pipelining.yml
\ No newline at end of file
+- include: setup-pipelining.yml
+
+- name: check if atomic host
+  stat:
+    path: /run/ostree-booted
+  register: ostree
+
+- set_fact:
+    is_atomic: "{{ ostree.stat.exists }}"
\ No newline at end of file
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index 3e7b342f2..cdfae8242 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -38,7 +38,7 @@
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{ docker_repo_key_info.repo_keys }}"
-  when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]
+  when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic)
 
 - name: ensure docker repository is enabled
   action: "{{ docker_repo_info.pkg_repo }}"
@@ -46,13 +46,13 @@
     repo: "{{item}}"
     state: present
   with_items: "{{ docker_repo_info.repos }}"
-  when: (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]) and (docker_repo_info.repos|length > 0)
+  when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic) and (docker_repo_info.repos|length > 0)
 
 - name: Configure docker repository on RedHat/CentOS
   template:
     src: "rh_docker.repo.j2"
     dest: "/etc/yum.repos.d/docker.repo"
-  when: ansible_distribution in ["CentOS","RedHat"]
+  when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic
 
 - name: ensure docker packages are installed
   action: "{{ docker_package_info.pkg_mgr }}"
@@ -66,7 +66,7 @@
   delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{ docker_package_info.pkgs }}"
   notify: restart docker
-  when: (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]) and (docker_package_info.pkgs|length > 0)
+  when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic) and (docker_package_info.pkgs|length > 0)
 
 - name: check minimum docker version for docker_dns mode. You need at least docker version >= 1.12 for resolvconf_mode=docker_dns
   command: "docker version -f '{{ '{{' }}.Client.Version{{ '}}' }}'"
diff --git a/roles/docker/tasks/systemd.yml b/roles/docker/tasks/systemd.yml
index 18710ac49..1275de5d7 100644
--- a/roles/docker/tasks/systemd.yml
+++ b/roles/docker/tasks/systemd.yml
@@ -15,7 +15,14 @@
     src: docker.service.j2
     dest: /etc/systemd/system/docker.service
   register: docker_service_file
-  when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]
+  when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic)
+
+- name: Write docker.service systemd file for atomic
+  template:
+    src: docker_atomic.service.j2
+    dest: /etc/systemd/system/docker.service
+  notify: restart docker
+  when: is_atomic
 
 - name: Write docker options systemd drop-in
   template:
diff --git a/roles/docker/templates/docker-dns.conf.j2 b/roles/docker/templates/docker-dns.conf.j2
index 01dbd3b20..d501a19c0 100644
--- a/roles/docker/templates/docker-dns.conf.j2
+++ b/roles/docker/templates/docker-dns.conf.j2
@@ -3,4 +3,4 @@ Environment="DOCKER_DNS_OPTIONS=\
     {% for d in docker_dns_servers %}--dns {{ d }} {% endfor %} \
     {% for d in docker_dns_search_domains %}--dns-search {{ d }} {% endfor %} \
     {% for o in docker_dns_options %}--dns-opt {{ o }} {% endfor %} \
-"
+"
\ No newline at end of file
diff --git a/roles/docker/templates/docker-options.conf.j2 b/roles/docker/templates/docker-options.conf.j2
index 50356a9f4..012795898 100644
--- a/roles/docker/templates/docker-options.conf.j2
+++ b/roles/docker/templates/docker-options.conf.j2
@@ -1,2 +1,2 @@
 [Service]
-Environment="DOCKER_OPTS={% if docker_options is defined %}{{ docker_options }}{% endif %}"
+Environment="DOCKER_OPTS={% if docker_options is defined %}{{ docker_options }}{% endif %}"
\ No newline at end of file
diff --git a/roles/docker/templates/docker_atomic.service.j2 b/roles/docker/templates/docker_atomic.service.j2
new file mode 100644
index 000000000..ba37bf4c3
--- /dev/null
+++ b/roles/docker/templates/docker_atomic.service.j2
@@ -0,0 +1,38 @@
+[Unit]
+Description=Docker Application Container Engine
+Documentation=http://docs.docker.com
+After=network.target
+Wants=docker-storage-setup.service
+
+[Service]
+Type=notify
+NotifyAccess=all
+EnvironmentFile=-/etc/sysconfig/docker
+EnvironmentFile=-/etc/sysconfig/docker-storage
+EnvironmentFile=-/etc/sysconfig/docker-network
+Environment=GOTRACEBACK=crash
+Environment=DOCKER_HTTP_HOST_COMPAT=1
+Environment=PATH=/usr/libexec/docker:/usr/bin:/usr/sbin
+ExecReload=/bin/kill -s HUP $MAINPID
+Delegate=yes
+KillMode=process
+ExecStart=/usr/bin/dockerd-current \
+          --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current \
+          --default-runtime=docker-runc \
+          --exec-opt native.cgroupdriver=systemd \
+          --userland-proxy-path=/usr/libexec/docker/docker-proxy-current \
+          $DOCKER_OPTS \
+          $DOCKER_STORAGE_OPTIONS \
+          $DOCKER_NETWORK_OPTIONS \
+          $DOCKER_DNS_OPTIONS \
+          $ADD_REGISTRY \
+          $BLOCK_REGISTRY \
+          $INSECURE_REGISTRY
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+TimeoutStartSec=1min
+Restart=on-abnormal
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml
index bff76a129..9bd6f02a3 100644
--- a/roles/etcd/meta/main.yml
+++ b/roles/etcd/meta/main.yml
@@ -2,7 +2,7 @@
 dependencies:
   - role: adduser
     user: "{{ addusers.etcd }}"
-    when: not ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
+    when: not (ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] or is_atomic)
   - role: download
     file: "{{ downloads.etcd }}"
     tags: download
diff --git a/roles/kernel-upgrade/tasks/main.yml b/roles/kernel-upgrade/tasks/main.yml
index 999eb94ae..a16f0f37b 100644
--- a/roles/kernel-upgrade/tasks/main.yml
+++ b/roles/kernel-upgrade/tasks/main.yml
@@ -2,4 +2,4 @@
 
 - include: centos-7.yml
   when: ansible_distribution in ["CentOS","RedHat"] and
-        ansible_distribution_major_version >= 7
+        ansible_distribution_major_version >= 7 and not is_atomic
\ No newline at end of file
diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
index a965ef792..2dbcf74d1 100644
--- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
+++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
@@ -50,7 +50,11 @@ spec:
   volumes:
   - name: ssl-certs-host
     hostPath:
+{% if ansible_os_family == 'RedHat' %}
+      path: /etc/pki/tls
+{% else %}
       path: /usr/share/ca-certificates
+{% endif %}
   - name: "kubeconfig"
     hostPath:
       path: "{{kube_config_dir}}/node-kubeconfig.yaml"
diff --git a/roles/kubernetes/preinstall/meta/main.yml b/roles/kubernetes/preinstall/meta/main.yml
index cf440f5e2..203d968a7 100644
--- a/roles/kubernetes/preinstall/meta/main.yml
+++ b/roles/kubernetes/preinstall/meta/main.yml
@@ -3,3 +3,4 @@ dependencies:
   - role: adduser
     user: "{{ addusers.kube }}"
     tags: kubelet
+    when: not is_atomic
\ No newline at end of file
diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml
index 5b79c101d..27e98949d 100644
--- a/roles/kubernetes/preinstall/tasks/main.yml
+++ b/roles/kubernetes/preinstall/tasks/main.yml
@@ -91,7 +91,7 @@
   yum:
     update_cache: yes
     name: '*'
-  when: ansible_pkg_mgr == 'yum'
+  when: ansible_pkg_mgr == 'yum' and not is_atomic
   tags: bootstrap-os
 
 - name: Install latest version of python-apt for Debian distribs
@@ -112,7 +112,7 @@
 
 - name: Install epel-release on RedHat/CentOS
   shell: rpm -qa | grep epel-release || rpm -ivh {{ epel_rpm_download_url }}
-  when: ansible_distribution in ["CentOS","RedHat"]
+  when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic
   changed_when: False
   check_mode: no
   tags: bootstrap-os
@@ -127,7 +127,7 @@
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{required_pkgs | default([]) | union(common_required_pkgs|default([]))}}"
-  when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]
+  when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic)
   tags: bootstrap-os
 
 # Todo : selinux configuration
diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml
index 2481fcd7f..6a25c785e 100644
--- a/roles/kubernetes/preinstall/tasks/set_facts.yml
+++ b/roles/kubernetes/preinstall/tasks/set_facts.yml
@@ -83,5 +83,17 @@
 - set_fact:
     peer_with_calico_rr: "{{ 'calico-rr' in groups and groups['calico-rr']|length > 0 }}"
 
+- name: check if atomic host
+  stat:
+    path: /run/ostree-booted
+  register: ostree
+
+- set_fact:
+    is_atomic: "{{ ostree.stat.exists }}"
+
+- set_fact:
+    kube_cert_group: "kube"
+  when: is_atomic
+
 - include: set_resolv_facts.yml
   tags: [bootstrap-os, resolvconf, facts]

From 3ef7365caeaf8bbfaf4e5e5d899bbcfee48f47b7 Mon Sep 17 00:00:00 2001
From: Vincent Schwarzer <vincent.schwarzer@yahoo.de>
Date: Mon, 27 Feb 2017 11:20:16 +0100
Subject: [PATCH 04/10] Modified how adding LB for the Kube API is handled
 (AWS)

Until now it was not possible to add an API Loadbalancer
without an static IP Address. But certain Loadbalancers
like AWS Elastic Loadbalanacer dontt have an fixed IP address.
With this commit it is possible to add these kind of Loadbalancers
to the Kargo deployment.
---
 roles/kubernetes/preinstall/tasks/etchosts.yml  | 2 +-
 roles/kubernetes/preinstall/tasks/set_facts.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/kubernetes/preinstall/tasks/etchosts.yml b/roles/kubernetes/preinstall/tasks/etchosts.yml
index 08c941e6f..edcc89c37 100644
--- a/roles/kubernetes/preinstall/tasks/etchosts.yml
+++ b/roles/kubernetes/preinstall/tasks/etchosts.yml
@@ -17,7 +17,7 @@
     line: "{{ loadbalancer_apiserver.address }} {{ apiserver_loadbalancer_domain_name| default('lb-apiserver.kubernetes.local') }}"
     state: present
     backup: yes
-  when: loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined
+  when: loadbalancer_apiserver.address is defined and apiserver_loadbalancer_domain_name is defined
 
 - name: Hosts | localhost ipv4 in hosts file
   lineinfile:
diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml
index 2481fcd7f..a2232d380 100644
--- a/roles/kubernetes/preinstall/tasks/set_facts.yml
+++ b/roles/kubernetes/preinstall/tasks/set_facts.yml
@@ -22,7 +22,7 @@
     kube_apiserver_endpoint: |-
       {% if not is_kube_master and loadbalancer_apiserver_localhost|default(false) -%}
            https://localhost:{{ nginx_kube_apiserver_port|default(kube_apiserver_port) }}
-      {%- elif is_kube_master and loadbalancer_apiserver is not defined -%}
+      {%- elif is_kube_master -%}
            http://127.0.0.1:{{ kube_apiserver_insecure_port }}
       {%- else -%}
       {%-   if loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined -%}

From 9164b9cdcf2ddc591a1186a35c304ef3a1604753 Mon Sep 17 00:00:00 2001
From: Vincent Schwarzer <vincent.schwarzer@yahoo.de>
Date: Tue, 28 Feb 2017 18:39:19 +0100
Subject: [PATCH 05/10] Changes based on feedback (additional ansible checks)

---
 roles/kubernetes/preinstall/tasks/etchosts.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/kubernetes/preinstall/tasks/etchosts.yml b/roles/kubernetes/preinstall/tasks/etchosts.yml
index edcc89c37..181fbcb0f 100644
--- a/roles/kubernetes/preinstall/tasks/etchosts.yml
+++ b/roles/kubernetes/preinstall/tasks/etchosts.yml
@@ -17,7 +17,7 @@
     line: "{{ loadbalancer_apiserver.address }} {{ apiserver_loadbalancer_domain_name| default('lb-apiserver.kubernetes.local') }}"
     state: present
     backup: yes
-  when: loadbalancer_apiserver.address is defined and apiserver_loadbalancer_domain_name is defined
+  when: loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined and apiserver_loadbalancer_domain_name is defined
 
 - name: Hosts | localhost ipv4 in hosts file
   lineinfile:

From 7e1aa3b43b76cea471e3b77c4703c58b7e7bb6a4 Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Fri, 3 Mar 2017 16:21:01 +0300
Subject: [PATCH 06/10] Use find module for checking for certificates

Also generate certs only when absent on master (rather than
when absent on target node)
---
 roles/etcd/tasks/check_certs.yml              | 73 +++++++++++--------
 roles/etcd/tasks/gen_certs_script.yml         | 45 +++++++++---
 .../kubernetes/secrets/tasks/check-certs.yml  | 67 ++++++++++-------
 .../secrets/tasks/gen_certs_script.yml        |  4 +-
 4 files changed, 121 insertions(+), 68 deletions(-)

diff --git a/roles/etcd/tasks/check_certs.yml b/roles/etcd/tasks/check_certs.yml
index bc14e255f..9bb32f162 100644
--- a/roles/etcd/tasks/check_certs.yml
+++ b/roles/etcd/tasks/check_certs.yml
@@ -1,18 +1,11 @@
 ---
 - name: "Check_certs | check if all certs have already been generated on first master"
-  stat:
-    path: "{{ etcd_cert_dir }}/{{ item }}"
-    get_md5: no
+  find:
+    paths: "{{ etcd_cert_dir }}"
+    patterns: "ca.pem,node*.pem"
   delegate_to: "{{groups['etcd'][0]}}"
   register: etcdcert_master
   run_once: true
-  with_items: >-
-       ['ca.pem',
-       {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique %}
-       {% for host in all_etcd_hosts %}
-       'node-{{ host }}-key.pem'
-       {% if not loop.last %}{{','}}{% endif %}
-       {% endfor %}]
 
 - name: "Check_certs | Set default value for 'sync_certs', 'gen_certs' and 'etcd_secret_changed' to false"
   set_fact:
@@ -20,34 +13,56 @@
     gen_certs: false
     etcd_secret_changed: false
 
-- name: "Check_certs | Set 'gen_certs' to true"
-  set_fact:
-    gen_certs: true
-  when: "not {{item.stat.exists}}"
-  run_once: true
-  with_items: "{{etcdcert_master.results}}"
-
-- name: "Check certs | check if a cert already exists"
+- name: "Check certs | check if a cert already exists on node"
   stat:
     path: "{{ etcd_cert_dir }}/{{ item }}"
-  register: etcdcert
+  register: etcdcert_node
   with_items:
     - ca.pem
     - node-{{ inventory_hostname }}-key.pem
 
+
+- name: "Check_certs | Set 'gen_certs' to true"
+  set_fact:
+    gen_certs: true
+  when: "not '{{ item }}' in etcdcert_master.files|map(attribute='path') | list"
+  run_once: true
+  with_items: >-
+       ['{{etcd_cert_dir}}/ca.pem',
+       {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort %}
+       {% for host in all_etcd_hosts %}
+       '{{etcd_cert_dir}}/node-{{ host }}-key.pem'
+       {% if not loop.last %}{{','}}{% endif %}
+       {% endfor %}]
+
+
+- name: "Check_certs | Set 'gen_node_certs' to true"
+  set_fact:
+    gen_node_certs: |-
+      {
+      {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort -%}
+      {% set existing_certs = etcdcert_master.files|map(attribute='path')|list|sort %}
+      {% for host in all_etcd_hosts -%}
+        {% set host_cert = "%s/node-%s-key.pem"|format(etcd_cert_dir, host) %}
+        {% if host_cert in existing_certs -%}
+        "{{ host }}": False,
+        {% else -%}
+        "{{ host }}": True,
+        {% endif -%}
+      {% endfor %}
+      }
+  run_once: true
+
 - name: "Check_certs | Set 'sync_certs' to true"
   set_fact:
     sync_certs: true
   when: >-
       {%- set certs = {'sync': False} -%}
-      {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique %}
-      {% for host in all_etcd_hosts %}
-        {% if host == inventory_hostname %}
-          {% if (not etcdcert.results[0].stat.exists|default(False)) or
-              (not etcdcert.results[1].stat.exists|default(False)) or
-              (etcdcert.results[1].stat.checksum|default('') != etcdcert_master.results[loop.index].stat.checksum|default('')) -%}
-            {%- set _ = certs.update({'sync': True}) -%}
-          {% endif %}
-        {% endif %}
-      {%- endfor -%}
+      {% if gen_node_certs[inventory_hostname] or 
+        (not etcdcert_node.results[0].stat.exists|default(False)) or
+          (not etcdcert_node.results[1].stat.exists|default(False)) or
+            (etcdcert_node.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcdcert_node.results[1].stat.path)|first|map(attribute="checksum")|default('')) -%}
+              {%- set _ = certs.update({'sync': True}) -%}
+      {% endif %}
       {{ certs.sync }}
+
diff --git a/roles/etcd/tasks/gen_certs_script.yml b/roles/etcd/tasks/gen_certs_script.yml
index bb6d55660..06d86257c 100644
--- a/roles/etcd/tasks/gen_certs_script.yml
+++ b/roles/etcd/tasks/gen_certs_script.yml
@@ -43,15 +43,15 @@
   when: gen_certs|default(false)
 
 - name: Gen_certs | run cert generation script
-  command: "{{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
+  command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
   environment:
     - MASTERS: "{% for m in groups['etcd'] %}
-                  {% if hostvars[m].sync_certs|default(false) %}
+                  {% if gen_node_certs[m] %}
                     {{ m }}
                   {% endif %}
                 {% endfor %}"
     - HOSTS: "{% for h in (groups['k8s-cluster'] + groups['calico-rr']|default([]))|unique %}
-                {% if hostvars[h].sync_certs|default(false) %}
+                {% if gen_node_certs[h] %}
                     {{ h }}
                 {% endif %}
               {% endfor %}"
@@ -107,14 +107,38 @@
         sync_certs|default(false) and inventory_hostname not in groups['etcd']
   notify: set etcd_secret_changed
 
-- name: Gen_certs | Copy certs on masters
-  shell: "base64 -d <<< '{{etcd_master_cert_data.stdout|quote}}' | tar xz -C {{ etcd_cert_dir }}"
-  args:
-    executable: /bin/bash
-  no_log: true
-  changed_when: false
+#NOTE(mattymo): Use temporary file to copy master certs because we have a ~200k 
+#char limit when using shell command                                            
+                                                                                
+#FIXME(mattymo): Use tempfile module in ansible 2.3                             
+- name: Gen_certs | Prepare tempfile for unpacking certs                        
+  shell: mktemp /tmp/certsXXXXX.tar.gz                                          
+  register: cert_tempfile                                                       
+                                                                                
+- name: Gen_certs | Write master certs to tempfile                              
+  copy:                                                                         
+    content: "{{etcd_master_cert_data.stdout}}"                                      
+    dest: "{{cert_tempfile.stdout}}"                                            
+    owner: root                                                                 
+    mode: "0600"                                                                
   when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
-        inventory_hostname != groups['etcd'][0]
+        inventory_hostname != groups['etcd'][0]                          
+                                                                                
+- name: Gen_certs | Unpack certs on masters                                     
+  shell: "base64 -d < {{ cert_tempfile.stdout }} | tar xz -C {{ etcd_cert_dir }}"
+  no_log: true                                                                  
+  changed_when: false                                                           
+  check_mode: no                                                                
+  when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
+        inventory_hostname != groups['etcd'][0]                          
+  notify: set secret_changed                                                    
+                                                                                
+- name: Gen_certs | Cleanup tempfile                                            
+  file:                                                                         
+    path: "{{cert_tempfile.stdout}}"                                            
+    state: absent                                                               
+  when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
+        inventory_hostname != groups['etcd'][0]  
 
 - name: Gen_certs | Copy certs on nodes
   shell: "base64 -d <<< '{{etcd_node_cert_data.stdout|quote}}' | tar xz -C {{ etcd_cert_dir }}"
@@ -163,4 +187,3 @@
 - name: Gen_certs | update ca-certificates (RedHat)
   command: update-ca-trust extract
   when: etcd_ca_cert.changed and ansible_os_family == "RedHat"
-
diff --git a/roles/kubernetes/secrets/tasks/check-certs.yml b/roles/kubernetes/secrets/tasks/check-certs.yml
index 0d5f23814..41cef85c1 100644
--- a/roles/kubernetes/secrets/tasks/check-certs.yml
+++ b/roles/kubernetes/secrets/tasks/check-certs.yml
@@ -1,16 +1,11 @@
 ---
 - name: "Check_certs | check if the certs have already been generated on first master"
-  stat:
-    path: "{{ kube_cert_dir }}/{{ item }}"
+  find:
+    paths: "{{ kube_cert_dir }}"
+    patterns: "*.pem"
   delegate_to: "{{groups['kube-master'][0]}}"
   register: kubecert_master
   run_once: true
-  with_items: >-
-       ['ca.pem',
-       {% for host in groups['k8s-cluster'] %}
-       'node-{{ host }}-key.pem'
-       {% if not loop.last %}{{','}}{% endif %}
-       {% endfor %}]
 
 - name: "Check_certs | Set default value for 'sync_certs', 'gen_certs', and 'secret_changed'  to false"
   set_fact:
@@ -18,33 +13,53 @@
     gen_certs: false
     secret_changed: false
 
-- name: "Check_certs | Set 'gen_certs' to true"
-  set_fact:
-    gen_certs: true
-  when: "not {{ item.stat.exists }}"
-  run_once: true
-  with_items: "{{ kubecert_master.results }}"
-
-- name: "Check certs | check if a cert already exists"
+- name: "Check certs | check if a cert already exists on node"
   stat:
     path: "{{ kube_cert_dir }}/{{ item }}"
-  register: kubecert
+  register: kubecert_node
   with_items:
     - ca.pem
     - node-{{ inventory_hostname }}-key.pem
 
+- name: "Check_certs | Set 'gen_certs' to true"
+  set_fact:
+    gen_certs: true
+  when: "not item in kubecert_master.files|map(attribute='path') | list"
+  run_once: true
+  with_items: >-
+       ['{{ kube_cert_dir }}/ca.pem',
+       {% for host in groups['k8s-cluster'] %}
+       '{{ kube_cert_dir }}/node-{{ host }}-key.pem'
+       {% if not loop.last %}{{','}}{% endif %}
+       {% endfor %}]
+
+- name: "Check_certs | Set 'gen_node_certs' to true"
+  set_fact:
+    gen_node_certs: |-
+      {
+      {% set existing_certs = kubecert_master.files|map(attribute='path')|list|sort %}
+      {% for host in groups['k8s-cluster'] -%}
+        {% set host_cert = "%s/node-%s-key.pem"|format(kube_cert_dir, host) %}
+        {% if host_cert in existing_certs -%}
+        "{{ host }}": False,
+        {% else -%}
+        "{{ host }}": True,
+        {% endif -%}
+      {% endfor %}
+      }
+  run_once: true
+
+
 - name: "Check_certs | Set 'sync_certs' to true"
   set_fact:
     sync_certs: true
   when: >-
       {%- set certs = {'sync': False} -%}
-      {%- for host in groups['k8s-cluster'] %}
-        {% if host == inventory_hostname %}
-          {% if (not kubecert.results[0].stat.exists|default(False)) or
-              (not kubecert.results[1].stat.exists|default(False)) or
-              (kubecert.results[1].stat.checksum|default('') != kubecert_master.results[loop.index].stat.checksum|default('')) -%}
-            {%- set _ = certs.update({'sync': True}) -%}
-          {% endif %}
-        {% endif %}
-      {%- endfor -%}
+      {% if gen_node_certs[inventory_hostname] or    
+        (not kubecert_node.results[0].stat.exists|default(False)) or
+          (not kubecert_node.results[1].stat.exists|default(False)) or
+            (kubecert_node.results[1].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[1].stat.path)|first|map(attribute="checksum")|default('')) -%}
+              {%- set _ = certs.update({'sync': True}) -%}
+      {% endif %}
       {{ certs.sync }}
+
diff --git a/roles/kubernetes/secrets/tasks/gen_certs_script.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml
index f75a45d1a..4a9188065 100644
--- a/roles/kubernetes/secrets/tasks/gen_certs_script.yml
+++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml
@@ -40,12 +40,12 @@
   command: "{{ kube_script_dir }}/make-ssl.sh -f {{ kube_config_dir }}/openssl.conf -d {{ kube_cert_dir }}"
   environment:
     - MASTERS: "{% for m in groups['kube-master'] %}
-                  {% if hostvars[m].sync_certs|default(true) %}
+                  {% if gen_node_certs[m]|default(false) %}
                     {{ m }}
                   {% endif %}
                 {% endfor %}"
     - HOSTS: "{% for h in groups['k8s-cluster'] %}
-                {% if hostvars[h].sync_certs|default(true) %}
+                {% if gen_node_certs[h]|default(true) %}
                     {{ h }}
                 {% endif %}
               {% endfor %}"

From 68ea8ad437f96c3305c8bf07c408ad02d77ccd6c Mon Sep 17 00:00:00 2001
From: Vincent Schwarzer <vincent.schwarzer@yahoo.de>
Date: Fri, 3 Mar 2017 15:19:28 +0100
Subject: [PATCH 07/10] Added Missing AWS IAM Profiles and Policies

The AWS IAM profiles and policies required to run Kargo on AWS
are no longer hosted in the kubernetes main repo since kube-up got
deprecated. Hence we have to move the files into the kargo repository.
---
 contrib/aws_iam/kubernetes-master-policy.json | 27 +++++++++++
 contrib/aws_iam/kubernetes-master-role.json   | 10 +++++
 contrib/aws_iam/kubernetes-minion-policy.json | 45 +++++++++++++++++++
 contrib/aws_iam/kubernetes-minion-role.json   | 10 +++++
 docs/aws.md                                   |  2 +-
 5 files changed, 93 insertions(+), 1 deletion(-)
 create mode 100644 contrib/aws_iam/kubernetes-master-policy.json
 create mode 100644 contrib/aws_iam/kubernetes-master-role.json
 create mode 100644 contrib/aws_iam/kubernetes-minion-policy.json
 create mode 100644 contrib/aws_iam/kubernetes-minion-role.json

diff --git a/contrib/aws_iam/kubernetes-master-policy.json b/contrib/aws_iam/kubernetes-master-policy.json
new file mode 100644
index 000000000..e5cbaea80
--- /dev/null
+++ b/contrib/aws_iam/kubernetes-master-policy.json
@@ -0,0 +1,27 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["ec2:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["elasticloadbalancing:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["route53:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": "s3:*",
+      "Resource": [
+        "arn:aws:s3:::kubernetes-*"
+      ]
+    }
+  ]
+}
diff --git a/contrib/aws_iam/kubernetes-master-role.json b/contrib/aws_iam/kubernetes-master-role.json
new file mode 100644
index 000000000..66d5de1d5
--- /dev/null
+++ b/contrib/aws_iam/kubernetes-master-role.json
@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
diff --git a/contrib/aws_iam/kubernetes-minion-policy.json b/contrib/aws_iam/kubernetes-minion-policy.json
new file mode 100644
index 000000000..af81e98c8
--- /dev/null
+++ b/contrib/aws_iam/kubernetes-minion-policy.json
@@ -0,0 +1,45 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": "s3:*",
+      "Resource": [
+        "arn:aws:s3:::kubernetes-*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:Describe*",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:AttachVolume",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": "ec2:DetachVolume",
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["route53:*"],
+      "Resource": ["*"]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ecr:GetAuthorizationToken",
+        "ecr:BatchCheckLayerAvailability",
+        "ecr:GetDownloadUrlForLayer",
+        "ecr:GetRepositoryPolicy",
+        "ecr:DescribeRepositories",
+        "ecr:ListImages",
+        "ecr:BatchGetImage"
+      ],
+      "Resource": "*"
+    }
+  ]
+}
diff --git a/contrib/aws_iam/kubernetes-minion-role.json b/contrib/aws_iam/kubernetes-minion-role.json
new file mode 100644
index 000000000..66d5de1d5
--- /dev/null
+++ b/contrib/aws_iam/kubernetes-minion-role.json
@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": { "Service": "ec2.amazonaws.com"},
+      "Action": "sts:AssumeRole"
+    }
+  ]
+}
diff --git a/docs/aws.md b/docs/aws.md
index 429e77a54..b16b8d725 100644
--- a/docs/aws.md
+++ b/docs/aws.md
@@ -3,7 +3,7 @@ AWS
 
 To deploy kubespray on [AWS](https://aws.amazon.com/) uncomment the `cloud_provider` option in `group_vars/all.yml` and set it to `'aws'`.
 
-Prior to creating your instances, you **must** ensure that you have created IAM roles and policies for both "kubernetes-master" and "kubernetes-node". You can find the IAM policies [here](https://github.com/kubernetes/kubernetes/tree/master/cluster/aws/templates/iam). See the [IAM Documentation](https://aws.amazon.com/documentation/iam/) if guidance is needed on how to set these up. When you bring your instances online, associate them with the respective IAM role. Nodes that are only to be used for Etcd do not need a role.
+Prior to creating your instances, you **must** ensure that you have created IAM roles and policies for both "kubernetes-master" and "kubernetes-node". You can find the IAM policies [here](https://github.com/kubernetes-incubator/kargo/tree/master/contrib/aws_iam/). See the [IAM Documentation](https://aws.amazon.com/documentation/iam/) if guidance is needed on how to set these up. When you bring your instances online, associate them with the respective IAM role. Nodes that are only to be used for Etcd do not need a role.
 
 The next step is to make sure the hostnames in your `inventory` file are identical to your internal hostnames in AWS. This may look something like `ip-111-222-333-444.us-west-2.compute.internal`. You can then specify how Ansible connects to these instances with `ansible_ssh_host` and `ansible_ssh_user`.
 

From f247a75afde0a70ebd4d8f53b2c108efb32ccc31 Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Sat, 4 Mar 2017 00:30:37 +0400
Subject: [PATCH 08/10] Remove standalone etcd specific play, cleanup host mode

Now etcd role can optionally disable etcd cluster setup for faster
deployment when it is combined with etcd role.
---
 cluster.yml                       |  6 +++---
 roles/etcd/defaults/main.yml      |  3 +++
 roles/etcd/tasks/install_host.yml |  9 ---------
 roles/etcd/tasks/main.yml         | 18 +++++++++---------
 upgrade-cluster.yml               |  6 +++---
 5 files changed, 18 insertions(+), 24 deletions(-)
 delete mode 100644 roles/etcd/tasks/install_host.yml

diff --git a/cluster.yml b/cluster.yml
index f0c324174..01b033b2f 100644
--- a/cluster.yml
+++ b/cluster.yml
@@ -39,17 +39,17 @@
     - { role: kargo-defaults, when: "cert_management == 'vault'" }
     - { role: vault, tags: vault, vault_bootstrap: true, when: "cert_management == 'vault'" }
 
-- hosts: etcd:!k8s-cluster
+- hosts: etcd
   any_errors_fatal: true
   roles:
     - { role: kargo-defaults}
-    - { role: etcd, tags: etcd }
+    - { role: etcd, tags: etcd, etcd_cluster_setup: true }
 
 - hosts: k8s-cluster
   any_errors_fatal: true
   roles:
     - { role: kargo-defaults}
-    - { role: etcd, tags: etcd }
+    - { role: etcd, tags: etcd, etcd_cluster_setup: false }
 
 - hosts: etcd:k8s-cluster:vault
   any_errors_fatal: true
diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml
index 228228020..6326741b3 100644
--- a/roles/etcd/defaults/main.yml
+++ b/roles/etcd/defaults/main.yml
@@ -1,4 +1,7 @@
 ---
+# Set to false to only do certificate management
+etcd_cluster_setup: true
+
 etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"
 etcd_data_dir: "/var/lib/etcd"
 
diff --git a/roles/etcd/tasks/install_host.yml b/roles/etcd/tasks/install_host.yml
deleted file mode 100644
index 6f588a2f0..000000000
--- a/roles/etcd/tasks/install_host.yml
+++ /dev/null
@@ -1,9 +0,0 @@
----
-- name: Install | Copy etcd binary from downloaddir
-  command: rsync -piu "{{ etcd_bin_dir }}/etcd" "{{ bin_dir }}/etcd"
-  register: etcd_copy
-  changed_when: false
-
-- name: Install | Copy etcdctl binary from downloaddir
-  command: rsync -piu "{{ etcd_bin_dir }}/etcdctl" "{{ bin_dir }}/etcdctl"
-  changed_when: false
diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml
index 1af265736..02737ea31 100644
--- a/roles/etcd/tasks/main.yml
+++ b/roles/etcd/tasks/main.yml
@@ -1,5 +1,6 @@
 ---
 - include: pre_upgrade.yml
+  when: etcd_cluster_setup
   tags: etcd-pre-upgrade
 
 - include: check_certs.yml
@@ -27,19 +28,18 @@
   tags: upgrade
 
 - include: set_cluster_health.yml
-  when: is_etcd_master
+  when: is_etcd_master and etcd_cluster_setup
 
 - include: configure.yml
-  when: is_etcd_master
+  when: is_etcd_master and etcd_cluster_setup
 
 - include: refresh_config.yml
-  when: is_etcd_master
+  when: is_etcd_master and etcd_cluster_setup
 
-- name: Restart etcd if binary or certs changed
+- name: Restart etcd if certs changed
   command: /bin/true
   notify: restart etcd
-  when: etcd_deployment_type == "host" and etcd_copy.stdout_lines and is_etcd_master
-    or etcd_secret_changed|default(false)
+  when: is_etcd_master and etcd_secret_changed|default(false)
 
 # reload-systemd
 - meta: flush_handlers
@@ -49,13 +49,13 @@
     name: etcd
     state: started
     enabled: yes
-  when: is_etcd_master
+  when: is_etcd_master and etcd_cluster_setup
 
 # After etcd cluster is assembled, make sure that
 # initial state of the cluster is in `existing`
 # state insted of `new`.
 - include: set_cluster_health.yml
-  when: is_etcd_master
+  when: is_etcd_master and etcd_cluster_setup
 
 - include: refresh_config.yml
-  when: is_etcd_master
+  when: is_etcd_master and etcd_cluster_setup
diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml
index eea3b69aa..f4f48d543 100644
--- a/upgrade-cluster.yml
+++ b/upgrade-cluster.yml
@@ -39,17 +39,17 @@
     - { role: kargo-defaults, when: "cert_management == 'vault'" }
     - { role: vault, tags: vault, vault_bootstrap: true, when: "cert_management == 'vault'" }
 
-- hosts: etcd:!k8s-cluster
+- hosts: etcd
   any_errors_fatal: true
   roles:
     - { role: kargo-defaults}
-    - { role: etcd, tags: etcd }
+    - { role: etcd, tags: etcd, etcd_cluster_setup: true }
 
 - hosts: k8s-cluster
   any_errors_fatal: true
   roles:
     - { role: kargo-defaults}
-    - { role: etcd, tags: etcd }
+    - { role: etcd, tags: etcd, etcd_cluster_setup: false }
 
 - hosts: etcd:k8s-cluster:vault
   any_errors_fatal: true

From 7a3956173a9cab4063bc21d5bcef2d101d84ce12 Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Fri, 3 Mar 2017 16:33:00 +0300
Subject: [PATCH 09/10] Disable vault role properly on ansible 2.2.0

when condition does not seem to work correctly at playbook
level for ansible 2.2.0.
---
 roles/kubernetes/secrets/tasks/main.yml | 7 +------
 roles/vault/tasks/main.yml              | 4 ++--
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/roles/kubernetes/secrets/tasks/main.yml b/roles/kubernetes/secrets/tasks/main.yml
index ab2cb76b2..6da147170 100644
--- a/roles/kubernetes/secrets/tasks/main.yml
+++ b/roles/kubernetes/secrets/tasks/main.yml
@@ -71,8 +71,7 @@
   delegate_to: "{{groups['kube-master'][0]}}"
   when: gen_tokens|default(false)
 
-- include: gen_certs_script.yml
-  when: cert_management == "script"
+- include: "gen_certs_{{ cert_management }}.yml"
   tags: k8s-secrets
 
 - include: sync_kube_master_certs.yml
@@ -83,9 +82,5 @@
   when: cert_management == "vault" and inventory_hostname in groups['k8s-cluster']
   tags: k8s-secrets
 
-- include: gen_certs_vault.yml
-  when: cert_management == "vault"
-  tags: k8s-secrets
-
 - include: gen_tokens.yml
   tags: k8s-secrets
diff --git a/roles/vault/tasks/main.yml b/roles/vault/tasks/main.yml
index f7414b74f..4aef875ce 100644
--- a/roles/vault/tasks/main.yml
+++ b/roles/vault/tasks/main.yml
@@ -12,8 +12,8 @@
 
 ## Bootstrap
 - include: bootstrap/main.yml
-  when: vault_bootstrap | d()
+  when: cert_management == 'vault' and vault_bootstrap | d()
 
 ## Cluster
 - include: cluster/main.yml
-  when: not vault_bootstrap | d()
+  when: cert_management == 'vault' and not vault_bootstrap | d()

From 0c96b5d3fcbd53082f4e68bab092009cc804dd7a Mon Sep 17 00:00:00 2001
From: Brad Beam <brad.beam@b-rad.info>
Date: Mon, 6 Mar 2017 10:51:38 -0600
Subject: [PATCH 10/10] Removing cloud_provider tag to fix scenario where
 cloud_provider is not defined

---
 roles/kubernetes/preinstall/tasks/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml
index 4d89a73dd..e79d92751 100644
--- a/roles/kubernetes/preinstall/tasks/main.yml
+++ b/roles/kubernetes/preinstall/tasks/main.yml
@@ -70,7 +70,7 @@
 
 - include: "{{ cloud_provider }}-credential-check.yml"
   when: cloud_provider is defined and cloud_provider in [ 'openstack', 'azure', 'vsphere' ]
-  tags: [cloud-provider, "{{ cloud_provider }}", facts]
+  tags: [cloud-provider, facts]
 
 - name: Create cni directories
   file:
@@ -182,7 +182,7 @@
     group: "{{ kube_cert_group }}"
     mode: 0640
   when: inventory_hostname in groups['k8s-cluster'] and cloud_provider is defined and cloud_provider in [ 'openstack', 'azure', 'vsphere' ]
-  tags: [cloud-provider, "{{ cloud_provider }}"]
+  tags: [cloud-provider]
 
 - include: etchosts.yml
   tags: [bootstrap-os, etchosts]