Adding the Vault role

.gitignore

@@ -12,3 +12,4 @@ temp
 *.tfstate.backup
 **/*.sw[pon]
 /ssh-bastion.conf
+**/*.sw[pon]

cluster.yml

@@ -30,15 +30,24 @@
     - { role: docker, tags: docker }
     - { role: rkt, tags: rkt, when: "'rkt' in [ etcd_deployment_type, kubelet_deployment_type ]" }
 
-- hosts: etcd:!k8s-cluster
+- hosts: all
+  any_errors_fatal: true
+  roles:
+    - { role: vault, tags: vault, vault_bootstrap: true, when: "cert_management == 'vault'" }
+
+- hosts: etcd:k8s-cluster
   any_errors_fatal: true
   roles:
     - { role: etcd, tags: etcd }
 
+- hosts: etcd:k8s-cluster:vault
+  any_errors_fatal: true
+  roles:
+    - { role: vault, tags: vault, when: "cert_management == 'vault'"}
+
 - hosts: k8s-cluster
   any_errors_fatal: true
   roles:
-    - { role: etcd, tags: etcd }
     - { role: kubernetes/node, tags: node }
     - { role: network_plugin, tags: network }
 

inventory/group_vars/all.yml

@@ -206,3 +206,9 @@ etcd_deployment_type: docker
 kubelet_deployment_type: docker
 
 efk_enabled: false
+
+## Certificate Management
+## This setting determines whether certs are generated via scripts or whether a
+## cluster of Hashicorp's Vault is started to issue certificates (using etcd
+## as a backend). Options are "script" or "vault"
+cert_management: script

roles/adduser/defaults/main.yml

@@ -14,6 +14,14 @@ addusers:
     system: yes
     group: "{{ kube_cert_group }}"
     createhome: no
+  vault:
+    comment: "Hashicorp Vault user"
+    createhome: no
+    name: vault
+    shell: /sbin/nologin
+    system: yes
+
+
 
 adduser:
   name: "{{ user.name }}"

roles/download/defaults/main.yml

@@ -26,6 +26,7 @@ calico_cni_version: "v1.5.5"
 weave_version: 1.8.2
 flannel_version: v0.6.2
 pod_infra_version: 3.0
+vault_version: 0.6.3
 
 # Download URL's
 etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd"

roles/etcd/defaults/main.yml

@@ -2,6 +2,7 @@
 etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"
 
 etcd_config_dir: /etc/ssl/etcd
+# Role vault.boostrap has an implicit requirement on this var. It should be set at a higher level (inventory+)
 etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
 etcd_cert_group: root
 

roles/vault/defaults/main.yml

@@ -0,0 +1,65 @@
+---
+
+vault_bootstrap: false
+vault_ca_options:
+  common_name: kube-cluster-ca
+  format: pem
+  ttl: 87600h
+vault_cert_dir: "{{ vault_config_dir }}/ssl"
+vault_client_headers:
+  Accept: "application/json"
+  Content-Type: "application/json"
+vault_config:
+  backend:
+    etcd:
+      address: "https://{{ hostvars[groups.etcd[0]]['ansible_default_ipv4']['address'] }}:2379"
+      ha_enabled: "true"
+      redirect_addr: "https://{{ ansible_default_ipv4.address }}:{{ vault_port }}"
+      tls_ca_file: "{{ vault_cert_dir }}/ca.pem"
+  cluster_name: "kubernetes-vault"
+  default_lease_ttl: "{{ vault_default_lease_ttl }}"
+  listener:
+    tcp:
+      address: "0.0.0.0:{{ vault_port }}"
+      tls_cert_file: "{{ vault_cert_dir }}/api.pem"
+      tls_key_file: "{{ vault_cert_dir }}/api-key.pem"
+  max_lease_ttl: 720h
+vault_config_dir: /etc/vault
+vault_container_name: kube-hashicorp-vault
+vault_default_lease_ttl: 720h
+vault_default_role_permissions:
+  allow_any_name: true
+vault_deployment_type: docker
+vault_etcd_needs_gen: false
+vault_etcd_sync_hosts: []
+vault_max_lease_ttl: 87600h 
+vault_needs_gen: false
+vault_port: 8200
+vault_secret_shares: 1
+vault_secret_threshold: 1
+vault_secrets_dir: "{{ vault_config_dir }}/secrets"
+vault_temp_config:
+  default_lease_ttl: "{{ vault_default_lease_ttl }}"
+  backend:
+    file:
+      path: /vault/file
+  listener:
+    tcp:
+      address: "0.0.0.0:{{ vault_temp_port }}"
+      tls_disable: "true"
+vault_temp_port: 8201
+
+# This should be set higher up, but setting defaults here to avoid issues
+etcd_cert_dir: /etc/ssl/etcd/ssl
+kube_cert_dir: /etc/kubernetes/ssl
+
+# Sync cert defaults (should be role, once include_role is fixed)
+sync_file: ''
+sync_file_dir: ''
+sync_file_host_count: 0
+sync_file_is_cert: false
+sync_file_key_path: ''
+sync_file_key_srcs: []
+sync_file_path: ''
+sync_file_results: []
+sync_file_srcs: []

roles/vault/meta/main.yml

@@ -0,0 +1,6 @@
+---
+# Implicit requirement on sync_cert role (include_role used in tasks)
+
+dependencies:
+  - role: download
+    file: "{{ downloads.vault }}"

roles/vault/tasks/bootstrap/ca_trust.yml

@@ -0,0 +1,32 @@
+---
+
+- name: trust_ca | pull CA from cert from groups.vault|first
+  command: "cat {{ vault_cert_dir }}/ca.pem"
+  register: vault_cert_file_cat
+  when: inventory_hostname == groups.vault|first
+
+# This part is mostly stolen from the etcd role
+- name: trust_ca | target ca-certificate store file
+  set_fact:
+    ca_cert_path: >-
+      {% if ansible_os_family == "Debian" -%}
+      /usr/local/share/ca-certificates/kube-cluster-ca.crt
+      {%- elif ansible_os_family == "RedHat" -%}
+      /etc/pki/ca-trust/source/anchors/kube-cluster-ca.crt
+      {%- elif ansible_os_family == "CoreOS" -%}
+      /etc/ssl/certs/kube-cluster-ca.pem
+      {%- endif %}
+
+- name: trust_ca | add CA to trusted CA dir
+  copy:
+    content: "{{ hostvars[groups.vault|first]['vault_cert_file_cat']['stdout'] }}"
+    dest: "{{ ca_cert_path }}"
+  register: vault_ca_cert
+
+- name: trust_ca | update ca-certificates (Debian/Ubuntu/CoreOS)
+  command: update-ca-certificates
+  when: vault_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"]
+
+- name: trust_ca | update ca-certificates (RedHat)
+  command: update-ca-trust extract
+  when: vault_ca_cert.changed and ansible_os_family == "RedHat"

roles/vault/tasks/bootstrap/gen_etcd_certs.yml

@@ -0,0 +1,29 @@
+---
+
+- name: bootstrap/gen_etcd_certs | Add the etcd role
+  uri:
+    url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}/v1/pki/roles/etcd"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: POST
+    body_format: json
+    body:
+      allow_any_name: true
+    status_code: 204
+  when: inventory_hostname == groups.etcd|first
+
+- include: ../gen_cert.yml
+  vars:
+    gen_cert_alt_names: "{{ groups.etcd | join(',') }},localhost"
+    gen_cert_copy_ca: "{{ true if item == vault_etcd_certs_needed|first else false }}"
+    gen_cert_hosts: "{{ groups.etcd }}"
+    gen_cert_ip_sans: >-
+        {%- for host in groups.etcd  -%}
+        {{ hostvars[host]["ansible_default_ipv4"]["address"] }}
+        {%- if not loop.last -%},{%- endif -%}
+        {%- endfor -%}
+        ,127.0.0.1,::1
+    gen_cert_path: "{{ item }}"
+    gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    gen_cert_vault_role: etcd
+    gen_cert_vault_url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}"
+  with_items: "{{ vault_etcd_certs_needed|default([]) }}"

roles/vault/tasks/bootstrap/gen_etcd_node_certs.yml

@@ -0,0 +1,29 @@
+---
+
+- name: bootstrap/gen_etcd_node_certs | Add the etcd role
+  uri:
+    url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}/v1/pki/roles/etcd"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: POST
+    body_format: json
+    body:
+      allow_any_name: true
+    status_code: 204
+  when: inventory_hostname == groups["k8s-cluster"]|first
+
+- include: ../gen_cert.yml
+  vars:
+    gen_cert_alt_names: "{{ groups['k8s-cluster'] | union(groups.etcd) | join(',') }},localhost"
+    gen_cert_copy_ca: "{{ true if item == vault_etcd_node_certs_needed|first else false }}"
+    gen_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.etcd) }}"
+    gen_cert_ip_sans: >-
+        {%- for host in groups["k8s-cluster"] | union(groups.etcd) -%}
+        {{ hostvars[host]["ansible_default_ipv4"]["address"] }}
+        {%- if not loop.last -%},{%- endif -%}
+        {%- endfor -%}
+        ,127.0.0.1,::1
+    gen_cert_path: "{{ item }}"
+    gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    gen_cert_vault_role: etcd
+    gen_cert_vault_url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}"
+  with_items: "{{ vault_etcd_node_certs_needed|default([]) }}"

roles/vault/tasks/bootstrap/gen_vault_certs.yml

@@ -0,0 +1,66 @@
+---
+
+- name: bootstrap/gen_vault_certs | Ensure vault_cert_dir exists
+  file:
+    path: "{{ vault_cert_dir }}"
+    state: directory
+
+- name: bootstrap/gen_vault_certs | Generate Root CA in vault-temp
+  uri:
+    url: "http://localhost:{{ vault_temp_port }}/v1/pki/root/generate/exported"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body: "{{ vault_ca_options }}"
+  register: vault_ca_gen
+  when: inventory_hostname == groups.vault|first and vault_ca_cert_needed
+
+- name: bootstrap/gen_vault_certs | Set facts for ca cert and key
+  set_fact:
+    vault_ca_cert: "{{ vault_ca_gen.json.data.certificate }}"
+    vault_ca_key: "{{ vault_ca_gen.json.data.private_key }}"
+  when: inventory_hostname == groups.vault|first and vault_ca_cert_needed
+
+- name: bootstrap/gen_vault_certs | Set cert and key facts for all hosts other than groups.vault|first
+  set_fact:
+    vault_ca_cert: "{{ hostvars[groups.vault|first]['vault_ca_cert'] }}"
+    vault_ca_key: "{{ hostvars[groups.vault|first]['vault_ca_key'] }}"
+  when: inventory_hostname != groups.vault|first and vault_ca_cert_needed
+
+- name: bootstrap/gen_vault_certs | Copy root CA cert locally
+  copy:
+    content: "{{ vault_ca_cert }}"
+    dest: "{{ vault_cert_dir }}/ca.pem"
+  when: vault_ca_cert_needed
+
+- name: bootstrap/gen_vault_certs | Copy root CA key locally
+  copy:
+    content: "{{vault_ca_key}}"
+    dest: "{{vault_cert_dir}}/ca-key.pem"
+  when: vault_ca_cert_needed
+
+- name: boostrap/gen_vault_certs | Add the vault role
+  uri:
+    url: "http://localhost:{{ vault_temp_port }}/v1/pki/roles/vault"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body: "{{ vault_default_role_permissions }}"
+    status_code: 204
+  when: inventory_hostname == groups.vault|first and vault_api_cert_needed
+
+- include: ../gen_cert.yml
+  vars:
+    gen_cert_alt_names: "{{ groups.vault | join(',') }},localhost"
+    gen_cert_hosts: "{{ groups.vault }}"
+    gen_cert_ip_sans: >-
+        {%- for host in groups.vault -%}
+        {{ hostvars[host]["ansible_default_ipv4"]["address"] }}
+        {%- if not loop.last -%},{%- endif -%}
+        {%- endfor -%}
+        ,127.0.0.1,::1
+    gen_cert_path: "{{ vault_cert_dir }}/api.pem"
+    gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    gen_cert_vault_role: vault
+    gen_cert_vault_url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}"
+  when: vault_api_cert_needed

roles/vault/tasks/bootstrap/main.yml

@@ -0,0 +1,60 @@
+---
+
+## Sync Certs
+
+- include: bootstrap/sync_vault_certs.yml
+  when: inventory_hostname in groups.vault
+
+- include: bootstrap/sync_etcd_certs.yml
+  when: inventory_hostname in groups.etcd
+
+- include: bootstrap/sync_etcd_node_certs.yml
+  when: inventory_hostname in groups["k8s-cluster"] | union(groups.etcd)
+
+## Generate Certs
+
+# Start a temporary instance of Vault
+- include: bootstrap/start_vault_temp.yml
+  when: >-
+        ( hostvars[groups.etcd|first].get("vault_etcd_certs_needed", [])|length > 0 or
+        hostvars[groups.etcd|first].get("vault_etcd_node_certs_needed", [])|length > 0 or
+        hostvars[groups.vault|first]["vault_ca_cert_needed"] ) and
+        inventory_hostname == groups.vault|first
+
+# Generate root CA certs for Vault if none exist
+- include: bootstrap/gen_vault_certs.yml
+  when: >-
+        ( hostvars[groups.vault|first]["vault_ca_cert_needed"] or
+        hostvars[groups.vault|first]["vault_api_cert_needed"] ) and
+        inventory_hostname in groups.vault
+
+# Change vault-temp's issuing CA to use existing ca.pem/ca-key.pem
+- include: config_ca.yml
+  vars:
+    vault_url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}"
+  when: >-
+        ( hostvars[groups.etcd|first].get("vault_etcd_certs_needed", [])|length > 0 or
+        hostvars[groups["k8s-cluster"]|first].get("vault_etcd_node_certs_needed", [])|length > 0 or
+        hostvars[groups.vault|first]["vault_api_cert_needed"] ) and
+        not hostvars[groups.vault|first]["vault_ca_cert_needed"] and
+        inventory_hostname == groups.vault|first
+
+# Generate etcd certs for etcd cluster members
+- include: bootstrap/gen_etcd_certs.yml
+  when: >- 
+        hostvars[groups.etcd|first].get("vault_etcd_certs_needed", [])|length > 0 and
+        inventory_hostname in groups.etcd
+
+# Generate etcd node certs for all k8s-cluster
+- include: bootstrap/gen_etcd_node_certs.yml
+  when: >-
+        hostvars[groups["k8s-cluster"]|first].get("vault_etcd_node_certs_needed", [])|length > 0 and
+        inventory_hostname in groups["k8s-cluster"] | union(groups.etcd)
+
+# Stop temporary vault
+- include: bootstrap/stop_vault_temp.yml
+  when: >-
+        inventory_hostname == groups.vault|first and
+        hostvars[groups.vault|first]["vault_temp_start"]|succeeded
+
+- include: ca_trust.yml

roles/vault/tasks/bootstrap/start_vault_temp.yml

@@ -0,0 +1,55 @@
+---
+
+- name: boostrap/start_vault_temp | Ensure vault-temp isn't already running
+  shell: if docker rm -f vault-temp 2>&1 1>/dev/null;then echo true;else echo false;fi
+  register: vault_temp_stop_check
+  changed_when: "{{ 'true' in vault_temp_stop_check.stdout }}"
+
+- name: bootstrap/start_vault_temp | Start single node Vault with file backend
+  command: >
+           docker run -d --cap-add=IPC_LOCK --name vault-temp -p {{ vault_temp_port }}:{{ vault_temp_port }}
+           -e 'VAULT_LOCAL_CONFIG={{ vault_temp_config|to_json }}'
+           -v /etc/vault:/etc/vault
+           {{ vault_image_repo }}:{{ vault_version }} server
+  register: vault_temp_start
+
+- name: bootstrap/start_vault_temp | Initialize vault-temp
+  uri:
+    url: "http://localhost:{{ vault_temp_port }}/v1/sys/init"
+    headers: "{{ vault_client_headers }}"
+    method: PUT
+    body_format: json
+    body:
+      secret_shares: 1
+      secret_threshold: 1
+  register: vault_temp_init
+
+# NOTE: vault_headers and vault_url are used by subsequent gen_cert calls
+- name: bootstrap/start_vault_temp | Set needed vault facts
+  set_fact:
+    vault_temp_unseal_keys: "{{ vault_temp_init.json['keys'] }}"
+    vault_temp_root_token: "{{ vault_temp_init.json.root_token }}"
+    vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_temp_init.json.root_token}) }}"
+
+- name: bootstrap/start_vault_temp | Unseal vault-temp
+  uri:
+    url: "http://localhost:{{ vault_temp_port }}/v1/sys/unseal"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body:
+      key: "{{ item }}"
+  with_items: "{{ vault_temp_unseal_keys|default([]) }}"
+
+- name: bootstrap/start_vault_temp | Create new PKI mount
+  uri:
+    url: "http://localhost:{{ vault_temp_port }}/v1/sys/mounts/pki"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body:
+      config:
+        default_lease_ttl: "{{ vault_default_lease_ttl }}"
+        max_lease_ttl: "{{ vault_max_lease_ttl }}"
+      type: pki
+    status_code: 204

roles/vault/tasks/bootstrap/stop_vault_temp.yml

@@ -0,0 +1,4 @@
+---
+
+- name: stop vault-temp container
+  command: docker stop vault-temp

roles/vault/tasks/bootstrap/sync_etcd_certs.yml

@@ -0,0 +1,38 @@
+---
+
+- name: bootstrap/sync_etcd_certs | Create list of certs needing creation
+  set_fact: 
+    vault_etcd_cert_list: >-
+        {{ vault_etcd_cert_list|default([]) +  [
+        "admin-" + item + ".pem",
+        "member-" + item + ".pem"
+        ] }}
+  with_items: "{{ groups.etcd }}"
+
+- include: ../sync_file.yml
+  vars: 
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ groups.etcd }}"
+    sync_file_is_cert: true
+  with_items: "{{ vault_etcd_cert_list|default([]) }}"
+
+- name: bootstrap/sync_etcd_certs | Set facts for etcd sync_file results
+  set_fact:
+    vault_etcd_certs_needed: "{{ vault_etcd_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results }}"
+  when: item.no_srcs|bool
+
+- name: bootstrap/sync_etcd_certs | Unset sync_file_results after etcd certs sync
+  set_fact:
+    sync_file_results: []
+
+- include: ../sync_file.yml
+  vars:
+    sync_file: ca.pem
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ groups.etcd }}"
+
+- name: bootstrap/sync_etcd_certs | Unset sync_file_results after ca.pem sync
+  set_fact:
+    sync_file_results: []

roles/vault/tasks/bootstrap/sync_etcd_node_certs.yml

@@ -0,0 +1,34 @@
+---
+
+- name: bootstrap/sync_etcd_node_certs | Create list of certs needing creation
+  set_fact: 
+    vault_etcd_node_cert_list: "{{ vault_etcd_node_cert_list|default([]) +  ['node-' + item + '.pem'] }}"
+  with_items: "{{ groups['k8s-cluster'] | union(groups.etcd) }}"
+
+- include: ../sync_file.yml
+  vars: 
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ groups['k8s-cluster'] | union(groups.etcd) }}"
+    sync_file_is_cert: true
+  with_items: "{{ vault_etcd_node_cert_list|default([]) }}"
+
+- name: bootstrap/sync_etcd_node_certs | Set facts for etcd sync_file results
+  set_fact:
+    vault_etcd_node_certs_needed: "{{ vault_etcd_node_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results }}"
+  when: item.no_srcs|bool
+
+- name: bootstrap/sync_etcd_node_certs | Unset sync_file_results after etcd node certs
+  set_fact:
+    sync_file_results: []
+
+- include: ../sync_file.yml
+  vars: 
+    sync_file: ca.pem
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ groups['k8s-cluster']| union(groups.etcd) }}"
+
+- name: bootstrap/sync_etcd_node_certs | Unset sync_file_results after ca.pem
+  set_fact:
+    sync_file_results: []

roles/vault/tasks/bootstrap/sync_vault_certs.yml

@@ -0,0 +1,32 @@
+---
+
+- include: ../sync_file.yml
+  vars:
+    sync_file: "ca.pem"
+    sync_file_dir: "{{ vault_cert_dir }}"
+    sync_file_hosts: "{{ groups.vault }}"
+    sync_file_is_cert: true
+
+- name: "bootstrap/sync_vault_certs | Set facts for vault sync_file results"
+  set_fact:
+    vault_ca_cert_needed: "{{ true if sync_file_results|length > 0 else false }}"
+
+- name: bootstrap/sync_vault_certs | Unset sync_file_results after ca.pem sync
+  set_fact:
+    sync_file_results: []
+
+- include: ../sync_file.yml
+  vars:
+    sync_file: "api.pem"
+    sync_file_dir: "{{ vault_cert_dir }}"
+    sync_file_hosts: "{{ groups.vault }}"
+    sync_file_is_cert: true
+
+- name: bootstrap/sync_vault_certs | Set fact if Vault's API cert is needed
+  set_fact:
+    vault_api_cert_needed: "{{ true if sync_file_results|length > 0 else false }}"
+
+- name: bootstrap/sync_vault_certs | Unset sync_file_results after api.pem sync
+  set_fact:
+    sync_file_results: []
+

roles/vault/tasks/check_vault.yml

@@ -0,0 +1,77 @@
+---
+
+# Check if vault is reachable on the localhost
+- name: check_vault | Attempt to pull local vault health
+  uri:
+    url: "https://localhost:{{ vault_port }}/v1/sys/health"
+    headers: "{{ vault_client_headers }}"
+    validate_certs: no
+  ignore_errors: true
+  register: vault_local_service_health
+
+- name: check_vault | Set facts about local Vault health
+  set_fact:
+    vault_is_running: "{{ vault_local_service_health|succeeded }}"
+    vault_is_initialized: "{{ vault_local_service_health.get('json', {}).get('initialized', false) }}"
+    vault_is_sealed: "{{ vault_local_service_health.get('json', {}).get('sealed', true) }}"
+    vault_in_standby: "{{ vault_local_service_health.get('json', {}).get('standby', true) }}"
+    vault_run_version: "{{ vault_local_service_health.get('json', {}).get('version', '') }}"
+
+- name: check_vault | Set fact about the Vault cluster's initialization state
+  set_fact:
+    vault_cluster_is_initialized: "{{ vault_is_initialized or hostvars[item]['vault_is_initialized'] }}"
+  with_items: "{{ groups.vault }}"
+
+- name: check_vault | Set fact about the Vault Cluster's available hosts
+  set_fact:
+    vault_available_hosts: "{{ vault_available_hosts|default([]) + [item] }}"
+  with_items: "{{ groups.vault }}"
+  when: "hostvars[item]['vault_is_running'] and not hostvars[item]['vault_is_sealed']"
+
+- include: sync_file.yml
+  vars:
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ vault_secrets_dir }}"
+    sync_file_hosts: "{{ groups.vault }}"
+  with_items:
+    - root_token
+    - unseal_keys
+
+# Logic is hard to follow on this one, probably need to simplify somehow
+- name: "check_vault | Set fact based on sync_file_results"
+  set_fact:
+    vault_secrets_available: "{{ vault_secrets_available|default(true) and not item.no_srcs }}"
+  with_items: "{{ sync_file_results }}"
+
+- name: "check_vault | Reset sync_file_results to avoid variable bleed"
+  set_fact:
+    sync_file_results: []
+
+- name: "check_vault | Print out warning message if secrets are not available"
+  pause:
+    prompt: >
+         Vault orchestration may not be able to proceed. The Vault cluster is initialzed, but
+         'root_token' or 'unseal_keys' were not found in {{ vault_secrets_dir }}. These are
+         needed for many orchestration steps.
+  when: vault_cluster_is_initialized and not vault_secrets_available
+
+- name: "check_vault | Cat root_token from a vault host"
+  command: "cat {{ vault_secrets_dir }}/root_token"
+  register: vault_root_token_cat
+  when: vault_secrets_available and inventory_hostname == groups.vault|first
+
+- name: "check_vault | Cat unseal_keys from a vault host"
+  command: "cat {{ vault_secrets_dir }}/unseal_keys"
+  register: vault_unseal_keys_cat
+  when: vault_secrets_available and inventory_hostname == groups.vault|first
+
+- name: "check_vault | Set needed facts for Vault API interaction when Vault is already running"
+  set_fact:
+    vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token_cat']['stdout'] }}"
+    vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys_cat']['stdout_lines'] }}"
+  when: vault_secrets_available
+
+- name: "check-vault | Update vault_headers if we have the root_token"
+  set_fact:
+    vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token}) }}"
+  when: vault_secrets_available

roles/vault/tasks/cluster/docker.yml

@@ -0,0 +1,25 @@
+---
+
+- name: docker | Check on state of docker instance
+  command: "docker inspect {{ vault_container_name }}"
+  ignore_errors: true
+  register: vault_container_inspect
+
+- name: docker | Set fact on container status
+  set_fact:
+      vault_container_inspect_json: "{{ vault_container_inspect.stdout|from_json }}"
+  when: vault_container_inspect|succeeded
+
+# Not sure if State.Running is the best check here...
+- name: docker | Remove old container if it's not currently running
+  command: "docker rm {{ vault_container_name }}"
+  when: vault_container_inspect|succeeded and not vault_container_inspect_json[0]["State"]["Running"]|bool
+
+- name: docker | Start a new Vault instance
+  command: >
+           docker run -d --cap-add=IPC_LOCK --name {{vault_container_name}} -p {{vault_port}}:{{vault_port}}
+           -e 'VAULT_LOCAL_CONFIG={{ vault_config|to_json }}'
+           -v /etc/vault:/etc/vault
+           {{vault_image_repo}}:{{vault_version}} server
+  register: vault_docker_start
+  when: vault_container_inspect|failed or not vault_container_inspect_json[0]["State"]["Running"]|bool

roles/vault/tasks/cluster/gen_kube_master_certs.yml

@@ -0,0 +1,33 @@
+---
+
+- name: "cluster/gen_kube_node_certs | Ensure kube_cert_dir exists"
+  file:
+    path: "{{ kube_cert_dir }}" 
+    state: directory
+
+- name: gen_kube_master_certs | Add the kube role
+  uri:
+    url: "https://{{ hostvars[groups.vault|first]['vault_leader'] }}:{{ vault_port }}/v1/pki/roles/kubernetes"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: POST
+    body_format: json
+    body: "{{ vault_default_role_permissions }}"
+    status_code: 204
+  when: inventory_hostname == groups["kube-master"]|first
+
+- include: ../gen_cert.yml
+  vars:
+    gen_cert_alt_names: "{{ groups['kube-master'] | join(',') }},localhost"
+    gen_cert_copy_ca: "{{ true if item == vault_kube_master_certs_needed|first else false }}"
+    gen_cert_hosts: "{{ groups['kube-master'] }}"
+    gen_cert_ip_sans: >-
+        {%- for host in groups["kube-master"] -%}
+        {{ hostvars[host]["ansible_default_ipv4"]["address"] }}
+        {%- if not loop.last -%},{%- endif -%}
+        {%- endfor -%}
+        ,127.0.0.1,::1
+    gen_cert_path: "{{ item }}"
+    gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    gen_cert_vault_role: kubernetes
+    gen_cert_vault_url: "https://{{ hostvars[groups.vault|first]['vault_leader'] }}:{{ vault_port }}"
+  with_items: "{{ vault_kube_master_certs_needed|default([]) }}"

roles/vault/tasks/cluster/gen_kube_node_certs.yml

@@ -0,0 +1,33 @@
+---
+
+- name: "cluster/gen_kube_node_certs | Ensure kube_cert_dir exists"
+  file:
+    path: "{{ kube_cert_dir }}" 
+    state: directory
+
+- name: "cluster/gen_kube_node_certs | Add the kubernetes role"
+  uri:
+    url: "https://{{ hostvars[groups.vault|first]['vault_leader'] }}:{{ vault_port }}/v1/pki/roles/kubernetes"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: POST
+    body_format: json
+    body: "{{ vault_default_role_permissions }}"
+    status_code: 204
+  when: inventory_hostname == groups["k8s-cluster"]|first
+
+- include: ../gen_cert.yml
+  vars:
+    gen_cert_alt_names: "{{ groups['k8s-cluster'] | join(',') }},localhost"
+    gen_cert_copy_ca: "{{ true if item == vault_kube_node_certs_needed|first else false }}"
+    gen_cert_hosts: "{{ groups['k8s-cluster'] }}"
+    gen_cert_ip_sans: >-
+        {%- for host in groups["k8s-cluster"] -%}
+        {{ hostvars[host]["ansible_default_ipv4"]["address"] }}
+        {%- if not loop.last -%},{%- endif -%}
+        {%- endfor -%}
+        ,127.0.0.1,::1
+    gen_cert_path: "{{ item }}"
+    gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    gen_cert_vault_role: kubernetes
+    gen_cert_vault_url: "https://{{ hostvars[groups.vault|first]['vault_leader'] }}:{{ vault_port }}"
+  with_items: "{{ vault_kube_node_certs_needed|default([]) }}"

roles/vault/tasks/cluster/init.yml

@@ -0,0 +1,49 @@
+---
+
+- name: cluster/init | Initialize Vault
+  uri:
+    url: "https://{{ groups.vault|first }}:{{ vault_port }}/v1/sys/init"
+    headers: "{{ vault_client_headers }}"
+    method: POST
+    body_format: json
+    body:
+      secret_shares: "{{ vault_secret_shares }}"
+      secret_threshold: "{{ vault_secret_threshold }}"
+    validate_certs: false
+  register: vault_init_result
+  when: not vault_cluster_is_initialized and inventory_hostname == groups.vault|first
+
+- name: cluster/init | Set facts on the results of the initialization
+  set_fact:
+    vault_unseal_keys: "{{ vault_init_result.json['keys'] }}"
+    vault_root_token: "{{ vault_init_result.json.root_token }}"
+    vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_init_result.json.root_token}) }}"
+  when: not vault_cluster_is_initialized and inventory_hostname == groups.vault|first
+
+- name: cluster/init | Ensure all hosts have these facts
+  set_fact:
+    vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys'] }}"
+    vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token'] }}"
+  when: not vault_cluster_is_initialized and inventory_hostname != groups.vault|first
+
+- name: cluster/init | Ensure the vault_secrets_dir exists
+  file:
+    path: "{{ vault_secrets_dir }}"
+    state: directory
+
+- name: cluster/init | Ensure all in groups.vault have the unseal_keys locally
+  copy:
+    content: "{{ vault_unseal_keys|join('\n') }}"
+    dest: "{{ vault_secrets_dir }}/unseal_keys"
+  when: not vault_cluster_is_initialized
+
+- name: cluster/init | Ensure all in groups.vault have the root_token locally
+  copy:
+    content: "{{ vault_root_token }}"
+    dest: "{{ vault_secrets_dir }}/root_token"
+  when: not vault_cluster_is_initialized
+
+- name: cluster/init | Ensure vault_headers and vault statuses are updated
+  set_fact:
+    vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token})}}"
+    vault_cluster_is_initialized: true

roles/vault/tasks/cluster/main.yml

@@ -0,0 +1,30 @@
+---
+
+## Vault Cluster Setup
+
+- include: docker.yml
+  when: inventory_hostname in groups.vault and vault_deployment_type == "docker"
+- include: init.yml
+  when: inventory_hostname in groups.vault
+- include: unseal.yml
+  when: inventory_hostname in groups.vault
+- include: pki_mount.yml
+  when: 'inventory_hostname == hostvars[groups.vault|first]["vault_leader"]'
+- include: config_ca.yml
+  vars:
+    vault_url: "https://{{ vault_leader }}:{{ vault_port }}"
+  when: 'inventory_hostname == hostvars[groups.vault|first]["vault_leader"]'
+
+## Sync Kubernetes Certs
+
+- include: sync_kube_master_certs.yml
+  when: inventory_hostname in groups["kube-master"]
+- include: sync_kube_node_certs.yml
+  when: inventory_hostname in groups["k8s-cluster"]
+
+## Generate Kubernetes Certs
+
+- include: gen_kube_master_certs.yml
+  when: inventory_hostname in groups["kube-master"]
+- include: gen_kube_node_certs.yml
+  when: inventory_hostname in groups["k8s-cluster"]

roles/vault/tasks/cluster/pki_mount.yml

@@ -0,0 +1,23 @@
+---
+
+- name: cluster/pki_mount | Test if default PKI mount exists
+  uri:
+    url: "https://localhost:{{ vault_port }}/v1/sys/mounts/pki/tune"
+    headers: "{{ vault_headers }}"
+    validate_certs: false
+  ignore_errors: true
+  register: vault_pki_mount_check
+
+- name: cluster/pki_mount | Mount default PKI mount if needed
+  uri:
+    url: "https://localhost:{{ vault_port }}/v1/sys/mounts/pki"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body:
+      config:
+        default_lease_ttl: "{{ vault_default_lease_ttl }}"
+        max_lease_ttl: "{{ vault_max_lease_ttl }}"
+      type: pki
+    status_code: 204
+  when: vault_pki_mount_check | failed

roles/vault/tasks/cluster/sync_kube_master_certs.yml

@@ -0,0 +1,38 @@
+---
+
+- name: cluster/sync_kube_master_certs | Create list of needed certs
+  set_fact:
+    vault_kube_master_cert_list: >-
+        {{  vault_kube_master_cert_list|default([]) + [
+        "admin-" + item + ".pem",
+        "apiserver-" + item + ".pem"
+        ] }}
+  with_items: "{{ groups['kube-master'] }}"
+
+- include: ../sync_file.yml
+  vars: 
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_hosts: "{{ groups['kube-master'] }}"
+    sync_file_is_cert: true
+  with_items: "{{ vault_kube_master_cert_list|default([]) }}"
+
+- name: cluster/sync_kube_master_certs | Set facts for kube-master sync_file results
+  set_fact:
+    vault_kube_master_certs_needed: "{{ vault_kube_master_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results }}"
+  when: item.no_srcs|bool
+
+- name: cluster/sync_kube_master_certs | Unset sync_file_results after kube master certs
+  set_fact:
+    sync_file_results: []
+
+- include: ../sync_file.yml
+  vars: 
+    sync_file: ca.pem
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_hosts: "{{ groups['kube-master'] }}"
+
+- name: cluster/sync_kube_master_certs | Unset sync_file_results after ca.pem
+  set_fact:
+    sync_file_results: []

roles/vault/tasks/cluster/sync_kube_node_certs.yml

@@ -0,0 +1,34 @@
+---
+
+- name: cluster/sync_kube_node_certs | Create list of needed certs
+  set_fact:
+    vault_kube_node_cert_list: "{{ vault_kube_node_cert_list|default([]) + ['node-' + item + '.pem'] }}"
+  with_items: "{{ groups['k8s-cluster'] }}"
+
+- include: ../sync_file.yml
+  vars: 
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_hosts: "{{ groups['k8s-cluster'] }}"
+    sync_file_is_cert: true
+  with_items: "{{ vault_kube_node_cert_list|default([]) }}"
+
+- name: cluster/sync_kube_node_certs | Set facts for kube-master sync_file results
+  set_fact:
+    vault_kube_node_certs_needed: "{{ vault_kube_node_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results }}"
+  when: item.no_srcs|bool
+
+- name: cluster/sync_kube_node_certs | Unset sync_file_results after kube node certs
+  set_fact:
+    sync_file_results: []
+
+- include: ../sync_file.yml
+  vars: 
+    sync_file: ca.pem
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_hosts: "{{ groups['k8s-cluster'] }}"
+
+- name: cluster/sync_kube_node_certs | Unset sync_file_results after ca.pem
+  set_fact:
+    sync_file_results: []

roles/vault/tasks/cluster/unseal.yml

@@ -0,0 +1,26 @@
+---
+
+- name: cluster/unseal | Unseal Vault
+  uri:
+    url: "https://localhost:{{ vault_port }}/v1/sys/unseal"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body:
+      key: "{{ item }}"
+  with_items: "{{ vault_unseal_keys|default([]) }}"
+  when: vault_is_sealed
+
+- name: cluster/unseal | Find the current leader
+  uri:
+    url: "https://localhost:{{ vault_port }}/v1/sys/health"
+    headers: "{{ vault_headers }}"
+    method: HEAD
+    status_code: 200,429
+  register: vault_leader_check
+
+- name: cluster/unseal | Set fact for current leader
+  set_fact:
+    vault_leader: "{{ item }}"
+  with_items: "{{ groups.vault }}"
+  when: 'hostvars[item]["vault_leader_check"]["status"] == 200'

roles/vault/tasks/config_ca.yml

@@ -0,0 +1,19 @@
+---
+
+- name: config_ca | Read root CA cert for Vault
+  command: cat /etc/vault/ssl/ca.pem
+  register: vault_ca_cert_cat
+
+- name: config_ca | Read root CA key for Vault
+  command: cat /etc/vault/ssl/ca-key.pem
+  register: vault_ca_key_cat
+
+- name: config_ca | Configure pki mount to use the found root CA cert and key
+  uri:
+    url: "{{ vault_url }}/v1/pki/config/ca"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body:
+      pem_bundle: "{{ vault_ca_cert_cat.stdout + '\n' + vault_ca_key_cat.stdout }}"
+    status_code: 204

roles/vault/tasks/gen_cert.yml

@@ -0,0 +1,50 @@
+---
+
+# This could be a role or custom module
+
+- name: gen_cert | Ensure target directory exists
+  file:
+    path: "{{ gen_cert_path | dirname }}" 
+    state: directory
+
+- name: gen_cert | Generate the cert
+  uri:
+    url: "{{ gen_cert_vault_url}}/v1/pki/issue/{{ gen_cert_vault_role }}"
+    headers: "{{ gen_cert_vault_headers }}"
+    method: POST
+    body_format: json
+    body:
+      alt_names: "{{ gen_cert_alt_names|default([]) }}"
+      common_name: "{{ gen_cert_path.rsplit('/', 1)[1].rsplit('.', 1)[0] }}"
+      format: "{{ gen_cert_format|default('pem') }}"
+      ip_sans: "{{ gen_cert_ip_sans|default([]) }}"
+  register: gen_cert_result
+  when: inventory_hostname == gen_cert_hosts|first
+
+- name: gen_cert | Copy the cert to all hosts
+  copy:
+    content: "{{ hostvars[gen_cert_hosts|first]['gen_cert_result']['json']['data']['certificate'] }}"
+    dest: "{{ gen_cert_path }}"
+
+- name: gen_cert | Copy the key to all hosts
+  copy:
+    content: "{{ hostvars[gen_cert_hosts|first]['gen_cert_result']['json']['data']['private_key'] }}"
+    dest: "{{ gen_cert_path.rsplit('.', 1)|first + '-key.' + gen_cert_path.rsplit('.', 1)|last }}"
+
+- name: gen_cert | Copy issuing CA cert
+  copy:
+    content: "{{ hostvars[gen_cert_hosts|first]['gen_cert_result']['json']['data']['issuing_ca'] }}"
+    dest: "{{ gen_cert_path | dirname }}/ca.pem"
+  when: gen_cert_copy_ca|default(false)|bool
+
+- name: gen_cert | Unset common variables to avoid bleed over
+  set_fact:
+    gen_cert_copy_ca: false
+    gen_cert_alt_names: []
+    gen_cert_format: pem
+    gen_cert_hosts: []
+    gen_cert_ip_sans: []
+    gen_cert_path: ''
+    gen_cert_vault_headers: ''
+    gen_cert_vault_role: ''
+    gen_cert_vault_url: ''

roles/vault/tasks/main.yml

@@ -0,0 +1,13 @@
+---
+
+- include: check_vault.yml
+  when: inventory_hostname in groups.vault
+
+# bootstrap.yml's sole purpose is to ensure certs exist for Vault and Etcd
+# prior to startup, so TLS can be enabled.
+- include: bootstrap/main.yml
+  when: vault_bootstrap|bool
+
+# cluster.yml should only run after the backend service is ready (default etcd)
+- include: cluster/main.yml
+  when: not vault_bootstrap|bool

roles/vault/tasks/sync.yml

@@ -0,0 +1,38 @@
+---
+
+- name: "sync_file | Cat the file"
+  command: "cat {{ sync_file_path }}"
+  register: sync_file_cat
+  when: inventory_hostname == sync_file_srcs|first
+
+- name: "sync_file | Cat the key file"
+  command: "cat {{ sync_file_key_path }}"
+  register: sync_file_key_cat
+  when: sync_file_is_cert|bool and inventory_hostname == sync_file_srcs|first
+
+- name: "sync_file | Set facts for file contents"
+  set_fact:
+    sync_file_contents: "{{ hostvars[sync_file_srcs|first]['sync_file_cat']['stdout'] }}"
+
+- name: "sync_file | Set fact for key contents"
+  set_fact:
+    sync_file_key_contents: "{{ hostvars[sync_file_srcs|first]['sync_file_key_cat']['stdout'] }}"
+  when: sync_file_is_cert|bool
+
+- name: "sync_file | Ensure the directory exists"
+  file:
+    path: "{{ sync_file_dir }}"
+    state: directory
+  when: inventory_hostname not in sync_file_srcs
+
+- name: "sync_file | Copy the file to hosts that don't have it"
+  copy:
+    content: "{{ sync_file_contents }}"
+    dest: "{{ sync_file_path }}"
+  when: inventory_hostname not in sync_file_srcs
+
+- name: "sync_file | Copy the key file to hosts that don't have it"
+  copy:
+    content: "{{ sync_file_key_contents }}"
+    dest: "{{ sync_file_key_path }}"
+  when: sync_file_is_cert|bool and inventory_hostname not in sync_file_srcs

roles/vault/tasks/sync_file.yml

@@ -0,0 +1,97 @@
+---
+
+# NOTE: This should be a role (or custom module), but currently include_role is too buggy to use
+
+- name: "sync_file | Set facts for directory and file when sync_file_path is defined"
+  set_fact:
+    sync_file_dir: "{{ sync_file_path | dirname }}"
+    sync_file: "{{ sync_file_path | basename }}"
+  when: sync_file_path|bool
+
+- name: "sync_file | Set fact for sync_file_path when undefined"
+  set_fact:
+    sync_file_path: "{{ (sync_file_dir, sync_file)|join('/') }}"
+  when: not sync_file_path|bool
+
+- name: "sync_file | Set fact for key path name"
+  set_fact:
+    sync_file_key_path: "{{ sync_file_path.rsplit('.', 1)|first + '-key.' + sync_file_path.rsplit('.', 1)|last }}"
+  when: sync_file_is_cert|bool and not sync_file_key_path|bool
+
+- name: "sync_file | Check if file exists"
+  stat:
+    path: "{{ sync_file_path }}"
+  register: sync_file_stat
+
+- name: "sync_file | Check if key file exists"
+  stat:
+    path: "{{ sync_file_key_path }}"
+  register: sync_file_key_stat
+  when: sync_file_is_cert|bool
+
+- name: "sync_file | Combine all possible file sync sources"
+  set_fact:
+    sync_file_srcs: "{{ sync_file_srcs|default([]) + [host_item] }}"
+  with_items: "{{ sync_file_hosts | unique }}"
+  loop_control:
+    loop_var: host_item
+  when: hostvars[host_item]["sync_file_stat"]["stat"]["exists"]|bool
+
+- name: "sync_file | Combine all possible key file sync sources"
+  set_fact:
+    sync_file_key_srcs: "{{ sync_file_key_srcs|default([]) + [host_item] }}"
+  with_items: "{{ sync_file_hosts | unique }}"
+  loop_control:
+    loop_var: host_item
+  when: sync_file_is_cert|bool and hostvars[host_item]["sync_file_key_stat"]["stat"]["exists"]|bool
+
+- name: "sync_file | Remove sync sources with files that do not match sync_file_srcs|first"
+  set_fact:
+    _: "{% if inventory_hostname in sync_file_srcs %}{{ sync_file_srcs.remove(inventory_hostname) }}{% endif %}"
+  when: >-
+        sync_file_srcs|length > 0 and
+        inventory_hostname != sync_file_srcs|first and
+        sync_file_stat.stat.get("checksum") != hostvars[sync_file_srcs|first]["sync_file_stat"]["stat"]["checksum"]
+
+- name: "sync_file | Remove sync sources with keys that do not match sync_file_srcs|first"
+  set_fact:
+    _: "{% if inventory_hostname in sync_file_srcs %}{{ sync_file_srcs.remove(inventory_hostname) }}{% endif %}"
+  when: >-
+        sync_file_is_cert|bool and
+        sync_file_key_srcs|length > 0 and
+        inventory_hostname != sync_file_srcs|first and
+        sync_file_key_stat.stat.checksum != hostvars[sync_file_srcs|first]["sync_file_key_stat"]["stat"]["checksum"]
+
+- name: "sync_file | Consolidate file and key sources"
+  set_fact:
+    sync_file_srcs: "{{ sync_file_srcs | intersect(sync_file_key_srcs) }}"
+  when: sync_file_is_cert|bool
+
+- name: "sync_file | Set facts for situations where sync is not needed"
+  set_fact:
+    sync_file_no_srcs: "{{ true if sync_file_srcs|length == 0 else false }}"
+    sync_file_unneeded: "{{ true if sync_file_srcs|length == sync_file_hosts|length else false }}"
+
+- name: "sync_file | Set sync_file_result fact"
+  set_fact:
+    sync_file_result:
+      no_srcs: "{{ sync_file_no_srcs }}"
+      path: "{{ sync_file_path }}"
+      sync_unneeded: "{{ sync_file_unneeded }}"
+
+- name: "sync_file | Update sync_file_results fact"
+  set_fact:
+    sync_file_results: "{{ sync_file_results|default([]) + [sync_file_result] }}"
+
+- include: sync.yml
+  when: not (sync_file_no_srcs or sync_file_unneeded)
+
+- name: "Unset local vars to avoid variable bleed into next iteration"
+  set_fact:
+    sync_file_dir: ''
+    sync_file: ''
+    sync_file_key_path: ''
+    sync_file_hosts: []
+    sync_file_path: ''
+    sync_file_srcs: []
+    sync_file_key_srcs: []