Merge pull request #902 from insequent/master

Adding vault role

.gitignore

@@ -12,3 +12,4 @@ temp
 *.tfstate.backup
 **/*.sw[pon]
 /ssh-bastion.conf
+**/*.sw[pon]

.gitlab-ci.yml

@@ -54,6 +54,7 @@ before_script:
   LOG_LEVEL: "-vv"
   ETCD_DEPLOYMENT: "docker"
   KUBELET_DEPLOYMENT: "docker"
+  VAULT_DEPLOYMENT: "docker"
   WEAVE_CPU_LIMIT: "100m"
   MAGIC: "ci check this"
 
@@ -106,6 +107,7 @@ before_script:
       -e ansible_python_interpreter=${PYPATH}
       -e ansible_ssh_user=${SSH_USER} 
       -e bootstrap_os=${BOOTSTRAP_OS}
+      -e cert_management=${CERT_MGMT:-script}
       -e cloud_provider=gce
       -e deploy_netchecker=true
       -e download_localhost=true
@@ -115,6 +117,7 @@ before_script:
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
       -e local_release_dir=${PWD}/downloads
       -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
       cluster.yml
 
 
@@ -292,6 +295,14 @@ before_script:
   ETCD_DEPLOYMENT: rkt
   KUBELET_DEPLOYMENT: rkt
 
+.ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: canal
+  CERT_MGMT: vault
+  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_REGION: us-central1-b
+  CLUSTER_MODE: separate
+
 # Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
 coreos-calico-sep:
   stage: deploy-gce-part1
@@ -506,6 +517,17 @@ ubuntu-rkt-sep:
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
+ubuntu-vault-sep:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_vault_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
 # Premoderated with manual actions
 ci-authorized:
   <<: *job

cluster.yml

@@ -28,7 +28,14 @@
   roles:
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker }
-    - { role: rkt, tags: rkt, when: "'rkt' in [ etcd_deployment_type, kubelet_deployment_type ]" }
+    - role: rkt
+      tags: rkt
+      when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
+
+- hosts: etcd:k8s-cluster:vault
+  any_errors_fatal: true
+  roles:
+    - { role: vault, tags: vault, vault_bootstrap: true, when: "cert_management == 'vault'" }
 
 - hosts: etcd:!k8s-cluster
   any_errors_fatal: true
@@ -39,6 +46,15 @@
   any_errors_fatal: true
   roles:
     - { role: etcd, tags: etcd }
+
+- hosts: etcd:k8s-cluster:vault
+  any_errors_fatal: true
+  roles:
+    - { role: vault, tags: vault, when: "cert_management == 'vault'"}
+
+- hosts: k8s-cluster
+  any_errors_fatal: true
+  roles:
     - { role: kubernetes/node, tags: node }
     - { role: network_plugin, tags: network }
 

docs/vault.md

@@ -0,0 +1,92 @@
+Hashicorp Vault Role
+====================
+
+Overview
+--------
+
+The Vault role is a two-step process:
+
+1. Bootstrap
+
+You cannot start your certificate management service securely with SSL (and 
+the datastore behind it) without having the certificates in-hand already. This
+presents an unfortunate chicken and egg scenario, with one requiring the other.
+To solve for this, the Bootstrap step was added.
+
+This step spins up a temporary instance of Vault to issue certificates for
+Vault itself. It then leaves the temporary instance running, so that the Etcd
+role can generate certs for itself as well. Eventually, this may be improved
+to allow alternate backends (such as Consul), but currently the tasks are
+hardcoded to only create a Vault role for Etcd.
+
+2. Cluster
+
+This step is where the long-term Vault cluster is started and configured. Its
+first task, is to stop any temporary instances of Vault, to free the port for
+the long-term. At the end of this task, the entire Vault cluster should be up
+and read to go.
+
+
+Keys to the Kingdom
+-------------------
+
+The two most important security pieces of Vault are the ``root_token``
+and ``unsealing_keys``. Both of these values are given exactly once, during
+the initialization of the Vault cluster. For convenience, they are saved
+to the ``vault_secret_dir`` (default: /etc/vault/secrets) of every host in the
+vault group.
+
+It is *highly* recommended that these secrets are removed from the servers after
+your cluster has been deployed, and kept in a safe location of your choosing.
+Naturally, the seriousness of the situation depends on what you're doing with
+your Kargo cluster, but with these secrets, an attacker will have the ability
+to authenticate to almost everything in Kubernetes and decode all private
+(HTTPS) traffic on your network signed by Vault certificates.
+
+For even greater security, you may want to remove and store elsewhere any
+CA keys generated as well (e.g. /etc/vault/ssl/ca-key.pem). 
+
+Vault by default encrypts all traffic to and from the datastore backend, all
+resting data, and uses TLS for its TCP listener. It is recommended that you
+do not change the Vault config to disable TLS, unless you absolutely have to.
+
+
+Usage
+-----
+
+To get the Vault role running, you must to do two things at a minimum:
+
+1. Assign the ``vault`` group to at least 1 node in your inventory
+2. Change ``cert_management`` to be ``vault`` instead of ``script``
+
+Nothing else is required, but customization is possible. Check
+``roles/vault/defaults/main.yml`` for the different variables that can be
+overridden, most common being ``vault_config``, ``vault_port``, and
+``vault_deployment_type``.
+
+Also, if you intend to use a Root or Intermediate CA generated elsewhere,
+you'll need to copy the certificate and key to the hosts in the vault group
+prior to running the vault role. By default, they'll be located at
+``/etc/vault/ssl/ca.pem`` and ``/etc/vault/ssl/ca-key.pem``, respectively.
+
+Additional Notes:
+
+- ``groups.vault|first`` is considered the source of truth for Vault variables
+- ``vault_leader_url`` is used as pointer for the current running Vault
+- Each service should have its own role and credentials. Currently those 
+  credentials are saved to ``/etc/vault/roles/<role>/``. The service will
+  need to read in those credentials, if they want to interact with Vault.
+
+
+Potential Work
+--------------
+
+- Change the Vault role to not run certain tasks when ``root_token`` and
+  ``unseal_keys`` are not present. Alternatively, allow user input for these
+  values when missing.
+- Add the ability to start temp Vault with Host, Rkt, or Docker
+- Add a dynamic way to change out the backend role creation during Bootstrap,
+  so other services can be used (such as Consul)
+- Segregate Server Cert generation from Auth Cert generation (separate CAs).
+  This work was partially started with the `auth_cert_backend` tasks, but would
+  need to be further applied to all roles (particularly Etcd and Kubernetes).

inventory/group_vars/all.yml

@@ -204,5 +204,12 @@ kpm_packages: []
 rkt_version: 1.21.0
 etcd_deployment_type: docker
 kubelet_deployment_type: docker
+vault_deployment_type: docker
 
 efk_enabled: false
+
+## Certificate Management
+## This setting determines whether certs are generated via scripts or whether a
+## cluster of Hashicorp's Vault is started to issue certificates (using etcd
+## as a backend). Options are "script" or "vault"
+cert_management: script

roles/download/tasks/main.yml

@@ -94,7 +94,7 @@
 
 - name: "Set default value for 'container_changed' to false"
   set_fact:
-    container_changed: "{{pull_required|bool|default(false)}}"
+    container_changed: "{{pull_required|default(false)|bool}}"
 
 - name: "Update the 'container_changed' fact"
   set_fact:

roles/etcd/defaults/main.yml

@@ -15,3 +15,5 @@ etcd_memory_limit: 512M
 
 # Uncomment to set CPU share for etcd
 #etcd_cpu_limit: 300m
+
+etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr', [])) }}"

roles/etcd/meta/main.yml

@@ -6,3 +6,5 @@ dependencies:
   - role: download
     file: "{{ downloads.etcd }}"
     tags: download
+
+# NOTE: Dynamic task dependency on Vault Role if cert_management == "vault"

roles/etcd/tasks/gen_certs_vault.yml

@@ -0,0 +1,77 @@
+---
+
+- name: gen_certs_vault | Read in the local credentials
+  command: cat /etc/vault/roles/etcd/userpass
+  register: etcd_vault_creds_cat
+  when: inventory_hostname == groups.etcd|first
+
+- name: gen_certs_vault | Set facts for read Vault Creds
+  set_fact:
+    etcd_vault_creds: "{{ hostvars[groups.etcd|first]['etcd_vault_creds_cat']['stdout']|from_json }}"
+  when: inventory_hostname == groups.etcd|first
+
+- name: gen_certs_vault | Log into Vault and obtain an token
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ etcd_vault_creds.username }}"
+    headers:
+      Accept: application/json
+      Content-Type: application/json 
+    method: POST
+    body_format: json
+    body:
+      password: "{{ etcd_vault_creds.password }}"
+  register: etcd_vault_login_result
+  when: inventory_hostname == groups.etcd|first
+
+- name: gen_certs_vault | Set fact for Vault API token
+  set_fact:
+    etcd_vault_headers:
+        Accept: application/json
+        Content-Type: application/json
+        X-Vault-Token: "{{ hostvars[groups.etcd|first]['etcd_vault_login_result']['json']['auth']['client_token'] }}"
+
+# Issue master certs to Etcd nodes
+- include: ../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_alt_names: "{{ groups.etcd + ['localhost'] }}"
+    issue_cert_copy_ca: "{{ item == etcd_master_certs_needed|first }}"
+    issue_cert_file_group: "{{ etcd_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ etcd_vault_headers }}" 
+    issue_cert_hosts: "{{ groups.etcd }}"
+    issue_cert_ip_sans: >-
+        [
+        {%- for host in groups.etcd  -%}
+        "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- endfor -%}
+        "127.0.0.1","::1"
+        ]
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: etcd
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ etcd_master_certs_needed|d([]) }}"
+  when: inventory_hostname in groups.etcd
+  notify: set etcd_secret_changed
+
+# Issue node certs to everyone else
+- include: ../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_alt_names: "{{ etcd_node_cert_hosts }}"
+    issue_cert_copy_ca: "{{ item == etcd_node_certs_needed|first }}"
+    issue_cert_file_group: "{{ etcd_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ etcd_vault_headers }}" 
+    issue_cert_hosts: "{{ etcd_node_cert_hosts }}"
+    issue_cert_ip_sans: >-
+        [
+        {%- for host in etcd_node_cert_hosts -%}
+        "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- endfor -%}
+        "127.0.0.1","::1"
+        ]
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: etcd
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ etcd_node_certs_needed|d([]) }}"
+  when: inventory_hostname in etcd_node_cert_hosts
+  notify: set etcd_secret_changed

roles/etcd/tasks/main.yml

@@ -1,10 +1,24 @@
 ---
 - include: pre_upgrade.yml
   tags: etcd-pre-upgrade
+
 - include: check_certs.yml
+  when: cert_management == "script"
   tags: [etcd-secrets, facts]
-- include: gen_certs.yml
+- include: gen_certs_script.yml
+  when: cert_management == "script"
   tags: etcd-secrets
+
+- include: sync_etcd_master_certs.yml
+  when: cert_management == "vault" and inventory_hostname in groups.etcd
+  tags: etcd-secrets
+- include: sync_etcd_node_certs.yml
+  when: cert_management == "vault" and inventory_hostname in etcd_node_cert_hosts 
+  tags: etcd-secrets
+- include: gen_certs_vault.yml
+  when: cert_management == "vault" and (etcd_master_certs_needed|d() or etcd_node_certs_needed|d())
+  tags: etcd-secrets
+
 - include: "install_{{ etcd_deployment_type }}.yml"
   when: is_etcd_master
   tags: upgrade

roles/etcd/tasks/sync_etcd_master_certs.yml

@@ -0,0 +1,38 @@
+---
+
+- name: sync_etcd_master_certs | Create list of master certs needing creation
+  set_fact: 
+    etcd_master_cert_list: >-
+        {{ etcd_master_cert_list|default([]) +  [
+        "admin-" + item + ".pem",
+        "member-" + item + ".pem"
+        ] }}
+  with_items: "{{ groups.etcd }}"
+
+- include: ../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ groups.etcd }}"
+    sync_file_is_cert: true
+  with_items: "{{ etcd_master_cert_list|d([]) }}"
+
+- name: sync_etcd_certs | Set facts for etcd sync_file results
+  set_fact:
+    etcd_master_certs_needed: "{{ etcd_master_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+  when: item.no_srcs|bool
+
+- name: sync_etcd_certs | Unset sync_file_results after etcd certs sync
+  set_fact:
+    sync_file_results: []
+
+- include: ../../vault/tasks/shared/sync_file.yml
+  vars:
+    sync_file: ca.pem
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ groups.etcd }}"
+
+- name: sync_etcd_certs | Unset sync_file_results after ca.pem sync
+  set_fact:
+    sync_file_results: []

roles/etcd/tasks/sync_etcd_node_certs.yml

@@ -0,0 +1,34 @@
+---
+
+- name: sync_etcd_node_certs | Create list of node certs needing creation
+  set_fact: 
+    etcd_node_cert_list: "{{ etcd_node_cert_list|default([]) +  ['node-' + item + '.pem'] }}"
+  with_items: "{{ etcd_node_cert_hosts }}"
+
+- include: ../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ etcd_node_cert_hosts }}"
+    sync_file_is_cert: true
+  with_items: "{{ etcd_node_cert_list|d([]) }}"
+
+- name: sync_etcd_node_certs | Set facts for etcd sync_file results
+  set_fact:
+    etcd_node_certs_needed: "{{ etcd_node_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+  when: item.no_srcs|bool
+
+- name: sync_etcd_node_certs | Unset sync_file_results after etcd node certs
+  set_fact:
+    sync_file_results: []
+
+- include: ../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: ca.pem
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ etcd_node_cert_hosts }}"
+
+- name: sync_etcd_node_certs | Unset sync_file_results after ca.pem
+  set_fact:
+    sync_file_results: []

roles/kubernetes/node/tasks/install.yml

@@ -25,6 +25,20 @@
   template: "src=kubelet.{{ kubelet_deployment_type }}.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes"
   notify: restart kubelet
 
+- name: install | Set SSL CA directories
+  set_fact:
+    ssl_ca_dirs: "[
+      {% if ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] -%}
+      '/usr/share/ca-certificates',
+      {% elif ansible_os_family == 'RedHat' -%}
+      '/etc/pki/tls',
+      '/etc/pki/ca-trust',
+      {% elif ansible_os_family == 'Debian' -%}
+      '/usr/share/ca-certificates',
+      {% endif -%}
+    ]"
+  tags: facts
+
 - name: install | Install kubelet launch script
   template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes
   notify: restart kubelet

roles/kubernetes/preinstall/defaults/main.yml

@@ -10,6 +10,7 @@ common_required_pkgs:
   - rsync
   - bash-completion
   - socat
+  - unzip
 
 # Set to true if your network does not support IPv6
 # This maybe necessary for pulling Docker images from

roles/kubernetes/secrets/meta/main.yml

@@ -0,0 +1,2 @@
+---
+# NOTE: Dynamic task dependency on Vault Role if cert_management == "vault"

roles/kubernetes/secrets/tasks/gen_certs_script.yml

@@ -160,20 +160,6 @@
       {%- endif %}
   tags: facts
 
-- name: SSL CA directories | Set SSL CA directories
-  set_fact:
-    ssl_ca_dirs: "[
-      {% if ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] -%}
-      '/usr/share/ca-certificates',
-      {% elif ansible_os_family == 'RedHat' -%}
-      '/etc/pki/tls',
-      '/etc/pki/ca-trust',
-      {% elif ansible_os_family == 'Debian' -%}
-      '/usr/share/ca-certificates',
-      {% endif -%}
-    ]"
-  tags: facts
-
 - name: Gen_certs | add CA to trusted CA dir
   copy:
     src: "{{ kube_cert_dir }}/ca.pem"

roles/kubernetes/secrets/tasks/gen_certs_vault.yml

@@ -0,0 +1,84 @@
+---
+
+- name: gen_certs_vault | Read in the local credentials
+  command: cat /etc/vault/roles/kube/userpass
+  register: kube_vault_creds_cat
+  when: inventory_hostname == groups['k8s-cluster']|first
+
+- name: gen_certs_vault | Set facts for read Vault Creds
+  set_fact:
+    kube_vault_creds: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_creds_cat']['stdout'] | from_json }}"
+  when: inventory_hostname == groups['k8s-cluster']|first
+
+- name: gen_certs_vault | Log into Vault and obtain an token
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ kube_vault_creds.username }}"
+    headers:
+      Accept: application/json
+      Content-Type: application/json 
+    method: POST
+    body_format: json
+    body:
+      password: "{{ kube_vault_creds.password }}"
+  register: kube_vault_login_result
+  when: inventory_hostname == groups['k8s-cluster']|first
+
+- name: gen_certs_vault | Set fact for Vault API token
+  set_fact:
+    kube_vault_headers:
+        Accept: application/json
+        Content-Type: application/json
+        X-Vault-Token: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_login_result']['json']['auth']['client_token'] }}"
+
+# Issue certs to kube-master nodes
+- include: ../../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_copy_ca: "{{ item == kube_master_certs_needed|first }}"
+    issue_cert_file_group: "{{ kube_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ kube_vault_headers }}"
+    issue_cert_hosts: "{{ groups['kube-master'] }}"
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: kube
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ kube_master_certs_needed|d([]) }}"
+  when: inventory_hostname in groups['kube-master']
+
+- include: ../../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_alt_names: >-
+        {{
+        groups['kube-master'] +
+        ['kubernetes.default.svc.cluster.local', 'kubernetes.default.svc', 'kubernetes.default', 'kubernetes'] +
+        ['localhost']
+        }}
+    issue_cert_file_group: "{{ kube_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ kube_vault_headers }}" 
+    issue_cert_hosts: "{{ groups['kube-master'] }}"
+    issue_cert_ip_sans: >-
+        [
+        {%- for host in groups['kube-master']  -%}
+        "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- endfor -%}
+        "127.0.0.1","::1","{{ kube_apiserver_ip }}"
+        ]
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: kube
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ kube_api_certs_needed|d([]) }}"
+  when: inventory_hostname in groups['kube-master']
+
+# Issue node certs to k8s-cluster nodes
+- include: ../../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_copy_ca: "{{ item == kube_node_certs_needed|first }}"
+    issue_cert_file_group: "{{ kube_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ kube_vault_headers }}" 
+    issue_cert_hosts: "{{ groups['k8s-cluster'] }}"
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: kube
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ kube_node_certs_needed|d([]) }}"
+  when: inventory_hostname in groups['k8s-cluster']

roles/kubernetes/secrets/tasks/main.yml

@@ -70,7 +70,19 @@
   delegate_to: "{{groups['kube-master'][0]}}"
   when: gen_tokens|default(false)
 
-- include: gen_certs.yml
+- include: gen_certs_script.yml
+  when: cert_management == "script"
   tags: k8s-secrets
+
+- include: sync_kube_master_certs.yml
+  when: cert_management == "vault" and inventory_hostname in groups['kube-master']
+  tags: k8s-secrets
+- include: sync_kube_node_certs.yml
+  when: cert_management == "vault" and inventory_hostname in groups['k8s-cluster']
+  tags: k8s-secrets
+- include: gen_certs_vault.yml
+  when: cert_management == "vault"
+  tags: k8s-secrets
+
 - include: gen_tokens.yml
   tags: k8s-secrets

roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml

@@ -0,0 +1,58 @@
+---
+
+- name: sync_kube_master_certs | Create list of needed kube admin certs
+  set_fact:
+    kube_master_cert_list: "{{ kube_master_cert_list|d([]) + ['admin-' + item + '.pem'] }}"
+  with_items: "{{ groups['kube-master'] }}"
+
+- include: ../../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_group: "{{ kube_cert_group }}"
+    sync_file_hosts: "{{ groups['kube-master'] }}"
+    sync_file_is_cert: true
+    sync_file_owner: kube
+  with_items: "{{ kube_master_cert_list|d([]) }}"
+
+- name: sync_kube_master_certs | Set facts for kube admin sync_file results
+  set_fact:
+    kube_master_certs_needed: "{{ kube_master_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+  when: item.no_srcs|bool
+
+- name: sync_kube_master_certs | Unset sync_file_results after kube admin certs
+  set_fact:
+    sync_file_results: []
+
+- include: ../../../vault/tasks/shared/sync_file.yml
+  vars:
+    sync_file: "apiserver.pem"
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_group: "{{ kube_cert_group }}"
+    sync_file_hosts: "{{ groups['kube-master'] }}"
+    sync_file_is_cert: true
+    sync_file_owner: kube
+
+- name: sync_kube_master_certs | Set facts for apiserver sync_file results
+  set_fact:
+    kube_api_certs_needed: "{{ item.path }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+  when: "{{ item.no_srcs }}"
+
+- name: sync_kube_master_certs | Unset sync_file_results after apiserver cert
+  set_fact:
+    sync_file_results: []
+
+
+- include: ../../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: ca.pem
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_group: "{{ kube_cert_group }}"
+    sync_file_hosts: "{{ groups['kube-master'] }}"
+    sync_file_owner: kube
+
+- name: sync_kube_master_certs | Unset sync_file_results after ca.pem
+  set_fact:
+    sync_file_results: []

roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml

@@ -0,0 +1,38 @@
+---
+
+- name: sync_kube_node_certs | Create list of needed certs
+  set_fact:
+    kube_node_cert_list: "{{ kube_node_cert_list|default([]) + ['node-' + item + '.pem'] }}"
+  with_items: "{{ groups['k8s-cluster'] }}"
+
+- include: ../../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_group: "{{ kuber_cert_group }}"
+    sync_file_hosts: "{{ groups['k8s-cluster'] }}"
+    sync_file_is_cert: true
+    sync_file_owner: kube
+  with_items: "{{ kube_node_cert_list|default([]) }}"
+
+- name: sync_kube_node_certs | Set facts for kube-master sync_file results
+  set_fact:
+    kube_node_certs_needed: "{{ kube_node_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+  when: item.no_srcs|bool
+
+- name: sync_kube_node_certs | Unset sync_file_results after kube node certs
+  set_fact:
+    sync_file_results: []
+
+- include: ../../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: ca.pem
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_group: "{{ kuber_cert_group }}"
+    sync_file_hosts: "{{ groups['k8s-cluster'] }}"
+    sync_file_owner: kube
+
+- name: sync_kube_node_certs | Unset sync_file_results after ca.pem
+  set_fact:
+    sync_file_results: []

roles/vault/defaults/main.yml

@@ -0,0 +1,90 @@
+---
+
+vault_adduser_vars:
+  comment: "Hashicorp Vault User"
+  createhome: no
+  name: vault
+  shell: /sbin/nologin
+  system: yes
+vault_base_dir: /etc/vault
+# https://releases.hashicorp.com/vault/0.6.4/vault_0.6.4_SHA256SUMS
+vault_binary_checksum: 04d87dd553aed59f3fe316222217a8d8777f40115a115dac4d88fac1611c51a6
+vault_bootstrap: false
+vault_ca_options:
+  common_name: kube-cluster-ca
+  format: pem
+  ttl: 87600h
+vault_cert_dir: "{{ vault_base_dir }}/ssl"
+vault_client_headers:
+  Accept: "application/json"
+  Content-Type: "application/json"
+vault_config:
+  backend:
+    etcd:
+      address: "{{ vault_etcd_url }}"
+      ha_enabled: "true"
+      redirect_addr: "https://{{ ansible_default_ipv4.address }}:{{ vault_port }}"
+      tls_ca_file: "{{ vault_cert_dir }}/ca.pem"
+  cluster_name: "kubernetes-vault"
+  default_lease_ttl: "{{ vault_default_lease_ttl }}"
+  listener:
+    tcp:
+      address: "0.0.0.0:{{ vault_port }}"
+      tls_cert_file: "{{ vault_cert_dir }}/api.pem"
+      tls_key_file: "{{ vault_cert_dir }}/api-key.pem"
+  max_lease_ttl: "{{ vault_max_lease_ttl }}"
+vault_config_dir: "{{ vault_base_dir }}/config"
+vault_container_name: kube-hashicorp-vault
+# This variable is meant to match the GID of vault inside Hashicorp's official Vault Container
+vault_default_lease_ttl: 720h
+vault_default_role_permissions:
+  allow_any_name: true
+vault_deployment_type: docker
+vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip"
+vault_download_vars:
+  container: "{{ vault_deployment_type != 'host' }}"
+  dest: "vault/vault_{{ vault_version }}_linux_amd64.zip"
+  enabled: true
+  mode: "0755"
+  owner: "vault"
+  repo: "{{ vault_image_repo }}"
+  sha256: "{{ vault_binary_checksum if vault_deployment_type == 'host' else vault_digest_checksum|d(none) }}"
+  source_url: "{{ vault_download_url }}"
+  tag: "{{ vault_image_tag }}"
+  unarchive: true
+  url: "{{ vault_download_url }}"
+  version: "{{ vault_version }}"
+vault_etcd_url: "https://{{ hostvars[groups.etcd[0]]['ansible_default_ipv4']['address'] }}:2379"
+vault_image_repo: "vault"
+vault_image_tag: "{{ vault_version }}"
+vault_log_dir: "/var/log/vault"
+vault_max_lease_ttl: 87600h
+vault_needs_gen: false
+vault_port: 8200
+# Although "cert" is an option, ansible has no way to auth via cert until
+# upstream merges: https://github.com/ansible/ansible/pull/18141
+vault_role_auth_method: userpass 
+vault_roles:
+  - name: etcd
+    group: etcd
+    policy_rules: default
+    role_options: default
+  - name: kube
+    group: k8s-cluster
+    policy_rules: default
+    role_options: default
+vault_roles_dir: "{{ vault_base_dir }}/roles"
+vault_secret_shares: 1
+vault_secret_threshold: 1
+vault_secrets_dir: "{{ vault_base_dir }}/secrets"
+vault_temp_config:
+  default_lease_ttl: "{{ vault_default_lease_ttl }}"
+  backend:
+    file:
+      path: /vault/file
+  listener:
+    tcp:
+      address: "0.0.0.0:{{ vault_port }}"
+      tls_disable: "true"
+vault_temp_container_name: vault-temp
+vault_version: 0.6.4

roles/vault/meta/main.yml

@@ -0,0 +1,8 @@
+---
+
+dependencies:
+  - role: adduser
+    user: "{{ vault_adduser_vars }}"
+  - role: download
+    file: "{{ vault_download_vars }}"
+    tags: download

roles/vault/tasks/bootstrap/ca_trust.yml

@@ -0,0 +1,32 @@
+---
+
+- name: bootstrap/ca_trust | pull CA from cert from groups.vault|first
+  command: "cat {{ vault_cert_dir }}/ca.pem"
+  register: vault_cert_file_cat
+  when: inventory_hostname == groups.vault|first
+
+# This part is mostly stolen from the etcd role
+- name: bootstrap/ca_trust | target ca-certificate store file
+  set_fact:
+    ca_cert_path: >-
+      {% if ansible_os_family == "Debian" -%}
+      /usr/local/share/ca-certificates/kube-cluster-ca.crt
+      {%- elif ansible_os_family == "RedHat" -%}
+      /etc/pki/ca-trust/source/anchors/kube-cluster-ca.crt
+      {%- elif ansible_os_family == "CoreOS" -%}
+      /etc/ssl/certs/kube-cluster-ca.pem
+      {%- endif %}
+
+- name: bootstrap/ca_trust | add CA to trusted CA dir
+  copy:
+    content: "{{ hostvars[groups.vault|first]['vault_cert_file_cat']['stdout'] }}"
+    dest: "{{ ca_cert_path }}"
+  register: vault_ca_cert
+
+- name: bootstrap/ca_trust | update ca-certificates (Debian/Ubuntu/CoreOS)
+  command: update-ca-certificates
+  when: vault_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"]
+
+- name: bootstrap/ca_trust | update ca-certificates (RedHat)
+  command: update-ca-trust extract
+  when: vault_ca_cert.changed and ansible_os_family == "RedHat"

roles/vault/tasks/bootstrap/create_etcd_role.yml

@@ -0,0 +1,10 @@
+---
+
+- include: ../shared/create_role.yml
+  vars:
+    create_role_name: "{{ item.name }}" 
+    create_role_group: "{{ item.group }}"
+    create_role_policy_rules: "{{ item.policy_rules }}"
+    create_role_options: "{{ item.role_options }}"
+  with_items: "{{ vault_roles }}"
+  when: item.name == "etcd"

roles/vault/tasks/bootstrap/gen_auth_ca.yml

@@ -0,0 +1,21 @@
+---
+
+- name: bootstrap/gen_auth_ca | Generate Root CA
+  uri:
+    url: "{{ vault_leader_url }}/v1/auth-pki/root/generate/exported"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body: "{{ vault_ca_options }}"
+  register: vault_auth_ca_gen
+  when: inventory_hostname == groups.vault|first
+
+- name: bootstrap/gen_auth_ca | Copy auth CA cert to Vault nodes
+  copy:
+    content: "{{ hostvars[groups.vault|first]['vault_auth_ca_gen']['json']['data']['certificate'] }}"
+    dest: "{{ vault_cert_dir }}/auth-ca.pem"
+
+- name: bootstrap/gen_auth_ca | Copy auth CA key to Vault nodes
+  copy:
+    content: "{{ hostvars[groups.vault|first]['vault_auth_ca_gen']['json']['data']['private_key'] }}"
+    dest: "{{ vault_cert_dir }}/auth-ca-key.pem"

roles/vault/tasks/bootstrap/gen_ca.yml

@@ -0,0 +1,31 @@
+---
+
+- name: bootstrap/gen_ca | Ensure vault_cert_dir exists
+  file:
+    mode: 0755
+    path: "{{ vault_cert_dir }}"
+    state: directory
+
+- name: bootstrap/gen_ca | Generate Root CA in vault-temp
+  uri:
+    url: "{{ vault_leader_url }}/v1/pki/root/generate/exported"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body: "{{ vault_ca_options }}"
+  register: vault_ca_gen
+  when: inventory_hostname == groups.vault|first and vault_ca_cert_needed
+
+- name: bootstrap/gen_ca | Copy root CA cert locally
+  copy:
+    content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['certificate'] }}"
+    dest: "{{ vault_cert_dir }}/ca.pem"
+    mode: 0644
+  when: vault_ca_cert_needed
+
+- name: bootstrap/gen_ca | Copy root CA key locally
+  copy:
+    content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['private_key'] }}"
+    dest: "{{ vault_cert_dir }}/ca-key.pem"
+    mode: 0640
+  when: vault_ca_cert_needed

roles/vault/tasks/bootstrap/gen_vault_certs.yml

@@ -0,0 +1,28 @@
+---
+
+- name: boostrap/gen_vault_certs | Add the vault role
+  uri:
+    url: "{{ vault_leader_url }}/v1/pki/roles/vault"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body: "{{ vault_default_role_permissions }}"
+    status_code: 204
+  when: inventory_hostname == groups.vault|first and vault_api_cert_needed
+
+- include: ../shared/issue_cert.yml
+  vars:
+    issue_cert_alt_names: "{{ groups.vault + ['localhost'] }}"
+    issue_cert_hosts: "{{ groups.vault }}"
+    issue_cert_ip_sans: >-
+        [
+        {%- for host in groups.vault -%}
+        "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- endfor -%}
+        "127.0.0.1","::1"
+        ]
+    issue_cert_path: "{{ vault_cert_dir }}/api.pem"
+    issue_cert_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    issue_cert_role: vault
+    issue_cert_url: "{{ vault_leader_url }}"
+  when: vault_api_cert_needed

roles/vault/tasks/bootstrap/main.yml

@@ -0,0 +1,58 @@
+---
+
+- include: ../shared/check_vault.yml
+  when: inventory_hostname in groups.vault
+- include: sync_secrets.yml
+  when: inventory_hostname in groups.vault
+- include: ../shared/find_leader.yml
+  when: inventory_hostname in groups.vault and vault_cluster_is_initialized|d()
+
+## Sync Certs
+
+- include: sync_vault_certs.yml
+  when: inventory_hostname in groups.vault
+
+## Generate Certs
+
+# Start a temporary instance of Vault
+- include: start_vault_temp.yml
+  when: >-
+        inventory_hostname == groups.vault|first and
+        not vault_cluster_is_initialized
+
+# NOTE: The next 2 steps run against temp Vault and long-term Vault
+
+# Ensure PKI mount exists
+- include: ../shared/pki_mount.yml
+  when: >-
+        inventory_hostname == groups.vault|first
+
+# If the Root CA already exists, ensure Vault's PKI is using it
+- include: ../shared/config_ca.yml
+  vars:
+    ca_name: ca
+    mount_name: pki
+  when: >-
+        inventory_hostname == groups.vault|first and
+        not vault_ca_cert_needed
+
+# Generate root CA certs for Vault if none exist
+- include: gen_ca.yml
+  when: >-
+        inventory_hostname in groups.vault and
+        not vault_cluster_is_initialized and
+        vault_ca_cert_needed
+
+# Generate Vault API certs
+- include: gen_vault_certs.yml
+  when: inventory_hostname in groups.vault and vault_api_cert_needed
+
+# Update all host's CA bundle
+- include: ca_trust.yml
+
+## Add Etcd Role to Vault (if needed)
+
+- include: role_auth_cert.yml
+  when: vault_role_auth_method == "cert"
+- include: role_auth_userpass.yml
+  when: vault_role_auth_method == "userpass"

roles/vault/tasks/bootstrap/role_auth_cert.yml

@@ -0,0 +1,25 @@
+---
+
+- include: ../shared/sync_auth_certs.yml
+  when: inventory_hostname in groups.vault
+
+- include: ../shared/cert_auth_mount.yml
+  when: inventory_hostname == groups.vault|first
+
+- include: ../shared/auth_backend.yml
+  vars:
+    auth_backend_description: A Cert-based Auth primarily for services needing to issue certificates
+    auth_backend_name: cert
+    auth_backend_type: cert
+  when: inventory_hostname == groups.vault|first
+
+- include: gen_auth_ca.yml
+  when: inventory_hostname in groups.vault and vault_auth_ca_cert_needed
+
+- include: ../shared/config_ca.yml
+  vars:
+    ca_name: auth-ca
+    mount_name: auth-pki
+  when: inventory_hostname == groups.vault|first and not vault_auth_ca_cert_needed
+- include: create_etcd_role.yml
+  when: inventory_hostname in groups.etcd

roles/vault/tasks/bootstrap/role_auth_userpass.yml

@@ -0,0 +1,10 @@
+---
+
+- include: ../shared/auth_backend.yml
+  vars:
+    auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates
+    auth_backend_path: userpass
+    auth_backend_type: userpass
+  when: inventory_hostname == groups.vault|first
+- include: create_etcd_role.yml
+  when: inventory_hostname in groups.etcd

roles/vault/tasks/bootstrap/start_vault_temp.yml

@@ -0,0 +1,43 @@
+---
+
+- name: bootstrap/start_vault_temp | Ensure vault-temp isn't already running
+  shell: if docker rm -f {{ vault_temp_container_name }} 2>&1 1>/dev/null;then echo true;else echo false;fi
+  register: vault_temp_stop_check
+  changed_when: "{{ 'true' in vault_temp_stop_check.stdout }}"
+
+- name: bootstrap/start_vault_temp | Start single node Vault with file backend
+  command: >
+           docker run -d --cap-add=IPC_LOCK --name {{ vault_temp_container_name }}
+           -p {{ vault_port }}:{{ vault_port }}
+           -e 'VAULT_LOCAL_CONFIG={{ vault_temp_config|to_json }}'
+           -v /etc/vault:/etc/vault
+           {{ vault_image_repo }}:{{ vault_version }} server
+
+- name: bootstrap/start_vault_temp | Initialize vault-temp
+  uri:
+    url: "http://localhost:{{ vault_port }}/v1/sys/init"
+    headers: "{{ vault_client_headers }}"
+    method: PUT
+    body_format: json
+    body:
+      secret_shares: 1
+      secret_threshold: 1
+  register: vault_temp_init
+
+# NOTE: vault_headers and vault_url are used by subsequent issue calls
+- name: bootstrap/start_vault_temp | Set needed vault facts
+  set_fact:
+    vault_leader_url: "http://{{ inventory_hostname }}:{{ vault_port }}"
+    vault_temp_unseal_keys: "{{ vault_temp_init.json['keys'] }}"
+    vault_temp_root_token: "{{ vault_temp_init.json.root_token }}"
+    vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_temp_init.json.root_token}) }}"
+
+- name: bootstrap/start_vault_temp | Unseal vault-temp
+  uri:
+    url: "http://localhost:{{ vault_port }}/v1/sys/unseal"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body:
+      key: "{{ item }}"
+  with_items: "{{ vault_temp_unseal_keys|default([]) }}"

roles/vault/tasks/bootstrap/sync_secrets.yml

@@ -0,0 +1,48 @@
+---
+
+- include: ../shared/sync_file.yml
+  vars:
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ vault_secrets_dir }}"
+    sync_file_hosts: "{{ groups.vault }}"
+  with_items:
+    - root_token
+    - unseal_keys
+
+- name: bootstrap/sync_secrets | Set fact based on sync_file_results
+  set_fact:
+    vault_secrets_available: "{{ vault_secrets_available|default(true) and not item.no_srcs }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+
+- name: bootstrap/sync_secrets | Reset sync_file_results to avoid variable bleed
+  set_fact:
+    sync_file_results: []
+
+- name: bootstrap/sync_secrets | Print out warning message if secrets are not available and vault is initialized
+  pause:
+    prompt: >
+         Vault orchestration may not be able to proceed. The Vault cluster is initialzed, but
+         'root_token' or 'unseal_keys' were not found in {{ vault_secrets_dir }}. These are
+         needed for many vault orchestration steps.
+  when: vault_cluster_is_initialized and not vault_secrets_available
+
+- name: bootstrap/sync_secrets | Cat root_token from a vault host
+  command: "cat {{ vault_secrets_dir }}/root_token"
+  register: vault_root_token_cat
+  when: vault_secrets_available and inventory_hostname == groups.vault|first
+
+- name: bootstrap/sync_secrets | Cat unseal_keys from a vault host
+  command: "cat {{ vault_secrets_dir }}/unseal_keys"
+  register: vault_unseal_keys_cat
+  when: vault_secrets_available and inventory_hostname == groups.vault|first
+
+- name: bootstrap/sync_secrets | Set needed facts for Vault API interaction when Vault is already running
+  set_fact:
+    vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token_cat']['stdout'] }}"
+    vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys_cat']['stdout_lines'] }}"
+  when: vault_secrets_available
+
+- name: bootstrap/sync_secrets | Update vault_headers if we have the root_token
+  set_fact:
+    vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token}) }}"
+  when: vault_secrets_available

roles/vault/tasks/bootstrap/sync_vault_certs.yml

@@ -0,0 +1,32 @@
+---
+
+- include: ../shared/sync_file.yml
+  vars:
+    sync_file: "ca.pem"
+    sync_file_dir: "{{ vault_cert_dir }}"
+    sync_file_hosts: "{{ groups.vault }}"
+    sync_file_is_cert: true
+
+- name: bootstrap/sync_vault_certs | Set facts for vault sync_file results
+  set_fact:
+    vault_ca_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}"
+
+- name: bootstrap/sync_vault_certs | Unset sync_file_results after ca.pem sync
+  set_fact:
+    sync_file_results: []
+
+- include: ../shared/sync_file.yml
+  vars:
+    sync_file: "api.pem"
+    sync_file_dir: "{{ vault_cert_dir }}"
+    sync_file_hosts: "{{ groups.vault }}"
+    sync_file_is_cert: true
+
+- name: bootstrap/sync_vault_certs | Set fact if Vault's API cert is needed
+  set_fact:
+    vault_api_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}"
+
+- name: bootstrap/sync_vault_certs | Unset sync_file_results after api.pem sync
+  set_fact:
+    sync_file_results: []
+

roles/vault/tasks/cluster/binary.yml

@@ -0,0 +1,9 @@
+---
+
+- name: cluster/binary | Copy vault binary from downloaddir
+  copy:
+    src: "{{ local_release_dir }}/vault/vault"
+    dest: "/usr/bin/vault"
+    remote_src: true
+    mode: "0755"
+    owner: vault

roles/vault/tasks/cluster/configure.yml

@@ -0,0 +1,14 @@
+---
+
+- name: cluster/configure | Ensure the vault/config directory exists
+  file:
+    dest: "{{ vault_config_dir }}"
+    mode: 0750
+    state: directory
+
+- name: cluster/configure | Lay down the configuration file
+  copy:
+    content: "{{ vault_config | to_nice_json(indent=4) }}"
+    dest: "{{ vault_config_dir }}/config.json"
+    mode: 0640
+  register: vault_config_change

roles/vault/tasks/cluster/create_roles.yml

@@ -0,0 +1,9 @@
+---
+
+- include: ../shared/create_role.yml
+  vars:
+    create_role_name: "{{ item.name }}"
+    create_role_group: "{{ item.group }}"
+    create_role_policy_rules: "{{ item.policy_rules }}"
+    create_role_options: "{{ item.role_options }}"
+  with_items: "{{ vault_roles|d([]) }}"

roles/vault/tasks/cluster/init.yml

@@ -0,0 +1,52 @@
+---
+
+- name: cluster/init | Initialize Vault
+  uri:
+    url: "https://{{ groups.vault|first }}:{{ vault_port }}/v1/sys/init"
+    headers: "{{ vault_client_headers }}"
+    method: POST
+    body_format: json
+    body:
+      secret_shares: "{{ vault_secret_shares }}"
+      secret_threshold: "{{ vault_secret_threshold }}"
+    validate_certs: false
+  register: vault_init_result
+  when: not vault_cluster_is_initialized and inventory_hostname == groups.vault|first
+
+- name: cluster/init | Set facts on the results of the initialization
+  set_fact:
+    vault_unseal_keys: "{{ vault_init_result.json['keys'] }}"
+    vault_root_token: "{{ vault_init_result.json.root_token }}"
+    vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_init_result.json.root_token}) }}"
+  when: not vault_cluster_is_initialized and inventory_hostname == groups.vault|first
+
+- name: cluster/init | Ensure all hosts have these facts
+  set_fact:
+    vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys'] }}"
+    vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token'] }}"
+  when: not vault_cluster_is_initialized and inventory_hostname != groups.vault|first
+
+- name: cluster/init | Ensure the vault_secrets_dir exists
+  file:
+    mode: 0750
+    path: "{{ vault_secrets_dir }}"
+    state: directory
+
+- name: cluster/init | Ensure all in groups.vault have the unseal_keys locally
+  copy:
+    content: "{{ vault_unseal_keys|join('\n') }}"
+    dest: "{{ vault_secrets_dir }}/unseal_keys"
+    mode: 0640
+  when: not vault_cluster_is_initialized
+
+- name: cluster/init | Ensure all in groups.vault have the root_token locally
+  copy:
+    content: "{{ vault_root_token }}"
+    dest: "{{ vault_secrets_dir }}/root_token"
+    mode: 0640
+  when: not vault_cluster_is_initialized
+
+- name: cluster/init | Ensure vault_headers and vault statuses are updated
+  set_fact:
+    vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token})}}"
+    vault_cluster_is_initialized: true

roles/vault/tasks/cluster/main.yml

@@ -0,0 +1,35 @@
+---
+
+- include: ../shared/check_vault.yml
+  when: inventory_hostname in groups.vault
+- include: ../shared/check_etcd.yml
+  when: inventory_hostname in groups.vault
+
+## Vault Cluster Setup
+
+- include: configure.yml
+  when: inventory_hostname in groups.vault
+- include: binary.yml
+  when: inventory_hostname in groups.vault and vault_deployment_type == "host"
+- include: systemd.yml
+  when: inventory_hostname in groups.vault
+- include: init.yml
+  when: inventory_hostname in groups.vault
+- include: unseal.yml
+  when: inventory_hostname in groups.vault
+- include: ../shared/find_leader.yml
+  when: inventory_hostname in groups.vault
+- include: ../shared/pki_mount.yml 
+  when: inventory_hostname == groups.vault|first
+- include: ../shared/config_ca.yml
+  vars:
+    ca_name: ca
+    mount_name: pki
+  when: inventory_hostname == groups.vault|first
+
+## Vault Policies, Roles, and Auth Backends
+
+- include: role_auth_cert.yml
+  when: vault_role_auth_method == "cert"
+- include: role_auth_userpass.yml
+  when: vault_role_auth_method == "userpass"

roles/vault/tasks/cluster/role_auth_cert.yml

@@ -0,0 +1,19 @@
+---
+
+- include: ../shared/cert_auth_mount.yml
+  when: inventory_hostname == groups.vault|first
+
+- include: ../shared/auth_backend.yml
+  vars:
+    auth_backend_description: A Cert-based Auth primarily for services needing to issue certificates
+    auth_backend_name: cert
+    auth_backend_type: cert
+  when: inventory_hostname == groups.vault|first
+
+- include: ../shared/config_ca.yml
+  vars:
+    ca_name: auth-ca
+    mount_name: auth-pki
+  when: inventory_hostname == groups.vault|first
+
+- include: create_roles.yml

roles/vault/tasks/cluster/role_auth_userpass.yml

@@ -0,0 +1,10 @@
+---
+
+- include: ../shared/auth_backend.yml
+  vars:
+    auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates
+    auth_backend_path: userpass
+    auth_backend_type: userpass
+  when: inventory_hostname == groups.vault|first
+
+- include: create_roles.yml

roles/vault/tasks/cluster/systemd.yml

@@ -0,0 +1,45 @@
+---
+
+- name: cluster/systemd | Ensure mount points exist prior to vault.service startup
+  file:
+    mode: 0750
+    path: "{{ item }}"
+    state: directory
+  with_items:
+    - "{{ vault_config_dir }}"
+    - "{{ vault_log_dir }}"
+    - "{{ vault_secrets_dir }}"
+    - /var/lib/vault/
+
+- name: cluster/systemd | Ensure the vault user has access to needed directories
+  file:
+    owner: vault
+    path: "{{ item }}"
+    recurse: true
+  with_items:
+    - "{{ vault_base_dir }}"
+    - "{{ vault_log_dir }}"
+    - /var/lib/vault
+
+- name: cluster/systemd | Copy down vault.service systemd file
+  template:
+    src: "{{ vault_deployment_type }}.service.j2"
+    dest: /etc/systemd/system/vault.service
+    backup: yes
+  register: vault_systemd_placement
+
+- name: cluster/systemd | Enable vault.service
+  systemd:
+    daemon_reload: true
+    enabled: yes
+    name: vault
+    state: started
+
+- name: cluster/systemd | Query local vault until service is up
+  uri:
+    url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
+    headers: "{{ vault_client_headers }}"
+    status_code: 200,429,500,501
+  register: vault_health_check
+  until: vault_health_check|succeeded
+  retries: 10

roles/vault/tasks/cluster/unseal.yml

@@ -0,0 +1,22 @@
+---
+
+- name: cluster/unseal | Unseal Vault
+  uri:
+    url: "https://localhost:{{ vault_port }}/v1/sys/unseal"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body:
+      key: "{{ item }}"
+  with_items: "{{ vault_unseal_keys|default([]) }}"
+  when: vault_is_sealed
+
+- name: cluster/unseal | Wait until server is ready
+  uri:
+    url: "https://localhost:{{ vault_port }}/v1/sys/health"
+    headers: "{{ vault_headers }}"
+    method: HEAD
+    status_code: 200, 429
+  register: vault_node_ready
+  until: vault_node_ready|succeeded
+  retries: 5

roles/vault/tasks/main.yml

@@ -0,0 +1,19 @@
+---
+# The Vault role is typically a two step process:
+# 1. Bootstrap
+#    This starts a temporary Vault to generate certs for Vault itself. This
+#    includes a Root CA for the cluster, assuming one doesn't exist already.
+#    The temporary instance will remain running after Bootstrap, to provide a
+#    running Vault for the Etcd role to generate certs against.
+# 2. Cluster
+#    Once Etcd is started, then the Cluster tasks can start up a long-term
+#    Vault cluster using Etcd as the backend. The same Root CA is mounted as
+#    used during step 1, allowing all certs to have the same chain of trust.
+
+## Bootstrap
+- include: bootstrap/main.yml
+  when: vault_bootstrap | d()
+
+## Cluster
+- include: cluster/main.yml
+  when: not vault_bootstrap | d()

roles/vault/tasks/shared/auth_backend.yml

@@ -0,0 +1,21 @@
+---
+
+- name: shared/auth_backend | Test if the auth backend exists
+  uri:
+    url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}/tune"
+    headers: "{{ vault_headers }}"
+    validate_certs: false
+  ignore_errors: true 
+  register: vault_auth_backend_check
+
+- name: shared/auth_backend | Add the cert auth backend if needed
+  uri:
+    url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body:
+      description: "{{ auth_backend_description|d('') }}"
+      type: "{{ auth_backend_type }}"
+    status_code: 204
+  when: vault_auth_backend_check|failed

roles/vault/tasks/shared/cert_auth_mount.yml

@@ -0,0 +1,21 @@
+---
+
+- include: ../shared/mount.yml
+  vars:
+    mount_name: auth-pki
+    mount_options:
+      description: PKI mount to generate certs for the Cert Auth Backend
+      config:
+        default_lease_ttl: "{{ vault_default_lease_ttl }}"
+        max_lease_ttl: "{{ vault_max_lease_ttl }}"
+      type: pki
+
+- name: shared/auth_mount | Create a dummy role for issuing certs from auth-pki
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth-pki/roles/dummy"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: POST
+    body_format: json
+    body:
+      {'allow_any_name': true}
+    status_code: 204

roles/vault/tasks/shared/check_etcd.yml

@@ -0,0 +1,19 @@
+---
+
+- name: check_etcd | Check if etcd is up an reachable
+  uri:
+    url: "{{ vault_etcd_url }}/health"
+    validate_certs: no
+  failed_when: false
+  register: vault_etcd_health_check
+
+- name: check_etcd | Set fact based off the etcd_health_check response
+  set_fact:
+    vault_etcd_available: "{{ vault_etcd_health_check.get('json', {}).get('health')|bool  }}"
+
+- name: check_etcd | Fail if etcd is not available and needed
+  fail:
+    msg: >
+         Unable to start Vault cluster! Etcd is not available at
+         {{ vault_etcd_url }} however it is needed by Vault as a backend.
+  when: vault_etcd_needed|d() and not vault_etcd_available

roles/vault/tasks/shared/check_vault.yml

@@ -0,0 +1,31 @@
+---
+
+# Stop temporary Vault if it's running (can linger if playbook fails out)
+- name: stop vault-temp container
+  shell: docker stop {{ vault_temp_container_name }} || rkt stop {{ vault_temp_container_name }}
+  failed_when: false
+  register: vault_temp_stop
+  changed_when: vault_temp_stop|succeeded
+
+# Check if vault is reachable on the localhost
+- name: check_vault | Attempt to pull local https Vault health
+  uri:
+    url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
+    headers: "{{ vault_client_headers }}"
+    status_code: 200,429,500,501
+    validate_certs: no
+  failed_when: false
+  register: vault_local_service_health
+
+- name: check_vault | Set facts about local Vault health
+  set_fact:
+    vault_is_running: "{{ vault_local_service_health|succeeded }}"
+    vault_is_initialized: "{{ vault_local_service_health.get('json', {}).get('initialized', false) }}"
+    vault_is_sealed: "{{ vault_local_service_health.get('json', {}).get('sealed', true) }}"
+    #vault_in_standby: "{{ vault_local_service_health.get('json', {}).get('standby', true) }}"
+    #vault_run_version: "{{ vault_local_service_health.get('json', {}).get('version', '') }}"
+
+- name: check_vault | Set fact about the Vault cluster's initialization state
+  set_fact:
+    vault_cluster_is_initialized: "{{ vault_is_initialized or hostvars[item]['vault_is_initialized'] }}"
+  with_items: "{{ groups.vault }}"

roles/vault/tasks/shared/config_ca.yml

@@ -0,0 +1,30 @@
+---
+
+- name: config_ca | Read root CA cert for Vault
+  command: "cat /etc/vault/ssl/{{ ca_name }}.pem"
+  register: vault_ca_cert_cat
+
+- name: config_ca | Pull current CA cert from Vault
+  uri:
+    url: "{{ vault_leader_url }}/v1/{{ mount_name }}/ca/pem"
+    headers: "{{ vault_headers }}"
+    return_content: true
+    status_code: 200,204
+    validate_certs: no
+  register: vault_pull_current_ca
+
+- name: config_ca | Read root CA key for Vault
+  command: "cat /etc/vault/ssl/{{ ca_name }}-key.pem"
+  register: vault_ca_key_cat
+  when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.content.strip()
+
+- name: config_ca | Configure pki mount to use the found root CA cert and key
+  uri:
+    url: "{{ vault_leader_url }}/v1/{{ mount_name }}/config/ca"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body:
+      pem_bundle: "{{ vault_ca_cert_cat.stdout + '\n' + vault_ca_key_cat.stdout }}"
+    status_code: 204
+  when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.get("content","").strip()

roles/vault/tasks/shared/create_role.yml

@@ -0,0 +1,74 @@
+---
+
+# The JSON inside JSON here is intentional (Vault API wants it)
+- name: create_role | Create a policy for the new role allowing issuing
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/sys/policy/{{ create_role_name }}"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: PUT
+    body_format: json
+    body:
+      rules: >-
+             {%- if create_role_policy_rules|d("default") == "default" -%}
+             {{
+             { 'path': {
+                 'pki/issue/' + create_role_name: {'policy': 'write'},
+                 'pki/roles/' + create_role_name: {'policy': 'read'}
+             }} | to_json + '\n'
+             }}
+             {%- else -%}
+             {{ create_role_policy_rules | to_json + '\n' }}
+             {%- endif -%}
+    status_code: 204
+  when: inventory_hostname == groups[create_role_group]|first
+
+- name: create_role | Create the new role in the pki mount
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/pki/roles/{{ create_role_name }}"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: POST
+    body_format: json
+    body: >-
+          {%- if create_role_options|d("default") == "default" -%}
+          {'allow_any_name': true}
+          {%- else -%}
+          {{ create_role_options }}
+          {%- endif -%}
+    status_code: 204
+  when: inventory_hostname == groups[create_role_group]|first
+
+## Cert based auth method
+
+- include: gen_cert.yml
+  vars:
+    gen_cert_copy_ca: true
+    gen_cert_hosts: "{{ groups[create_role_group] }}"
+    gen_cert_mount: "auth-pki"
+    gen_cert_path: "{{ vault_roles_dir }}/{{ create_role_name }}/issuer.pem"
+    gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    gen_cert_vault_role: "dummy"
+    gen_cert_vault_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  when: vault_role_auth_method == "cert" and inventory_hostname in groups[create_role_group]
+
+- name: create_role | Insert the auth-pki CA as the authenticating CA for that role
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/cert/certs/{{ create_role_name }}"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: POST
+    body_format: json
+    body:
+      certificate: "{{ hostvars[groups[create_role_group]|first]['gen_cert_result']['json']['data']['issuing_ca'] }}"
+      policies: "{{ create_role_name }}"
+    status_code: 204
+  when: vault_role_auth_method == "cert" and inventory_hostname == groups[create_role_group]|first
+
+## Userpass based auth method
+
+- include: gen_userpass.yml
+  vars:
+    gen_userpass_group: "{{ create_role_group }}"
+    gen_userpass_password: "{{ create_role_password|d(''|to_uuid) }}"
+    gen_userpass_policies: "{{ create_role_name }}"
+    gen_userpass_role: "{{ create_role_name }}"
+    gen_userpass_username: "{{ create_role_name }}"
+  when: vault_role_auth_method == "userpass" and inventory_hostname in groups[create_role_group]

roles/vault/tasks/shared/find_leader.yml

@@ -0,0 +1,17 @@
+---
+
+- name: find_leader | Find the current http Vault leader
+  uri:
+    url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: HEAD
+    status_code: 200,429
+  register: vault_leader_check
+  until: "vault_leader_check|succeeded"
+  retries: 10
+
+- name: find_leader | Set fact for current http leader
+  set_fact:
+    vault_leader_url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://{{ item }}:{{ vault_port }}"
+  with_items: "{{ groups.vault }}"
+  when: "hostvars[item]['vault_leader_check'].get('status') == 200"

roles/vault/tasks/shared/gen_userpass.yml

@@ -0,0 +1,30 @@
+---
+
+- name: shared/gen_userpass | Create the Username/Password combo for the role
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/users/{{ gen_userpass_username }}"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: POST
+    body_format: json
+    body:
+      username: "{{ gen_userpass_username }}"
+      password: "{{ gen_userpass_password }}"
+      policies: "{{ gen_userpass_role }}"
+    status_code: 204
+  when: inventory_hostname == groups[gen_userpass_group]|first
+
+- name: shared/gen_userpass | Ensure destination directory exists
+  file:
+    path: "{{ vault_roles_dir }}/{{ gen_userpass_role }}"
+    state: directory
+  when: inventory_hostname in groups[gen_userpass_group]
+
+- name: shared/gen_userpass | Copy credentials to all hosts in the group
+  copy:
+    content: >
+             {{ 
+             {'username': gen_userpass_username,
+              'password': gen_userpass_password} | to_nice_json(indent=4)
+             }}
+    dest: "{{ vault_roles_dir }}/{{ gen_userpass_role }}/userpass"
+  when: inventory_hostname in groups[gen_userpass_group]

roles/vault/tasks/shared/issue_cert.yml

@@ -0,0 +1,66 @@
+---
+
+# This could be a role or custom module
+
+# Vars:
+#   issue_cert_alt_name:    Requested Subject Alternative Names, in a list.
+#   issue_cert_common_name: Common Name included in the cert
+#   issue_cert_dir_mode:    Mode of the placed cert directory
+#   issue_cert_file_group:  Group of the placed cert file and directory
+#   issue_cert_file_mode:   Mode of the placed cert file
+#   issue_cert_file_owner:  Owner of the placed cert file and directory
+#   issue_cert_format:      Format for returned data. Can be pem, der, or pem_bundle
+#   issue_cert_headers:     Headers passed into the issue request
+#   issue_cert_hosts:       List of hosts to distribute the cert to
+#   issue_cert_ip_sans:     Requested IP Subject Alternative Names, in a list
+#   issue_cert_mount:       Mount point in Vault to make the request to
+#   issue_cert_path:        Full path to the cert, include its name
+#   issue_cert_role:        The Vault role to issue the cert with
+#   issue_cert_url:         Url to reach Vault, including protocol and port
+
+- name: issue_cert | Ensure target directory exists
+  file:
+    path: "{{ issue_cert_path | dirname }}" 
+    state: directory
+    group: "{{ issue_cert_file_group | d('root' )}}"
+    mode: "{{ issue_cert_dir_mode | d('0755') }}"
+    owner: "{{ issue_cert_file_owner | d('root') }}"
+
+- name: issue_cert | Generate the cert
+  uri:
+    url: "{{ issue_cert_url }}/v1/{{ issue_cert_mount|d('pki') }}/issue/{{ issue_cert_role }}"
+    headers: "{{ issue_cert_headers }}"
+    method: POST
+    body_format: json
+    body:
+      alt_names: "{{ issue_cert_alt_names | d([]) | join(',') }}"
+      common_name: "{{ issue_cert_common_name | d(issue_cert_path.rsplit('/', 1)[1].rsplit('.', 1)[0]) }}"
+      format: "{{ issue_cert_format | d('pem') }}"
+      ip_sans: "{{ issue_cert_ip_sans | default([]) | join(',') }}"
+  register: issue_cert_result
+  when: inventory_hostname == issue_cert_hosts|first
+
+- name: issue_cert | Copy the cert to all hosts
+  copy:
+    content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['certificate'] }}"
+    dest: "{{ issue_cert_path }}"
+    group: "{{ issue_cert_file_group | d('root' )}}"
+    mode: "{{ issue_cert_file_mode | d('0644') }}"
+    owner: "{{ issue_cert_file_owner | d('root') }}"
+
+- name: issue_cert | Copy the key to all hosts
+  copy:
+    content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['private_key'] }}"
+    dest: "{{ issue_cert_path.rsplit('.', 1)|first }}-key.{{ issue_cert_path.rsplit('.', 1)|last }}"
+    group: "{{ issue_cert_file_group | d('root' )}}"
+    mode: "{{ issue_cert_file_mode | d('0640') }}"
+    owner: "{{ issue_cert_file_owner | d('root') }}"
+
+- name: issue_cert | Copy issuing CA cert
+  copy:
+    content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['issuing_ca'] }}"
+    dest: "{{ issue_cert_path | dirname }}/ca.pem"
+    group: "{{ issue_cert_file_group | d('root' )}}"
+    mode: "{{ issue_cert_file_mode | d('0644') }}"
+    owner: "{{ issue_cert_file_owner | d('root') }}"
+  when: issue_cert_copy_ca|default(false)

roles/vault/tasks/shared/mount.yml

@@ -0,0 +1,18 @@
+---
+
+- name: shared/mount | Test if PKI mount exists
+  uri:
+    url: "{{ vault_leader_url }}/v1/sys/mounts/{{ mount_name }}/tune"
+    headers: "{{ vault_headers }}"
+  ignore_errors: true
+  register: vault_pki_mount_check
+
+- name: shared/mount | Mount PKI mount if needed
+  uri:
+    url: "{{ vault_leader_url }}/v1/sys/mounts/{{ mount_name }}"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body: "{{ mount_options|d() }}"
+    status_code: 204
+  when: vault_pki_mount_check|failed

roles/vault/tasks/shared/pki_mount.yml

@@ -0,0 +1,11 @@
+---
+
+- include: mount.yml
+  vars:
+    mount_name: pki
+    mount_options:
+      config:
+        default_lease_ttl: "{{ vault_default_lease_ttl }}"
+        max_lease_ttl: "{{ vault_max_lease_ttl }}"
+      description: The default PKI mount for Kubernetes
+      type: pki

roles/vault/tasks/shared/sync.yml

@@ -0,0 +1,47 @@
+---
+
+- name: "sync_file | Cat the file"
+  command: "cat {{ sync_file_path }}"
+  register: sync_file_cat
+  when: inventory_hostname == sync_file_srcs|first
+
+- name: "sync_file | Cat the key file"
+  command: "cat {{ sync_file_key_path }}"
+  register: sync_file_key_cat
+  when: sync_file_is_cert|d() and inventory_hostname == sync_file_srcs|first
+
+- name: "sync_file | Set facts for file contents"
+  set_fact:
+    sync_file_contents: "{{ hostvars[sync_file_srcs|first]['sync_file_cat']['stdout'] }}"
+
+- name: "sync_file | Set fact for key contents"
+  set_fact:
+    sync_file_key_contents: "{{ hostvars[sync_file_srcs|first]['sync_file_key_cat']['stdout'] }}"
+  when: sync_file_is_cert|d()
+
+- name: "sync_file | Ensure the directory exists"
+  file:
+    group: "{{ sync_file_group|d('root') }}"
+    mode: "{{ sync_file_dir_mode|default('0750') }}"
+    owner: "{{ sync_file_owner|d('root') }}"
+    path: "{{ sync_file_dir }}"
+    state: directory
+  when: inventory_hostname not in sync_file_srcs
+
+- name: "sync_file | Copy the file to hosts that don't have it"
+  copy:
+    content: "{{ sync_file_contents }}"
+    dest: "{{ sync_file_path }}"
+    group: "{{ sync_file_group|d('root') }}"
+    mode: "{{ sync_file_mode|default('0640') }}"
+    owner: "{{ sync_file_owner|d('root') }}"
+  when: inventory_hostname not in sync_file_srcs
+
+- name: "sync_file | Copy the key file to hosts that don't have it"
+  copy:
+    content: "{{ sync_file_key_contents }}"
+    dest: "{{ sync_file_key_path }}"
+    group: "{{ sync_file_group|d('root') }}"
+    mode: "{{ sync_file_mode|default('0640') }}"
+    owner: "{{ sync_file_owner|d('root') }}"
+  when: sync_file_is_cert|d() and inventory_hostname not in sync_file_srcs

roles/vault/tasks/shared/sync_auth_certs.yml

@@ -0,0 +1,17 @@
+---
+
+- include: sync_file.yml
+  vars:
+    sync_file: "auth-ca.pem"
+    sync_file_dir: "{{ vault_cert_dir }}"
+    sync_file_hosts: "{{ groups.vault }}"
+    sync_file_is_cert: true
+
+- name: shared/sync_auth_certs | Set facts for vault sync_file results
+  set_fact:
+    vault_auth_ca_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}"
+
+
+- name: shared/sync_auth_certs | Unset sync_file_results after auth-ca.pem sync
+  set_fact:
+    sync_file_results: []

roles/vault/tasks/shared/sync_file.yml

@@ -0,0 +1,97 @@
+---
+
+# NOTE: This should be a role (or custom module), but currently include_role is too buggy to use
+
+- name: "sync_file | Set facts for directory and file when sync_file_path is defined"
+  set_fact:
+    sync_file_dir: "{{ sync_file_path | dirname }}"
+    sync_file: "{{ sync_file_path | basename }}"
+  when: sync_file_path is defined and sync_file_path != ''
+
+- name: "sync_file | Set fact for sync_file_path when undefined"
+  set_fact:
+    sync_file_path: "{{ (sync_file_dir, sync_file)|join('/') }}"
+  when: sync_file_path is not defined or sync_file_path == ''
+
+- name: "sync_file | Set fact for key path name"
+  set_fact:
+    sync_file_key_path: "{{ sync_file_path.rsplit('.', 1)|first + '-key.' + sync_file_path.rsplit('.', 1)|last }}"
+  when: >-
+        sync_file_is_cert|d() and (sync_file_key_path is not defined or sync_file_key_path == '')
+
+- name: "sync_file | Check if file exists"
+  stat:
+    path: "{{ sync_file_path }}"
+  register: sync_file_stat
+
+- name: "sync_file | Check if key file exists"
+  stat:
+    path: "{{ sync_file_key_path }}"
+  register: sync_file_key_stat
+  when: sync_file_is_cert|d()
+
+- name: "sync_file | Combine all possible file sync sources"
+  set_fact:
+    sync_file_srcs: "{{ sync_file_srcs|default([]) + [host_item] }}"
+  with_items: "{{ sync_file_hosts | unique }}"
+  loop_control:
+    loop_var: host_item
+  when: hostvars[host_item]["sync_file_stat"]["stat"]["exists"]|bool
+
+- name: "sync_file | Combine all possible key file sync sources"
+  set_fact:
+    sync_file_key_srcs: "{{ sync_file_key_srcs|default([]) + [host_item] }}"
+  with_items: "{{ sync_file_hosts | unique }}"
+  loop_control:
+    loop_var: host_item
+  when: sync_file_is_cert|d() and hostvars[host_item]["sync_file_key_stat"]["stat"]["exists"]|bool
+
+- name: "sync_file | Remove sync sources with files that do not match sync_file_srcs|first"
+  set_fact:
+    _: "{% if inventory_hostname in sync_file_srcs %}{{ sync_file_srcs.remove(inventory_hostname) }}{% endif %}"
+  when: >-
+        sync_file_srcs|d([])|length > 1 and
+        inventory_hostname != sync_file_srcs|first and
+        sync_file_stat.stat.get("checksum") != hostvars[sync_file_srcs|first]["sync_file_stat"]["stat"]["checksum"]
+
+- name: "sync_file | Remove sync sources with keys that do not match sync_file_srcs|first"
+  set_fact:
+    _: "{% if inventory_hostname in sync_file_srcs %}{{ sync_file_srcs.remove(inventory_hostname) }}{% endif %}"
+  when: >-
+        sync_file_is_cert|d() and
+        sync_file_key_srcs|d([])|length > 1 and
+        inventory_hostname != sync_file_key_srcs|first and
+        sync_file_key_stat.stat.checksum != hostvars[sync_file_srcs|first]["sync_file_key_stat"]["stat"]["checksum"]
+
+- name: "sync_file | Consolidate file and key sources"
+  set_fact:
+    sync_file_srcs: "{{ sync_file_srcs|d([]) | intersect(sync_file_key_srcs) }}"
+  when: sync_file_is_cert|d()
+
+- name: "sync_file | Set facts for situations where sync is not needed"
+  set_fact:
+    sync_file_no_srcs: "{{ true if sync_file_srcs|d([])|length == 0 else false }}"
+    sync_file_unneeded: "{{ true if sync_file_srcs|d([])|length == sync_file_hosts|length else false }}"
+
+- name: "sync_file | Set sync_file_result fact"
+  set_fact:
+    sync_file_result:
+      no_srcs: "{{ sync_file_no_srcs }}"
+      path: "{{ sync_file_path }}"
+      sync_unneeded: "{{ sync_file_unneeded }}"
+
+- name: "sync_file | Update sync_file_results fact"
+  set_fact:
+    sync_file_results: "{{ sync_file_results|default([]) + [sync_file_result] }}"
+
+- include: sync.yml
+  when: not (sync_file_no_srcs or sync_file_unneeded)
+
+- name: "Unset local vars to avoid variable bleed into next iteration"
+  set_fact:
+    sync_file: ''
+    sync_file_dir: ''
+    sync_file_key_path: ''
+    sync_file_key_srcs: []
+    sync_file_path: ''
+    sync_file_srcs: []

roles/vault/templates/docker.service.j2

@@ -0,0 +1,32 @@
+[Unit]
+Description=hashicorp vault on docker
+Documentation=https://github.com/hashicorp/vault
+Wants=docker.socket
+After=docker.service
+
+[Service]
+User=root
+Restart=always
+RestartSec=15s
+TimeoutStartSec=5
+LimitNOFILE=10000
+ExecReload={{ docker_bin_dir }}/docker restart {{ vault_container_name }}
+ExecStop={{ docker_bin_dir }}/docker stop {{ vault_container_name }}
+ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ vault_container_name }}
+# Container has the following internal mount points:
+#   /vault/file/    # File backend storage location
+#   /vault/logs/    # Log files
+ExecStart={{ docker_bin_dir }}/docker run \
+--name {{ vault_container_name }} --net=host \
+--cap-add=IPC_LOCK \
+-v {{ vault_cert_dir }}:{{ vault_cert_dir }} \
+-v {{ vault_config_dir }}:{{ vault_config_dir }} \
+-v {{ vault_log_dir }}:/vault/logs \
+-v {{ vault_roles_dir }}:{{ vault_roles_dir }} \
+-v {{ vault_secrets_dir }}:{{ vault_secrets_dir }} \
+--entrypoint=vault \
+{{ vault_image_repo }}:{{ vault_image_tag }} \
+server --config={{ vault_config_dir }}/config.json
+
+[Install]
+WantedBy=multi-user.target

roles/vault/templates/host.service.j2

@@ -0,0 +1,15 @@
+[Unit]
+Description=vault
+After=network.target
+
+[Service]
+AmbientCapabilities=CAP_IPC_LOCK
+ExecStart=/usr/bin/vault server --config={{ vault_config_dir }}/config.json
+LimitNOFILE=40000
+NotifyAccess=all
+Restart=always
+RestartSec=10s
+User={{ vault_adduser_vars.name }}
+
+[Install]
+WantedBy=multi-user.target

roles/vault/templates/rkt.service.j2

@@ -0,0 +1,33 @@
+[Unit]
+Description=hashicorp vault on rkt
+Documentation=https://github.com/hashicorp/vault
+Wants=network.target
+
+[Service]
+User=root
+Restart=on-failure
+RestartSec=10s
+TimeoutStartSec=5
+LimitNOFILE=40000
+# Container has the following internal mount points:
+#   /vault/file/    # File backend storage location
+#   /vault/logs/    # Log files
+ExecStart=/usr/bin/rkt run \
+--insecure-options=image \
+--volume=volume-vault-file,kind=host,source=/var/lib/vault \
+--volume=volume-vault-logs,kind=host,source={{ vault_log_dir }} \
+--volume=vault-cert-dir,kind=host,source={{ vault_cert_dir }} \
+--mount=volume=vault-cert-dir,target={{ vault_cert_dir }} \
+--volume=vault-conf-dir,kind=host,source={{ vault_config_dir }} \
+--mount=volume=vault-conf-dir,target={{ vault_config_dir }} \
+--volume=vault-secrets-dir,kind=host,source={{ vault_secrets_dir }} \
+--mount=volume=vault-secrets-dir,target={{ vault_secrets_dir }} \
+--volume=vault-roles-dir,kind=host,source={{ vault_roles_dir }} \
+--mount=volume=vault-roles-dir,target={{ vault_roles_dir }} \
+docker://{{ vault_image_repo }}:{{ vault_image_tag }} \
+--name={{ vault_container_name }} --net=host \
+--caps-retain=CAP_IPC_LOCK \
+--exec vault -- server --config={{ vault_config_dir }}/config.json
+
+[Install]
+WantedBy=multi-user.target

tests/templates/inventory-aws.j2

@@ -15,6 +15,10 @@ node3
 node1
 node2
 
+[vault]
+node1
+node2
+
 [k8s-cluster:children]
 kube-node
 kube-master

tests/templates/inventory-gce.j2

@@ -13,6 +13,9 @@ node2
 
 [etcd]
 node3
+
+[vault]
+node3
 {% elif mode is defined and mode == "ha" %}
 [kube-master]
 node1
@@ -24,6 +27,10 @@ node3
 [etcd]
 node2
 node3
+
+[vault]
+node2
+node3
 {% else %}
 [kube-master]
 node1
@@ -33,6 +40,9 @@ node2
 
 [etcd]
 node1
+
+[vault]
+node1
 {% endif %}
 
 [k8s-cluster:children]