Vault security hardening and role isolation

.gitlab-ci.yml

@@ -54,6 +54,7 @@ before_script:
   LOG_LEVEL: "-vv"
   ETCD_DEPLOYMENT: "docker"
   KUBELET_DEPLOYMENT: "docker"
+  VAULT_DEPLOYMENT: "docker"
   WEAVE_CPU_LIMIT: "100m"
   MAGIC: "ci check this"
 
@@ -106,6 +107,7 @@ before_script:
       -e ansible_python_interpreter=${PYPATH}
       -e ansible_ssh_user=${SSH_USER} 
       -e bootstrap_os=${BOOTSTRAP_OS}
+      -e cert_management=${CERT_MGMT:-script}
       -e cloud_provider=gce
       -e deploy_netchecker=true
       -e download_localhost=true
@@ -115,6 +117,7 @@ before_script:
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
       -e local_release_dir=${PWD}/downloads
       -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
       cluster.yml
 
 
@@ -292,6 +295,14 @@ before_script:
   ETCD_DEPLOYMENT: rkt
   KUBELET_DEPLOYMENT: rkt
 
+.ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: canal
+  CERT_MGMT: vault
+  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_REGION: us-central1-b
+  CLUSTER_MODE: separate
+
 # Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
 coreos-calico-sep:
   stage: deploy-gce-part1
@@ -506,6 +517,17 @@ ubuntu-rkt-sep:
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
+ubuntu-vault-sep:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_vault_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
 # Premoderated with manual actions
 ci-authorized:
   <<: *job

cluster.yml

@@ -28,14 +28,21 @@
   roles:
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: docker, tags: docker }
-    - { role: rkt, tags: rkt, when: "'rkt' in [ etcd_deployment_type, kubelet_deployment_type ]" }
+    - role: rkt
+      tags: rkt
+      when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
 
-- hosts: all
+- hosts: etcd:k8s-cluster:vault
   any_errors_fatal: true
   roles:
     - { role: vault, tags: vault, vault_bootstrap: true, when: "cert_management == 'vault'" }
 
-- hosts: etcd:k8s-cluster
+- hosts: etcd:!k8s-cluster
+  any_errors_fatal: true
+  roles:
+    - { role: etcd, tags: etcd }
+
+- hosts: k8s-cluster
   any_errors_fatal: true
   roles:
     - { role: etcd, tags: etcd }

docs/vault.md

@@ -0,0 +1,92 @@
+Hashicorp Vault Role
+====================
+
+Overview
+--------
+
+The Vault role is a two-step process:
+
+1. Bootstrap
+
+You cannot start your certificate management service securely with SSL (and 
+the datastore behind it) without having the certificates in-hand already. This
+presents an unfortunate chicken and egg scenario, with one requiring the other.
+To solve for this, the Bootstrap step was added.
+
+This step spins up a temporary instance of Vault to issue certificates for
+Vault itself. It then leaves the temporary instance running, so that the Etcd
+role can generate certs for itself as well. Eventually, this may be improved
+to allow alternate backends (such as Consul), but currently the tasks are
+hardcoded to only create a Vault role for Etcd.
+
+2. Cluster
+
+This step is where the long-term Vault cluster is started and configured. Its
+first task, is to stop any temporary instances of Vault, to free the port for
+the long-term. At the end of this task, the entire Vault cluster should be up
+and read to go.
+
+
+Keys to the Kingdom
+-------------------
+
+The two most important security pieces of Vault are the ``root_token``
+and ``unsealing_keys``. Both of these values are given exactly once, during
+the initialization of the Vault cluster. For convenience, they are saved
+to the ``vault_secret_dir`` (default: /etc/vault/secrets) of every host in the
+vault group.
+
+It is *highly* recommended that these secrets are removed from the servers after
+your cluster has been deployed, and kept in a safe location of your choosing.
+Naturally, the seriousness of the situation depends on what you're doing with
+your Kargo cluster, but with these secrets, an attacker will have the ability
+to authenticate to almost everything in Kubernetes and decode all private
+(HTTPS) traffic on your network signed by Vault certificates.
+
+For even greater security, you may want to remove and store elsewhere any
+CA keys generated as well (e.g. /etc/vault/ssl/ca-key.pem). 
+
+Vault by default encrypts all traffic to and from the datastore backend, all
+resting data, and uses TLS for its TCP listener. It is recommended that you
+do not change the Vault config to disable TLS, unless you absolutely have to.
+
+
+Usage
+-----
+
+To get the Vault role running, you must to do two things at a minimum:
+
+1. Assign the ``vault`` group to at least 1 node in your inventory
+2. Change ``cert_management`` to be ``vault`` instead of ``script``
+
+Nothing else is required, but customization is possible. Check
+``roles/vault/defaults/main.yml`` for the different variables that can be
+overridden, most common being ``vault_config``, ``vault_port``, and
+``vault_deployment_type``.
+
+Also, if you intend to use a Root or Intermediate CA generated elsewhere,
+you'll need to copy the certificate and key to the hosts in the vault group
+prior to running the vault role. By default, they'll be located at
+``/etc/vault/ssl/ca.pem`` and ``/etc/vault/ssl/ca-key.pem``, respectively.
+
+Additional Notes:
+
+- ``groups.vault|first`` is considered the source of truth for Vault variables
+- ``vault_leader_url`` is used as pointer for the current running Vault
+- Each service should have its own role and credentials. Currently those 
+  credentials are saved to ``/etc/vault/roles/<role>/``. The service will
+  need to read in those credentials, if they want to interact with Vault.
+
+
+Potential Work
+--------------
+
+- Change the Vault role to not run certain tasks when ``root_token`` and
+  ``unseal_keys`` are not present. Alternatively, allow user input for these
+  values when missing.
+- Add the ability to start temp Vault with Host, Rkt, or Docker
+- Add a dynamic way to change out the backend role creation during Bootstrap,
+  so other services can be used (such as Consul)
+- Segregate Server Cert generation from Auth Cert generation (separate CAs).
+  This work was partially started with the `auth_cert_backend` tasks, but would
+  need to be further applied to all roles (particularly Etcd and Kubernetes).

inventory/group_vars/all.yml

@@ -204,6 +204,7 @@ kpm_packages: []
 rkt_version: 1.21.0
 etcd_deployment_type: docker
 kubelet_deployment_type: docker
+vault_deployment_type: docker
 
 efk_enabled: false
 

roles/adduser/defaults/main.yml

@@ -14,14 +14,6 @@ addusers:
     system: yes
     group: "{{ kube_cert_group }}"
     createhome: no
-  vault:
-    comment: "Hashicorp Vault user"
-    createhome: no
-    name: vault
-    shell: /sbin/nologin
-    system: yes
-
-
 
 adduser:
   name: "{{ user.name }}"

roles/download/defaults/main.yml

@@ -26,7 +26,6 @@ calico_cni_version: "v1.5.5"
 weave_version: 1.8.2
 flannel_version: v0.6.2
 pod_infra_version: 3.0
-vault_version: 0.6.3
 
 # Download URL's
 etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd"

roles/download/tasks/main.yml

@@ -94,7 +94,7 @@
 
 - name: "Set default value for 'container_changed' to false"
   set_fact:
-    container_changed: "{{pull_required|bool|default(false)}}"
+    container_changed: "{{pull_required|default(false)|bool}}"
 
 - name: "Update the 'container_changed' fact"
   set_fact:

roles/etcd/defaults/main.yml

@@ -2,7 +2,6 @@
 etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"
 
 etcd_config_dir: /etc/ssl/etcd
-# Role vault.boostrap has an implicit requirement on this var. It should be set at a higher level (inventory+)
 etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
 etcd_cert_group: root
 
@@ -16,3 +15,5 @@ etcd_memory_limit: 512M
 
 # Uncomment to set CPU share for etcd
 #etcd_cpu_limit: 300m
+
+etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr', [])) }}"

roles/etcd/meta/main.yml

@@ -6,3 +6,5 @@ dependencies:
   - role: download
     file: "{{ downloads.etcd }}"
     tags: download
+
+# NOTE: Dynamic task dependency on Vault Role if cert_management == "vault"

roles/etcd/tasks/gen_certs_vault.yml

@@ -0,0 +1,77 @@
+---
+
+- name: gen_certs_vault | Read in the local credentials
+  command: cat /etc/vault/roles/etcd/userpass
+  register: etcd_vault_creds_cat
+  when: inventory_hostname == groups.etcd|first
+
+- name: gen_certs_vault | Set facts for read Vault Creds
+  set_fact:
+    etcd_vault_creds: "{{ hostvars[groups.etcd|first]['etcd_vault_creds_cat']['stdout']|from_json }}"
+  when: inventory_hostname == groups.etcd|first
+
+- name: gen_certs_vault | Log into Vault and obtain an token
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ etcd_vault_creds.username }}"
+    headers:
+      Accept: application/json
+      Content-Type: application/json 
+    method: POST
+    body_format: json
+    body:
+      password: "{{ etcd_vault_creds.password }}"
+  register: etcd_vault_login_result
+  when: inventory_hostname == groups.etcd|first
+
+- name: gen_certs_vault | Set fact for Vault API token
+  set_fact:
+    etcd_vault_headers:
+        Accept: application/json
+        Content-Type: application/json
+        X-Vault-Token: "{{ hostvars[groups.etcd|first]['etcd_vault_login_result']['json']['auth']['client_token'] }}"
+
+# Issue master certs to Etcd nodes
+- include: ../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_alt_names: "{{ groups.etcd + ['localhost'] }}"
+    issue_cert_copy_ca: "{{ item == etcd_master_certs_needed|first }}"
+    issue_cert_file_group: "{{ etcd_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ etcd_vault_headers }}" 
+    issue_cert_hosts: "{{ groups.etcd }}"
+    issue_cert_ip_sans: >-
+        [
+        {%- for host in groups.etcd  -%}
+        "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- endfor -%}
+        "127.0.0.1","::1"
+        ]
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: etcd
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ etcd_master_certs_needed|d([]) }}"
+  when: inventory_hostname in groups.etcd
+  notify: set etcd_secret_changed
+
+# Issue node certs to everyone else
+- include: ../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_alt_names: "{{ etcd_node_cert_hosts }}"
+    issue_cert_copy_ca: "{{ item == etcd_node_certs_needed|first }}"
+    issue_cert_file_group: "{{ etcd_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ etcd_vault_headers }}" 
+    issue_cert_hosts: "{{ etcd_node_cert_hosts }}"
+    issue_cert_ip_sans: >-
+        [
+        {%- for host in etcd_node_cert_hosts -%}
+        "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- endfor -%}
+        "127.0.0.1","::1"
+        ]
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: etcd
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ etcd_node_certs_needed|d([]) }}"
+  when: inventory_hostname in etcd_node_cert_hosts
+  notify: set etcd_secret_changed

roles/etcd/tasks/main.yml

@@ -1,10 +1,24 @@
 ---
 - include: pre_upgrade.yml
   tags: etcd-pre-upgrade
+
 - include: check_certs.yml
+  when: cert_management == "script"
   tags: [etcd-secrets, facts]
-- include: gen_certs.yml
+- include: gen_certs_script.yml
+  when: cert_management == "script"
   tags: etcd-secrets
+
+- include: sync_etcd_master_certs.yml
+  when: cert_management == "vault" and inventory_hostname in groups.etcd
+  tags: etcd-secrets
+- include: sync_etcd_node_certs.yml
+  when: cert_management == "vault" and inventory_hostname in etcd_node_cert_hosts 
+  tags: etcd-secrets
+- include: gen_certs_vault.yml
+  when: cert_management == "vault" and (etcd_master_certs_needed|d() or etcd_node_certs_needed|d())
+  tags: etcd-secrets
+
 - include: "install_{{ etcd_deployment_type }}.yml"
   when: is_etcd_master
   tags: upgrade

roles/etcd/tasks/sync_etcd_master_certs.yml

@@ -0,0 +1,38 @@
+---
+
+- name: sync_etcd_master_certs | Create list of master certs needing creation
+  set_fact: 
+    etcd_master_cert_list: >-
+        {{ etcd_master_cert_list|default([]) +  [
+        "admin-" + item + ".pem",
+        "member-" + item + ".pem"
+        ] }}
+  with_items: "{{ groups.etcd }}"
+
+- include: ../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ groups.etcd }}"
+    sync_file_is_cert: true
+  with_items: "{{ etcd_master_cert_list|d([]) }}"
+
+- name: sync_etcd_certs | Set facts for etcd sync_file results
+  set_fact:
+    etcd_master_certs_needed: "{{ etcd_master_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+  when: item.no_srcs|bool
+
+- name: sync_etcd_certs | Unset sync_file_results after etcd certs sync
+  set_fact:
+    sync_file_results: []
+
+- include: ../../vault/tasks/shared/sync_file.yml
+  vars:
+    sync_file: ca.pem
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ groups.etcd }}"
+
+- name: sync_etcd_certs | Unset sync_file_results after ca.pem sync
+  set_fact:
+    sync_file_results: []

roles/etcd/tasks/sync_etcd_node_certs.yml

@@ -0,0 +1,34 @@
+---
+
+- name: sync_etcd_node_certs | Create list of node certs needing creation
+  set_fact: 
+    etcd_node_cert_list: "{{ etcd_node_cert_list|default([]) +  ['node-' + item + '.pem'] }}"
+  with_items: "{{ etcd_node_cert_hosts }}"
+
+- include: ../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ etcd_node_cert_hosts }}"
+    sync_file_is_cert: true
+  with_items: "{{ etcd_node_cert_list|d([]) }}"
+
+- name: sync_etcd_node_certs | Set facts for etcd sync_file results
+  set_fact:
+    etcd_node_certs_needed: "{{ etcd_node_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+  when: item.no_srcs|bool
+
+- name: sync_etcd_node_certs | Unset sync_file_results after etcd node certs
+  set_fact:
+    sync_file_results: []
+
+- include: ../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: ca.pem
+    sync_file_dir: "{{ etcd_cert_dir }}"
+    sync_file_hosts: "{{ etcd_node_cert_hosts }}"
+
+- name: sync_etcd_node_certs | Unset sync_file_results after ca.pem
+  set_fact:
+    sync_file_results: []

roles/kubernetes/node/tasks/install.yml

@@ -25,6 +25,20 @@
   template: "src=kubelet.{{ kubelet_deployment_type }}.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes"
   notify: restart kubelet
 
+- name: install | Set SSL CA directories
+  set_fact:
+    ssl_ca_dirs: "[
+      {% if ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] -%}
+      '/usr/share/ca-certificates',
+      {% elif ansible_os_family == 'RedHat' -%}
+      '/etc/pki/tls',
+      '/etc/pki/ca-trust',
+      {% elif ansible_os_family == 'Debian' -%}
+      '/usr/share/ca-certificates',
+      {% endif -%}
+    ]"
+  tags: facts
+
 - name: install | Install kubelet launch script
   template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes
   notify: restart kubelet

roles/kubernetes/preinstall/defaults/main.yml

@@ -10,6 +10,7 @@ common_required_pkgs:
   - rsync
   - bash-completion
   - socat
+  - unzip
 
 # Set to true if your network does not support IPv6
 # This maybe necessary for pulling Docker images from

roles/kubernetes/secrets/meta/main.yml

@@ -0,0 +1,2 @@
+---
+# NOTE: Dynamic task dependency on Vault Role if cert_management == "vault"

roles/kubernetes/secrets/tasks/gen_certs_script.yml

@@ -160,20 +160,6 @@
       {%- endif %}
   tags: facts
 
-- name: SSL CA directories | Set SSL CA directories
-  set_fact:
-    ssl_ca_dirs: "[
-      {% if ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] -%}
-      '/usr/share/ca-certificates',
-      {% elif ansible_os_family == 'RedHat' -%}
-      '/etc/pki/tls',
-      '/etc/pki/ca-trust',
-      {% elif ansible_os_family == 'Debian' -%}
-      '/usr/share/ca-certificates',
-      {% endif -%}
-    ]"
-  tags: facts
-
 - name: Gen_certs | add CA to trusted CA dir
   copy:
     src: "{{ kube_cert_dir }}/ca.pem"

roles/kubernetes/secrets/tasks/gen_certs_vault.yml

@@ -0,0 +1,84 @@
+---
+
+- name: gen_certs_vault | Read in the local credentials
+  command: cat /etc/vault/roles/kube/userpass
+  register: kube_vault_creds_cat
+  when: inventory_hostname == groups['k8s-cluster']|first
+
+- name: gen_certs_vault | Set facts for read Vault Creds
+  set_fact:
+    kube_vault_creds: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_creds_cat']['stdout'] | from_json }}"
+  when: inventory_hostname == groups['k8s-cluster']|first
+
+- name: gen_certs_vault | Log into Vault and obtain an token
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ kube_vault_creds.username }}"
+    headers:
+      Accept: application/json
+      Content-Type: application/json 
+    method: POST
+    body_format: json
+    body:
+      password: "{{ kube_vault_creds.password }}"
+  register: kube_vault_login_result
+  when: inventory_hostname == groups['k8s-cluster']|first
+
+- name: gen_certs_vault | Set fact for Vault API token
+  set_fact:
+    kube_vault_headers:
+        Accept: application/json
+        Content-Type: application/json
+        X-Vault-Token: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_login_result']['json']['auth']['client_token'] }}"
+
+# Issue certs to kube-master nodes
+- include: ../../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_copy_ca: "{{ item == kube_master_certs_needed|first }}"
+    issue_cert_file_group: "{{ kube_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ kube_vault_headers }}"
+    issue_cert_hosts: "{{ groups['kube-master'] }}"
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: kube
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ kube_master_certs_needed|d([]) }}"
+  when: inventory_hostname in groups['kube-master']
+
+- include: ../../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_alt_names: >-
+        {{
+        groups['kube-master'] +
+        ['kubernetes.default.svc.cluster.local', 'kubernetes.default.svc', 'kubernetes.default', 'kubernetes'] +
+        ['localhost']
+        }}
+    issue_cert_file_group: "{{ kube_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ kube_vault_headers }}" 
+    issue_cert_hosts: "{{ groups['kube-master'] }}"
+    issue_cert_ip_sans: >-
+        [
+        {%- for host in groups['kube-master']  -%}
+        "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- endfor -%}
+        "127.0.0.1","::1","{{ kube_apiserver_ip }}"
+        ]
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: kube
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ kube_api_certs_needed|d([]) }}"
+  when: inventory_hostname in groups['kube-master']
+
+# Issue node certs to k8s-cluster nodes
+- include: ../../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_copy_ca: "{{ item == kube_node_certs_needed|first }}"
+    issue_cert_file_group: "{{ kube_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ kube_vault_headers }}" 
+    issue_cert_hosts: "{{ groups['k8s-cluster'] }}"
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: kube
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ kube_node_certs_needed|d([]) }}"
+  when: inventory_hostname in groups['k8s-cluster']

roles/kubernetes/secrets/tasks/main.yml

@@ -70,7 +70,19 @@
   delegate_to: "{{groups['kube-master'][0]}}"
   when: gen_tokens|default(false)
 
-- include: gen_certs.yml
+- include: gen_certs_script.yml
+  when: cert_management == "script"
   tags: k8s-secrets
+
+- include: sync_kube_master_certs.yml
+  when: cert_management == "vault" and inventory_hostname in groups['kube-master']
+  tags: k8s-secrets
+- include: sync_kube_node_certs.yml
+  when: cert_management == "vault" and inventory_hostname in groups['k8s-cluster']
+  tags: k8s-secrets
+- include: gen_certs_vault.yml
+  when: cert_management == "vault"
+  tags: k8s-secrets
+
 - include: gen_tokens.yml
   tags: k8s-secrets

roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml

@@ -0,0 +1,58 @@
+---
+
+- name: sync_kube_master_certs | Create list of needed kube admin certs
+  set_fact:
+    kube_master_cert_list: "{{ kube_master_cert_list|d([]) + ['admin-' + item + '.pem'] }}"
+  with_items: "{{ groups['kube-master'] }}"
+
+- include: ../../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_group: "{{ kube_cert_group }}"
+    sync_file_hosts: "{{ groups['kube-master'] }}"
+    sync_file_is_cert: true
+    sync_file_owner: kube
+  with_items: "{{ kube_master_cert_list|d([]) }}"
+
+- name: sync_kube_master_certs | Set facts for kube admin sync_file results
+  set_fact:
+    kube_master_certs_needed: "{{ kube_master_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+  when: item.no_srcs|bool
+
+- name: sync_kube_master_certs | Unset sync_file_results after kube admin certs
+  set_fact:
+    sync_file_results: []
+
+- include: ../../../vault/tasks/shared/sync_file.yml
+  vars:
+    sync_file: "apiserver.pem"
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_group: "{{ kube_cert_group }}"
+    sync_file_hosts: "{{ groups['kube-master'] }}"
+    sync_file_is_cert: true
+    sync_file_owner: kube
+
+- name: sync_kube_master_certs | Set facts for apiserver sync_file results
+  set_fact:
+    kube_api_certs_needed: "{{ item.path }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+  when: "{{ item.no_srcs }}"
+
+- name: sync_kube_master_certs | Unset sync_file_results after apiserver cert
+  set_fact:
+    sync_file_results: []
+
+
+- include: ../../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: ca.pem
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_group: "{{ kube_cert_group }}"
+    sync_file_hosts: "{{ groups['kube-master'] }}"
+    sync_file_owner: kube
+
+- name: sync_kube_master_certs | Unset sync_file_results after ca.pem
+  set_fact:
+    sync_file_results: []

roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml

@@ -0,0 +1,38 @@
+---
+
+- name: sync_kube_node_certs | Create list of needed certs
+  set_fact:
+    kube_node_cert_list: "{{ kube_node_cert_list|default([]) + ['node-' + item + '.pem'] }}"
+  with_items: "{{ groups['k8s-cluster'] }}"
+
+- include: ../../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_group: "{{ kuber_cert_group }}"
+    sync_file_hosts: "{{ groups['k8s-cluster'] }}"
+    sync_file_is_cert: true
+    sync_file_owner: kube
+  with_items: "{{ kube_node_cert_list|default([]) }}"
+
+- name: sync_kube_node_certs | Set facts for kube-master sync_file results
+  set_fact:
+    kube_node_certs_needed: "{{ kube_node_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+  when: item.no_srcs|bool
+
+- name: sync_kube_node_certs | Unset sync_file_results after kube node certs
+  set_fact:
+    sync_file_results: []
+
+- include: ../../../vault/tasks/shared/sync_file.yml
+  vars: 
+    sync_file: ca.pem
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_group: "{{ kuber_cert_group }}"
+    sync_file_hosts: "{{ groups['k8s-cluster'] }}"
+    sync_file_owner: kube
+
+- name: sync_kube_node_certs | Unset sync_file_results after ca.pem
+  set_fact:
+    sync_file_results: []

roles/vault/defaults/main.yml

@@ -1,18 +1,27 @@
 ---
 
+vault_adduser_vars:
+  comment: "Hashicorp Vault User"
+  createhome: no
+  name: vault
+  shell: /sbin/nologin
+  system: yes
+vault_base_dir: /etc/vault
+# https://releases.hashicorp.com/vault/0.6.4/vault_0.6.4_SHA256SUMS
+vault_binary_checksum: 04d87dd553aed59f3fe316222217a8d8777f40115a115dac4d88fac1611c51a6
 vault_bootstrap: false
 vault_ca_options:
   common_name: kube-cluster-ca
   format: pem
   ttl: 87600h
-vault_cert_dir: "{{ vault_config_dir }}/ssl"
+vault_cert_dir: "{{ vault_base_dir }}/ssl"
 vault_client_headers:
   Accept: "application/json"
   Content-Type: "application/json"
 vault_config:
   backend:
     etcd:
-      address: "https://{{ hostvars[groups.etcd[0]]['ansible_default_ipv4']['address'] }}:2379"
+      address: "{{ vault_etcd_url }}"
       ha_enabled: "true"
       redirect_addr: "https://{{ ansible_default_ipv4.address }}:{{ vault_port }}"
       tls_ca_file: "{{ vault_cert_dir }}/ca.pem"
@@ -23,21 +32,51 @@ vault_config:
       address: "0.0.0.0:{{ vault_port }}"
       tls_cert_file: "{{ vault_cert_dir }}/api.pem"
       tls_key_file: "{{ vault_cert_dir }}/api-key.pem"
-  max_lease_ttl: 720h
-vault_config_dir: /etc/vault
+  max_lease_ttl: "{{ vault_max_lease_ttl }}"
+vault_config_dir: "{{ vault_base_dir }}/config"
 vault_container_name: kube-hashicorp-vault
+# This variable is meant to match the GID of vault inside Hashicorp's official Vault Container
 vault_default_lease_ttl: 720h
 vault_default_role_permissions:
   allow_any_name: true
 vault_deployment_type: docker
-vault_etcd_needs_gen: false
-vault_etcd_sync_hosts: []
+vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip"
+vault_download_vars:
+  container: "{{ vault_deployment_type != 'host' }}"
+  dest: "vault/vault_{{ vault_version }}_linux_amd64.zip"
+  enabled: true
+  mode: "0755"
+  owner: "vault"
+  repo: "{{ vault_image_repo }}"
+  sha256: "{{ vault_binary_checksum if vault_deployment_type == 'host' else vault_digest_checksum|d(none) }}"
+  source_url: "{{ vault_download_url }}"
+  tag: "{{ vault_image_tag }}"
+  unarchive: true
+  url: "{{ vault_download_url }}"
+  version: "{{ vault_version }}"
+vault_etcd_url: "https://{{ hostvars[groups.etcd[0]]['ansible_default_ipv4']['address'] }}:2379"
+vault_image_repo: "vault"
+vault_image_tag: "{{ vault_version }}"
+vault_log_dir: "/var/log/vault"
 vault_max_lease_ttl: 87600h
 vault_needs_gen: false
 vault_port: 8200
+# Although "cert" is an option, ansible has no way to auth via cert until
+# upstream merges: https://github.com/ansible/ansible/pull/18141
+vault_role_auth_method: userpass 
+vault_roles:
+  - name: etcd
+    group: etcd
+    policy_rules: default
+    role_options: default
+  - name: kube
+    group: k8s-cluster
+    policy_rules: default
+    role_options: default
+vault_roles_dir: "{{ vault_base_dir }}/roles"
 vault_secret_shares: 1
 vault_secret_threshold: 1
-vault_secrets_dir: "{{ vault_config_dir }}/secrets"
+vault_secrets_dir: "{{ vault_base_dir }}/secrets"
 vault_temp_config:
   default_lease_ttl: "{{ vault_default_lease_ttl }}"
   backend:
@@ -45,21 +84,7 @@ vault_temp_config:
       path: /vault/file
   listener:
     tcp:
-      address: "0.0.0.0:{{ vault_temp_port }}"
+      address: "0.0.0.0:{{ vault_port }}"
       tls_disable: "true"
-vault_temp_port: 8201
-
-# This should be set higher up, but setting defaults here to avoid issues
-etcd_cert_dir: /etc/ssl/etcd/ssl
-kube_cert_dir: /etc/kubernetes/ssl
-
-# Sync cert defaults (should be role, once include_role is fixed)
-sync_file: ''
-sync_file_dir: ''
-sync_file_host_count: 0
-sync_file_is_cert: false
-sync_file_key_path: ''
-sync_file_key_srcs: []
-sync_file_path: ''
-sync_file_results: []
-sync_file_srcs: []
+vault_temp_container_name: vault-temp
+vault_version: 0.6.4

roles/vault/meta/main.yml

@@ -1,6 +1,8 @@
 ---
-# Implicit requirement on sync_cert role (include_role used in tasks)
 
 dependencies:
+  - role: adduser
+    user: "{{ vault_adduser_vars }}"
   - role: download
-    file: "{{ downloads.vault }}"
+    file: "{{ vault_download_vars }}"
+    tags: download

roles/vault/tasks/bootstrap/ca_trust.yml

@@ -1,12 +1,12 @@
 ---
 
-- name: trust_ca | pull CA from cert from groups.vault|first
+- name: bootstrap/ca_trust | pull CA from cert from groups.vault|first
   command: "cat {{ vault_cert_dir }}/ca.pem"
   register: vault_cert_file_cat
   when: inventory_hostname == groups.vault|first
 
 # This part is mostly stolen from the etcd role
-- name: trust_ca | target ca-certificate store file
+- name: bootstrap/ca_trust | target ca-certificate store file
   set_fact:
     ca_cert_path: >-
       {% if ansible_os_family == "Debian" -%}
@@ -17,16 +17,16 @@
       /etc/ssl/certs/kube-cluster-ca.pem
       {%- endif %}
 
-- name: trust_ca | add CA to trusted CA dir
+- name: bootstrap/ca_trust | add CA to trusted CA dir
   copy:
     content: "{{ hostvars[groups.vault|first]['vault_cert_file_cat']['stdout'] }}"
     dest: "{{ ca_cert_path }}"
   register: vault_ca_cert
 
-- name: trust_ca | update ca-certificates (Debian/Ubuntu/CoreOS)
+- name: bootstrap/ca_trust | update ca-certificates (Debian/Ubuntu/CoreOS)
   command: update-ca-certificates
   when: vault_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"]
 
-- name: trust_ca | update ca-certificates (RedHat)
+- name: bootstrap/ca_trust | update ca-certificates (RedHat)
   command: update-ca-trust extract
   when: vault_ca_cert.changed and ansible_os_family == "RedHat"

roles/vault/tasks/bootstrap/create_etcd_role.yml

@@ -0,0 +1,10 @@
+---
+
+- include: ../shared/create_role.yml
+  vars:
+    create_role_name: "{{ item.name }}" 
+    create_role_group: "{{ item.group }}"
+    create_role_policy_rules: "{{ item.policy_rules }}"
+    create_role_options: "{{ item.role_options }}"
+  with_items: "{{ vault_roles }}"
+  when: item.name == "etcd"

roles/vault/tasks/bootstrap/gen_auth_ca.yml

@@ -0,0 +1,21 @@
+---
+
+- name: bootstrap/gen_auth_ca | Generate Root CA
+  uri:
+    url: "{{ vault_leader_url }}/v1/auth-pki/root/generate/exported"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body: "{{ vault_ca_options }}"
+  register: vault_auth_ca_gen
+  when: inventory_hostname == groups.vault|first
+
+- name: bootstrap/gen_auth_ca | Copy auth CA cert to Vault nodes
+  copy:
+    content: "{{ hostvars[groups.vault|first]['vault_auth_ca_gen']['json']['data']['certificate'] }}"
+    dest: "{{ vault_cert_dir }}/auth-ca.pem"
+
+- name: bootstrap/gen_auth_ca | Copy auth CA key to Vault nodes
+  copy:
+    content: "{{ hostvars[groups.vault|first]['vault_auth_ca_gen']['json']['data']['private_key'] }}"
+    dest: "{{ vault_cert_dir }}/auth-ca-key.pem"

roles/vault/tasks/bootstrap/gen_ca.yml

@@ -0,0 +1,31 @@
+---
+
+- name: bootstrap/gen_ca | Ensure vault_cert_dir exists
+  file:
+    mode: 0755
+    path: "{{ vault_cert_dir }}"
+    state: directory
+
+- name: bootstrap/gen_ca | Generate Root CA in vault-temp
+  uri:
+    url: "{{ vault_leader_url }}/v1/pki/root/generate/exported"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body: "{{ vault_ca_options }}"
+  register: vault_ca_gen
+  when: inventory_hostname == groups.vault|first and vault_ca_cert_needed
+
+- name: bootstrap/gen_ca | Copy root CA cert locally
+  copy:
+    content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['certificate'] }}"
+    dest: "{{ vault_cert_dir }}/ca.pem"
+    mode: 0644
+  when: vault_ca_cert_needed
+
+- name: bootstrap/gen_ca | Copy root CA key locally
+  copy:
+    content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['private_key'] }}"
+    dest: "{{ vault_cert_dir }}/ca-key.pem"
+    mode: 0640
+  when: vault_ca_cert_needed

roles/vault/tasks/bootstrap/gen_etcd_certs.yml

@@ -1,29 +0,0 @@
----
-
-- name: bootstrap/gen_etcd_certs | Add the etcd role
-  uri:
-    url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}/v1/pki/roles/etcd"
-    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    method: POST
-    body_format: json
-    body:
-      allow_any_name: true
-    status_code: 204
-  when: inventory_hostname == groups.etcd|first
-
-- include: ../gen_cert.yml
-  vars:
-    gen_cert_alt_names: "{{ groups.etcd | join(',') }},localhost"
-    gen_cert_copy_ca: "{{ true if item == vault_etcd_certs_needed|first else false }}"
-    gen_cert_hosts: "{{ groups.etcd }}"
-    gen_cert_ip_sans: >-
-        {%- for host in groups.etcd  -%}
-        {{ hostvars[host]["ansible_default_ipv4"]["address"] }}
-        {%- if not loop.last -%},{%- endif -%}
-        {%- endfor -%}
-        ,127.0.0.1,::1
-    gen_cert_path: "{{ item }}"
-    gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    gen_cert_vault_role: etcd
-    gen_cert_vault_url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}"
-  with_items: "{{ vault_etcd_certs_needed|default([]) }}"

roles/vault/tasks/bootstrap/gen_etcd_node_certs.yml

@@ -1,29 +0,0 @@
----
-
-- name: bootstrap/gen_etcd_node_certs | Add the etcd role
-  uri:
-    url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}/v1/pki/roles/etcd"
-    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    method: POST
-    body_format: json
-    body:
-      allow_any_name: true
-    status_code: 204
-  when: inventory_hostname == groups["k8s-cluster"]|first
-
-- include: ../gen_cert.yml
-  vars:
-    gen_cert_alt_names: "{{ groups['k8s-cluster'] | union(groups.etcd) | join(',') }},localhost"
-    gen_cert_copy_ca: "{{ true if item == vault_etcd_node_certs_needed|first else false }}"
-    gen_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.etcd) }}"
-    gen_cert_ip_sans: >-
-        {%- for host in groups["k8s-cluster"] | union(groups.etcd) -%}
-        {{ hostvars[host]["ansible_default_ipv4"]["address"] }}
-        {%- if not loop.last -%},{%- endif -%}
-        {%- endfor -%}
-        ,127.0.0.1,::1
-    gen_cert_path: "{{ item }}"
-    gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    gen_cert_vault_role: etcd
-    gen_cert_vault_url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}"
-  with_items: "{{ vault_etcd_node_certs_needed|default([]) }}"

roles/vault/tasks/bootstrap/gen_vault_certs.yml

@@ -1,47 +1,8 @@
 ---
 
-- name: bootstrap/gen_vault_certs | Ensure vault_cert_dir exists
-  file:
-    path: "{{ vault_cert_dir }}"
-    state: directory
-
-- name: bootstrap/gen_vault_certs | Generate Root CA in vault-temp
-  uri:
-    url: "http://localhost:{{ vault_temp_port }}/v1/pki/root/generate/exported"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body: "{{ vault_ca_options }}"
-  register: vault_ca_gen
-  when: inventory_hostname == groups.vault|first and vault_ca_cert_needed
-
-- name: bootstrap/gen_vault_certs | Set facts for ca cert and key
-  set_fact:
-    vault_ca_cert: "{{ vault_ca_gen.json.data.certificate }}"
-    vault_ca_key: "{{ vault_ca_gen.json.data.private_key }}"
-  when: inventory_hostname == groups.vault|first and vault_ca_cert_needed
-
-- name: bootstrap/gen_vault_certs | Set cert and key facts for all hosts other than groups.vault|first
-  set_fact:
-    vault_ca_cert: "{{ hostvars[groups.vault|first]['vault_ca_cert'] }}"
-    vault_ca_key: "{{ hostvars[groups.vault|first]['vault_ca_key'] }}"
-  when: inventory_hostname != groups.vault|first and vault_ca_cert_needed
-
-- name: bootstrap/gen_vault_certs | Copy root CA cert locally
-  copy:
-    content: "{{ vault_ca_cert }}"
-    dest: "{{ vault_cert_dir }}/ca.pem"
-  when: vault_ca_cert_needed
-
-- name: bootstrap/gen_vault_certs | Copy root CA key locally
-  copy:
-    content: "{{vault_ca_key}}"
-    dest: "{{vault_cert_dir}}/ca-key.pem"
-  when: vault_ca_cert_needed
-
 - name: boostrap/gen_vault_certs | Add the vault role
   uri:
-    url: "http://localhost:{{ vault_temp_port }}/v1/pki/roles/vault"
+    url: "{{ vault_leader_url }}/v1/pki/roles/vault"
     headers: "{{ vault_headers }}"
     method: POST
     body_format: json
@@ -49,18 +10,19 @@
     status_code: 204
   when: inventory_hostname == groups.vault|first and vault_api_cert_needed
 
-- include: ../gen_cert.yml
+- include: ../shared/issue_cert.yml
   vars:
-    gen_cert_alt_names: "{{ groups.vault | join(',') }},localhost"
-    gen_cert_hosts: "{{ groups.vault }}"
-    gen_cert_ip_sans: >-
+    issue_cert_alt_names: "{{ groups.vault + ['localhost'] }}"
+    issue_cert_hosts: "{{ groups.vault }}"
+    issue_cert_ip_sans: >-
+        [
         {%- for host in groups.vault -%}
-        {{ hostvars[host]["ansible_default_ipv4"]["address"] }}
-        {%- if not loop.last -%},{%- endif -%}
+        "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
         {%- endfor -%}
-        ,127.0.0.1,::1
-    gen_cert_path: "{{ vault_cert_dir }}/api.pem"
-    gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    gen_cert_vault_role: vault
-    gen_cert_vault_url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}"
+        "127.0.0.1","::1"
+        ]
+    issue_cert_path: "{{ vault_cert_dir }}/api.pem"
+    issue_cert_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    issue_cert_role: vault
+    issue_cert_url: "{{ vault_leader_url }}"
   when: vault_api_cert_needed

roles/vault/tasks/bootstrap/main.yml

@@ -1,60 +1,58 @@
 ---
 
+- include: ../shared/check_vault.yml
+  when: inventory_hostname in groups.vault
+- include: sync_secrets.yml
+  when: inventory_hostname in groups.vault
+- include: ../shared/find_leader.yml
+  when: inventory_hostname in groups.vault and vault_cluster_is_initialized|d()
+
 ## Sync Certs
 
-- include: bootstrap/sync_vault_certs.yml
+- include: sync_vault_certs.yml
   when: inventory_hostname in groups.vault
 
-- include: bootstrap/sync_etcd_certs.yml
-  when: inventory_hostname in groups.etcd
-
-- include: bootstrap/sync_etcd_node_certs.yml
-  when: inventory_hostname in groups["k8s-cluster"] | union(groups.etcd)
-
 ## Generate Certs
 
 # Start a temporary instance of Vault
-- include: bootstrap/start_vault_temp.yml
-  when: >-
-        ( hostvars[groups.etcd|first].get("vault_etcd_certs_needed", [])|length > 0 or
-        hostvars[groups.etcd|first].get("vault_etcd_node_certs_needed", [])|length > 0 or
-        hostvars[groups.vault|first]["vault_ca_cert_needed"] ) and
-        inventory_hostname == groups.vault|first
-
-# Generate root CA certs for Vault if none exist
-- include: bootstrap/gen_vault_certs.yml
-  when: >-
-        ( hostvars[groups.vault|first]["vault_ca_cert_needed"] or
-        hostvars[groups.vault|first]["vault_api_cert_needed"] ) and
-        inventory_hostname in groups.vault
-
-# Change vault-temp's issuing CA to use existing ca.pem/ca-key.pem
-- include: config_ca.yml
-  vars:
-    vault_url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}"
-  when: >-
-        ( hostvars[groups.etcd|first].get("vault_etcd_certs_needed", [])|length > 0 or
-        hostvars[groups["k8s-cluster"]|first].get("vault_etcd_node_certs_needed", [])|length > 0 or
-        hostvars[groups.vault|first]["vault_api_cert_needed"] ) and
-        not hostvars[groups.vault|first]["vault_ca_cert_needed"] and
-        inventory_hostname == groups.vault|first
-
-# Generate etcd certs for etcd cluster members
-- include: bootstrap/gen_etcd_certs.yml
-  when: >- 
-        hostvars[groups.etcd|first].get("vault_etcd_certs_needed", [])|length > 0 and
-        inventory_hostname in groups.etcd
-
-# Generate etcd node certs for all k8s-cluster
-- include: bootstrap/gen_etcd_node_certs.yml
-  when: >-
-        hostvars[groups["k8s-cluster"]|first].get("vault_etcd_node_certs_needed", [])|length > 0 and
-        inventory_hostname in groups["k8s-cluster"] | union(groups.etcd)
-
-# Stop temporary vault
-- include: bootstrap/stop_vault_temp.yml
+- include: start_vault_temp.yml
   when: >-
         inventory_hostname == groups.vault|first and
-        hostvars[groups.vault|first]["vault_temp_start"]|succeeded
+        not vault_cluster_is_initialized
 
+# NOTE: The next 2 steps run against temp Vault and long-term Vault
+
+# Ensure PKI mount exists
+- include: ../shared/pki_mount.yml
+  when: >-
+        inventory_hostname == groups.vault|first
+
+# If the Root CA already exists, ensure Vault's PKI is using it
+- include: ../shared/config_ca.yml
+  vars:
+    ca_name: ca
+    mount_name: pki
+  when: >-
+        inventory_hostname == groups.vault|first and
+        not vault_ca_cert_needed
+
+# Generate root CA certs for Vault if none exist
+- include: gen_ca.yml
+  when: >-
+        inventory_hostname in groups.vault and
+        not vault_cluster_is_initialized and
+        vault_ca_cert_needed
+
+# Generate Vault API certs
+- include: gen_vault_certs.yml
+  when: inventory_hostname in groups.vault and vault_api_cert_needed
+
+# Update all host's CA bundle
 - include: ca_trust.yml
+
+## Add Etcd Role to Vault (if needed)
+
+- include: role_auth_cert.yml
+  when: vault_role_auth_method == "cert"
+- include: role_auth_userpass.yml
+  when: vault_role_auth_method == "userpass"

roles/vault/tasks/bootstrap/role_auth_cert.yml

@@ -0,0 +1,25 @@
+---
+
+- include: ../shared/sync_auth_certs.yml
+  when: inventory_hostname in groups.vault
+
+- include: ../shared/cert_auth_mount.yml
+  when: inventory_hostname == groups.vault|first
+
+- include: ../shared/auth_backend.yml
+  vars:
+    auth_backend_description: A Cert-based Auth primarily for services needing to issue certificates
+    auth_backend_name: cert
+    auth_backend_type: cert
+  when: inventory_hostname == groups.vault|first
+
+- include: gen_auth_ca.yml
+  when: inventory_hostname in groups.vault and vault_auth_ca_cert_needed
+
+- include: ../shared/config_ca.yml
+  vars:
+    ca_name: auth-ca
+    mount_name: auth-pki
+  when: inventory_hostname == groups.vault|first and not vault_auth_ca_cert_needed
+- include: create_etcd_role.yml
+  when: inventory_hostname in groups.etcd

roles/vault/tasks/bootstrap/role_auth_userpass.yml

@@ -0,0 +1,10 @@
+---
+
+- include: ../shared/auth_backend.yml
+  vars:
+    auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates
+    auth_backend_path: userpass
+    auth_backend_type: userpass
+  when: inventory_hostname == groups.vault|first
+- include: create_etcd_role.yml
+  when: inventory_hostname in groups.etcd

roles/vault/tasks/bootstrap/start_vault_temp.yml

@@ -1,21 +1,21 @@
 ---
 
-- name: boostrap/start_vault_temp | Ensure vault-temp isn't already running
-  shell: if docker rm -f vault-temp 2>&1 1>/dev/null;then echo true;else echo false;fi
+- name: bootstrap/start_vault_temp | Ensure vault-temp isn't already running
+  shell: if docker rm -f {{ vault_temp_container_name }} 2>&1 1>/dev/null;then echo true;else echo false;fi
   register: vault_temp_stop_check
   changed_when: "{{ 'true' in vault_temp_stop_check.stdout }}"
 
 - name: bootstrap/start_vault_temp | Start single node Vault with file backend
   command: >
-           docker run -d --cap-add=IPC_LOCK --name vault-temp -p {{ vault_temp_port }}:{{ vault_temp_port }}
+           docker run -d --cap-add=IPC_LOCK --name {{ vault_temp_container_name }}
+           -p {{ vault_port }}:{{ vault_port }}
            -e 'VAULT_LOCAL_CONFIG={{ vault_temp_config|to_json }}'
            -v /etc/vault:/etc/vault
            {{ vault_image_repo }}:{{ vault_version }} server
-  register: vault_temp_start
 
 - name: bootstrap/start_vault_temp | Initialize vault-temp
   uri:
-    url: "http://localhost:{{ vault_temp_port }}/v1/sys/init"
+    url: "http://localhost:{{ vault_port }}/v1/sys/init"
     headers: "{{ vault_client_headers }}"
     method: PUT
     body_format: json
@@ -24,32 +24,20 @@
       secret_threshold: 1
   register: vault_temp_init
 
-# NOTE: vault_headers and vault_url are used by subsequent gen_cert calls
+# NOTE: vault_headers and vault_url are used by subsequent issue calls
 - name: bootstrap/start_vault_temp | Set needed vault facts
   set_fact:
+    vault_leader_url: "http://{{ inventory_hostname }}:{{ vault_port }}"
     vault_temp_unseal_keys: "{{ vault_temp_init.json['keys'] }}"
     vault_temp_root_token: "{{ vault_temp_init.json.root_token }}"
     vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_temp_init.json.root_token}) }}"
 
 - name: bootstrap/start_vault_temp | Unseal vault-temp
   uri:
-    url: "http://localhost:{{ vault_temp_port }}/v1/sys/unseal"
+    url: "http://localhost:{{ vault_port }}/v1/sys/unseal"
     headers: "{{ vault_headers }}"
     method: POST
     body_format: json
     body:
       key: "{{ item }}"
   with_items: "{{ vault_temp_unseal_keys|default([]) }}"
-
-- name: bootstrap/start_vault_temp | Create new PKI mount
-  uri:
-    url: "http://localhost:{{ vault_temp_port }}/v1/sys/mounts/pki"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body:
-      config:
-        default_lease_ttl: "{{ vault_default_lease_ttl }}"
-        max_lease_ttl: "{{ vault_max_lease_ttl }}"
-      type: pki
-    status_code: 204

roles/vault/tasks/bootstrap/stop_vault_temp.yml

@@ -1,4 +0,0 @@
----
-
-- name: stop vault-temp container
-  command: docker stop vault-temp

roles/vault/tasks/bootstrap/sync_etcd_certs.yml

@@ -1,38 +0,0 @@
----
-
-- name: bootstrap/sync_etcd_certs | Create list of certs needing creation
-  set_fact: 
-    vault_etcd_cert_list: >-
-        {{ vault_etcd_cert_list|default([]) +  [
-        "admin-" + item + ".pem",
-        "member-" + item + ".pem"
-        ] }}
-  with_items: "{{ groups.etcd }}"
-
-- include: ../sync_file.yml
-  vars: 
-    sync_file: "{{ item }}"
-    sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: "{{ groups.etcd }}"
-    sync_file_is_cert: true
-  with_items: "{{ vault_etcd_cert_list|default([]) }}"
-
-- name: bootstrap/sync_etcd_certs | Set facts for etcd sync_file results
-  set_fact:
-    vault_etcd_certs_needed: "{{ vault_etcd_certs_needed|default([]) + [item.path] }}"
-  with_items: "{{ sync_file_results }}"
-  when: item.no_srcs|bool
-
-- name: bootstrap/sync_etcd_certs | Unset sync_file_results after etcd certs sync
-  set_fact:
-    sync_file_results: []
-
-- include: ../sync_file.yml
-  vars:
-    sync_file: ca.pem
-    sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: "{{ groups.etcd }}"
-
-- name: bootstrap/sync_etcd_certs | Unset sync_file_results after ca.pem sync
-  set_fact:
-    sync_file_results: []

roles/vault/tasks/bootstrap/sync_etcd_node_certs.yml

@@ -1,34 +0,0 @@
----
-
-- name: bootstrap/sync_etcd_node_certs | Create list of certs needing creation
-  set_fact: 
-    vault_etcd_node_cert_list: "{{ vault_etcd_node_cert_list|default([]) +  ['node-' + item + '.pem'] }}"
-  with_items: "{{ groups['k8s-cluster'] | union(groups.etcd) }}"
-
-- include: ../sync_file.yml
-  vars: 
-    sync_file: "{{ item }}"
-    sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: "{{ groups['k8s-cluster'] | union(groups.etcd) }}"
-    sync_file_is_cert: true
-  with_items: "{{ vault_etcd_node_cert_list|default([]) }}"
-
-- name: bootstrap/sync_etcd_node_certs | Set facts for etcd sync_file results
-  set_fact:
-    vault_etcd_node_certs_needed: "{{ vault_etcd_node_certs_needed|default([]) + [item.path] }}"
-  with_items: "{{ sync_file_results }}"
-  when: item.no_srcs|bool
-
-- name: bootstrap/sync_etcd_node_certs | Unset sync_file_results after etcd node certs
-  set_fact:
-    sync_file_results: []
-
-- include: ../sync_file.yml
-  vars: 
-    sync_file: ca.pem
-    sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: "{{ groups['k8s-cluster']| union(groups.etcd) }}"
-
-- name: bootstrap/sync_etcd_node_certs | Unset sync_file_results after ca.pem
-  set_fact:
-    sync_file_results: []

roles/vault/tasks/bootstrap/sync_secrets.yml

@@ -0,0 +1,48 @@
+---
+
+- include: ../shared/sync_file.yml
+  vars:
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ vault_secrets_dir }}"
+    sync_file_hosts: "{{ groups.vault }}"
+  with_items:
+    - root_token
+    - unseal_keys
+
+- name: bootstrap/sync_secrets | Set fact based on sync_file_results
+  set_fact:
+    vault_secrets_available: "{{ vault_secrets_available|default(true) and not item.no_srcs }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+
+- name: bootstrap/sync_secrets | Reset sync_file_results to avoid variable bleed
+  set_fact:
+    sync_file_results: []
+
+- name: bootstrap/sync_secrets | Print out warning message if secrets are not available and vault is initialized
+  pause:
+    prompt: >
+         Vault orchestration may not be able to proceed. The Vault cluster is initialzed, but
+         'root_token' or 'unseal_keys' were not found in {{ vault_secrets_dir }}. These are
+         needed for many vault orchestration steps.
+  when: vault_cluster_is_initialized and not vault_secrets_available
+
+- name: bootstrap/sync_secrets | Cat root_token from a vault host
+  command: "cat {{ vault_secrets_dir }}/root_token"
+  register: vault_root_token_cat
+  when: vault_secrets_available and inventory_hostname == groups.vault|first
+
+- name: bootstrap/sync_secrets | Cat unseal_keys from a vault host
+  command: "cat {{ vault_secrets_dir }}/unseal_keys"
+  register: vault_unseal_keys_cat
+  when: vault_secrets_available and inventory_hostname == groups.vault|first
+
+- name: bootstrap/sync_secrets | Set needed facts for Vault API interaction when Vault is already running
+  set_fact:
+    vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token_cat']['stdout'] }}"
+    vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys_cat']['stdout_lines'] }}"
+  when: vault_secrets_available
+
+- name: bootstrap/sync_secrets | Update vault_headers if we have the root_token
+  set_fact:
+    vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token}) }}"
+  when: vault_secrets_available

roles/vault/tasks/bootstrap/sync_vault_certs.yml

@@ -1,21 +1,21 @@
 ---
 
-- include: ../sync_file.yml
+- include: ../shared/sync_file.yml
   vars:
     sync_file: "ca.pem"
     sync_file_dir: "{{ vault_cert_dir }}"
     sync_file_hosts: "{{ groups.vault }}"
     sync_file_is_cert: true
 
-- name: "bootstrap/sync_vault_certs | Set facts for vault sync_file results"
+- name: bootstrap/sync_vault_certs | Set facts for vault sync_file results
   set_fact:
-    vault_ca_cert_needed: "{{ true if sync_file_results|length > 0 else false }}"
+    vault_ca_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}"
 
 - name: bootstrap/sync_vault_certs | Unset sync_file_results after ca.pem sync
   set_fact:
     sync_file_results: []
 
-- include: ../sync_file.yml
+- include: ../shared/sync_file.yml
   vars:
     sync_file: "api.pem"
     sync_file_dir: "{{ vault_cert_dir }}"
@@ -24,7 +24,7 @@
 
 - name: bootstrap/sync_vault_certs | Set fact if Vault's API cert is needed
   set_fact:
-    vault_api_cert_needed: "{{ true if sync_file_results|length > 0 else false }}"
+    vault_api_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}"
 
 - name: bootstrap/sync_vault_certs | Unset sync_file_results after api.pem sync
   set_fact:

roles/vault/tasks/check_vault.yml

@@ -1,77 +0,0 @@
----
-
-# Check if vault is reachable on the localhost
-- name: check_vault | Attempt to pull local vault health
-  uri:
-    url: "https://localhost:{{ vault_port }}/v1/sys/health"
-    headers: "{{ vault_client_headers }}"
-    validate_certs: no
-  ignore_errors: true
-  register: vault_local_service_health
-
-- name: check_vault | Set facts about local Vault health
-  set_fact:
-    vault_is_running: "{{ vault_local_service_health|succeeded }}"
-    vault_is_initialized: "{{ vault_local_service_health.get('json', {}).get('initialized', false) }}"
-    vault_is_sealed: "{{ vault_local_service_health.get('json', {}).get('sealed', true) }}"
-    vault_in_standby: "{{ vault_local_service_health.get('json', {}).get('standby', true) }}"
-    vault_run_version: "{{ vault_local_service_health.get('json', {}).get('version', '') }}"
-
-- name: check_vault | Set fact about the Vault cluster's initialization state
-  set_fact:
-    vault_cluster_is_initialized: "{{ vault_is_initialized or hostvars[item]['vault_is_initialized'] }}"
-  with_items: "{{ groups.vault }}"
-
-- name: check_vault | Set fact about the Vault Cluster's available hosts
-  set_fact:
-    vault_available_hosts: "{{ vault_available_hosts|default([]) + [item] }}"
-  with_items: "{{ groups.vault }}"
-  when: "hostvars[item]['vault_is_running'] and not hostvars[item]['vault_is_sealed']"
-
-- include: sync_file.yml
-  vars:
-    sync_file: "{{ item }}"
-    sync_file_dir: "{{ vault_secrets_dir }}"
-    sync_file_hosts: "{{ groups.vault }}"
-  with_items:
-    - root_token
-    - unseal_keys
-
-# Logic is hard to follow on this one, probably need to simplify somehow
-- name: "check_vault | Set fact based on sync_file_results"
-  set_fact:
-    vault_secrets_available: "{{ vault_secrets_available|default(true) and not item.no_srcs }}"
-  with_items: "{{ sync_file_results }}"
-
-- name: "check_vault | Reset sync_file_results to avoid variable bleed"
-  set_fact:
-    sync_file_results: []
-
-- name: "check_vault | Print out warning message if secrets are not available"
-  pause:
-    prompt: >
-         Vault orchestration may not be able to proceed. The Vault cluster is initialzed, but
-         'root_token' or 'unseal_keys' were not found in {{ vault_secrets_dir }}. These are
-         needed for many orchestration steps.
-  when: vault_cluster_is_initialized and not vault_secrets_available
-
-- name: "check_vault | Cat root_token from a vault host"
-  command: "cat {{ vault_secrets_dir }}/root_token"
-  register: vault_root_token_cat
-  when: vault_secrets_available and inventory_hostname == groups.vault|first
-
-- name: "check_vault | Cat unseal_keys from a vault host"
-  command: "cat {{ vault_secrets_dir }}/unseal_keys"
-  register: vault_unseal_keys_cat
-  when: vault_secrets_available and inventory_hostname == groups.vault|first
-
-- name: "check_vault | Set needed facts for Vault API interaction when Vault is already running"
-  set_fact:
-    vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token_cat']['stdout'] }}"
-    vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys_cat']['stdout_lines'] }}"
-  when: vault_secrets_available
-
-- name: "check-vault | Update vault_headers if we have the root_token"
-  set_fact:
-    vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token}) }}"
-  when: vault_secrets_available

roles/vault/tasks/cluster/binary.yml

@@ -0,0 +1,9 @@
+---
+
+- name: cluster/binary | Copy vault binary from downloaddir
+  copy:
+    src: "{{ local_release_dir }}/vault/vault"
+    dest: "/usr/bin/vault"
+    remote_src: true
+    mode: "0755"
+    owner: vault

roles/vault/tasks/cluster/configure.yml

@@ -0,0 +1,14 @@
+---
+
+- name: cluster/configure | Ensure the vault/config directory exists
+  file:
+    dest: "{{ vault_config_dir }}"
+    mode: 0750
+    state: directory
+
+- name: cluster/configure | Lay down the configuration file
+  copy:
+    content: "{{ vault_config | to_nice_json(indent=4) }}"
+    dest: "{{ vault_config_dir }}/config.json"
+    mode: 0640
+  register: vault_config_change

roles/vault/tasks/cluster/create_roles.yml

@@ -0,0 +1,9 @@
+---
+
+- include: ../shared/create_role.yml
+  vars:
+    create_role_name: "{{ item.name }}"
+    create_role_group: "{{ item.group }}"
+    create_role_policy_rules: "{{ item.policy_rules }}"
+    create_role_options: "{{ item.role_options }}"
+  with_items: "{{ vault_roles|d([]) }}"

roles/vault/tasks/cluster/docker.yml

@@ -1,25 +0,0 @@
----
-
-- name: docker | Check on state of docker instance
-  command: "docker inspect {{ vault_container_name }}"
-  ignore_errors: true
-  register: vault_container_inspect
-
-- name: docker | Set fact on container status
-  set_fact:
-      vault_container_inspect_json: "{{ vault_container_inspect.stdout|from_json }}"
-  when: vault_container_inspect|succeeded
-
-# Not sure if State.Running is the best check here...
-- name: docker | Remove old container if it's not currently running
-  command: "docker rm {{ vault_container_name }}"
-  when: vault_container_inspect|succeeded and not vault_container_inspect_json[0]["State"]["Running"]|bool
-
-- name: docker | Start a new Vault instance
-  command: >
-           docker run -d --cap-add=IPC_LOCK --name {{vault_container_name}} -p {{vault_port}}:{{vault_port}}
-           -e 'VAULT_LOCAL_CONFIG={{ vault_config|to_json }}'
-           -v /etc/vault:/etc/vault
-           {{vault_image_repo}}:{{vault_version}} server
-  register: vault_docker_start
-  when: vault_container_inspect|failed or not vault_container_inspect_json[0]["State"]["Running"]|bool

roles/vault/tasks/cluster/gen_kube_master_certs.yml

@@ -1,33 +0,0 @@
----
-
-- name: "cluster/gen_kube_node_certs | Ensure kube_cert_dir exists"
-  file:
-    path: "{{ kube_cert_dir }}" 
-    state: directory
-
-- name: gen_kube_master_certs | Add the kube role
-  uri:
-    url: "https://{{ hostvars[groups.vault|first]['vault_leader'] }}:{{ vault_port }}/v1/pki/roles/kubernetes"
-    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    method: POST
-    body_format: json
-    body: "{{ vault_default_role_permissions }}"
-    status_code: 204
-  when: inventory_hostname == groups["kube-master"]|first
-
-- include: ../gen_cert.yml
-  vars:
-    gen_cert_alt_names: "{{ groups['kube-master'] | join(',') }},localhost"
-    gen_cert_copy_ca: "{{ true if item == vault_kube_master_certs_needed|first else false }}"
-    gen_cert_hosts: "{{ groups['kube-master'] }}"
-    gen_cert_ip_sans: >-
-        {%- for host in groups["kube-master"] -%}
-        {{ hostvars[host]["ansible_default_ipv4"]["address"] }}
-        {%- if not loop.last -%},{%- endif -%}
-        {%- endfor -%}
-        ,127.0.0.1,::1
-    gen_cert_path: "{{ item }}"
-    gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    gen_cert_vault_role: kubernetes
-    gen_cert_vault_url: "https://{{ hostvars[groups.vault|first]['vault_leader'] }}:{{ vault_port }}"
-  with_items: "{{ vault_kube_master_certs_needed|default([]) }}"

roles/vault/tasks/cluster/gen_kube_node_certs.yml

@@ -1,33 +0,0 @@
----
-
-- name: "cluster/gen_kube_node_certs | Ensure kube_cert_dir exists"
-  file:
-    path: "{{ kube_cert_dir }}" 
-    state: directory
-
-- name: "cluster/gen_kube_node_certs | Add the kubernetes role"
-  uri:
-    url: "https://{{ hostvars[groups.vault|first]['vault_leader'] }}:{{ vault_port }}/v1/pki/roles/kubernetes"
-    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    method: POST
-    body_format: json
-    body: "{{ vault_default_role_permissions }}"
-    status_code: 204
-  when: inventory_hostname == groups["k8s-cluster"]|first
-
-- include: ../gen_cert.yml
-  vars:
-    gen_cert_alt_names: "{{ groups['k8s-cluster'] | join(',') }},localhost"
-    gen_cert_copy_ca: "{{ true if item == vault_kube_node_certs_needed|first else false }}"
-    gen_cert_hosts: "{{ groups['k8s-cluster'] }}"
-    gen_cert_ip_sans: >-
-        {%- for host in groups["k8s-cluster"] -%}
-        {{ hostvars[host]["ansible_default_ipv4"]["address"] }}
-        {%- if not loop.last -%},{%- endif -%}
-        {%- endfor -%}
-        ,127.0.0.1,::1
-    gen_cert_path: "{{ item }}"
-    gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    gen_cert_vault_role: kubernetes
-    gen_cert_vault_url: "https://{{ hostvars[groups.vault|first]['vault_leader'] }}:{{ vault_port }}"
-  with_items: "{{ vault_kube_node_certs_needed|default([]) }}"

roles/vault/tasks/cluster/init.yml

@@ -28,6 +28,7 @@
 
 - name: cluster/init | Ensure the vault_secrets_dir exists
   file:
+    mode: 0750
     path: "{{ vault_secrets_dir }}"
     state: directory
 
@@ -35,12 +36,14 @@
   copy:
     content: "{{ vault_unseal_keys|join('\n') }}"
     dest: "{{ vault_secrets_dir }}/unseal_keys"
+    mode: 0640
   when: not vault_cluster_is_initialized
 
 - name: cluster/init | Ensure all in groups.vault have the root_token locally
   copy:
     content: "{{ vault_root_token }}"
     dest: "{{ vault_secrets_dir }}/root_token"
+    mode: 0640
   when: not vault_cluster_is_initialized
 
 - name: cluster/init | Ensure vault_headers and vault statuses are updated

roles/vault/tasks/cluster/main.yml

@@ -1,30 +1,35 @@
 ---
 
+- include: ../shared/check_vault.yml
+  when: inventory_hostname in groups.vault
+- include: ../shared/check_etcd.yml
+  when: inventory_hostname in groups.vault
+
 ## Vault Cluster Setup
 
-- include: docker.yml
-  when: inventory_hostname in groups.vault and vault_deployment_type == "docker"
+- include: configure.yml
+  when: inventory_hostname in groups.vault
+- include: binary.yml
+  when: inventory_hostname in groups.vault and vault_deployment_type == "host"
+- include: systemd.yml
+  when: inventory_hostname in groups.vault
 - include: init.yml
   when: inventory_hostname in groups.vault
 - include: unseal.yml
   when: inventory_hostname in groups.vault
-- include: pki_mount.yml
-  when: 'inventory_hostname == hostvars[groups.vault|first]["vault_leader"]'
-- include: config_ca.yml
+- include: ../shared/find_leader.yml
+  when: inventory_hostname in groups.vault
+- include: ../shared/pki_mount.yml 
+  when: inventory_hostname == groups.vault|first
+- include: ../shared/config_ca.yml
   vars:
-    vault_url: "https://{{ vault_leader }}:{{ vault_port }}"
-  when: 'inventory_hostname == hostvars[groups.vault|first]["vault_leader"]'
+    ca_name: ca
+    mount_name: pki
+  when: inventory_hostname == groups.vault|first
 
-## Sync Kubernetes Certs
+## Vault Policies, Roles, and Auth Backends
 
-- include: sync_kube_master_certs.yml
-  when: inventory_hostname in groups["kube-master"]
-- include: sync_kube_node_certs.yml
-  when: inventory_hostname in groups["k8s-cluster"]
-
-## Generate Kubernetes Certs
-
-- include: gen_kube_master_certs.yml
-  when: inventory_hostname in groups["kube-master"]
-- include: gen_kube_node_certs.yml
-  when: inventory_hostname in groups["k8s-cluster"]
+- include: role_auth_cert.yml
+  when: vault_role_auth_method == "cert"
+- include: role_auth_userpass.yml
+  when: vault_role_auth_method == "userpass"

roles/vault/tasks/cluster/pki_mount.yml

@@ -1,23 +0,0 @@
----
-
-- name: cluster/pki_mount | Test if default PKI mount exists
-  uri:
-    url: "https://localhost:{{ vault_port }}/v1/sys/mounts/pki/tune"
-    headers: "{{ vault_headers }}"
-    validate_certs: false
-  ignore_errors: true
-  register: vault_pki_mount_check
-
-- name: cluster/pki_mount | Mount default PKI mount if needed
-  uri:
-    url: "https://localhost:{{ vault_port }}/v1/sys/mounts/pki"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body:
-      config:
-        default_lease_ttl: "{{ vault_default_lease_ttl }}"
-        max_lease_ttl: "{{ vault_max_lease_ttl }}"
-      type: pki
-    status_code: 204
-  when: vault_pki_mount_check | failed

roles/vault/tasks/cluster/role_auth_cert.yml

@@ -0,0 +1,19 @@
+---
+
+- include: ../shared/cert_auth_mount.yml
+  when: inventory_hostname == groups.vault|first
+
+- include: ../shared/auth_backend.yml
+  vars:
+    auth_backend_description: A Cert-based Auth primarily for services needing to issue certificates
+    auth_backend_name: cert
+    auth_backend_type: cert
+  when: inventory_hostname == groups.vault|first
+
+- include: ../shared/config_ca.yml
+  vars:
+    ca_name: auth-ca
+    mount_name: auth-pki
+  when: inventory_hostname == groups.vault|first
+
+- include: create_roles.yml

roles/vault/tasks/cluster/role_auth_userpass.yml

@@ -0,0 +1,10 @@
+---
+
+- include: ../shared/auth_backend.yml
+  vars:
+    auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates
+    auth_backend_path: userpass
+    auth_backend_type: userpass
+  when: inventory_hostname == groups.vault|first
+
+- include: create_roles.yml

roles/vault/tasks/cluster/sync_kube_master_certs.yml

@@ -1,38 +0,0 @@
----
-
-- name: cluster/sync_kube_master_certs | Create list of needed certs
-  set_fact:
-    vault_kube_master_cert_list: >-
-        {{  vault_kube_master_cert_list|default([]) + [
-        "admin-" + item + ".pem",
-        "apiserver-" + item + ".pem"
-        ] }}
-  with_items: "{{ groups['kube-master'] }}"
-
-- include: ../sync_file.yml
-  vars: 
-    sync_file: "{{ item }}"
-    sync_file_dir: "{{ kube_cert_dir }}"
-    sync_file_hosts: "{{ groups['kube-master'] }}"
-    sync_file_is_cert: true
-  with_items: "{{ vault_kube_master_cert_list|default([]) }}"
-
-- name: cluster/sync_kube_master_certs | Set facts for kube-master sync_file results
-  set_fact:
-    vault_kube_master_certs_needed: "{{ vault_kube_master_certs_needed|default([]) + [item.path] }}"
-  with_items: "{{ sync_file_results }}"
-  when: item.no_srcs|bool
-
-- name: cluster/sync_kube_master_certs | Unset sync_file_results after kube master certs
-  set_fact:
-    sync_file_results: []
-
-- include: ../sync_file.yml
-  vars: 
-    sync_file: ca.pem
-    sync_file_dir: "{{ kube_cert_dir }}"
-    sync_file_hosts: "{{ groups['kube-master'] }}"
-
-- name: cluster/sync_kube_master_certs | Unset sync_file_results after ca.pem
-  set_fact:
-    sync_file_results: []

roles/vault/tasks/cluster/sync_kube_node_certs.yml

@@ -1,34 +0,0 @@
----
-
-- name: cluster/sync_kube_node_certs | Create list of needed certs
-  set_fact:
-    vault_kube_node_cert_list: "{{ vault_kube_node_cert_list|default([]) + ['node-' + item + '.pem'] }}"
-  with_items: "{{ groups['k8s-cluster'] }}"
-
-- include: ../sync_file.yml
-  vars: 
-    sync_file: "{{ item }}"
-    sync_file_dir: "{{ kube_cert_dir }}"
-    sync_file_hosts: "{{ groups['k8s-cluster'] }}"
-    sync_file_is_cert: true
-  with_items: "{{ vault_kube_node_cert_list|default([]) }}"
-
-- name: cluster/sync_kube_node_certs | Set facts for kube-master sync_file results
-  set_fact:
-    vault_kube_node_certs_needed: "{{ vault_kube_node_certs_needed|default([]) + [item.path] }}"
-  with_items: "{{ sync_file_results }}"
-  when: item.no_srcs|bool
-
-- name: cluster/sync_kube_node_certs | Unset sync_file_results after kube node certs
-  set_fact:
-    sync_file_results: []
-
-- include: ../sync_file.yml
-  vars: 
-    sync_file: ca.pem
-    sync_file_dir: "{{ kube_cert_dir }}"
-    sync_file_hosts: "{{ groups['k8s-cluster'] }}"
-
-- name: cluster/sync_kube_node_certs | Unset sync_file_results after ca.pem
-  set_fact:
-    sync_file_results: []

roles/vault/tasks/cluster/systemd.yml

@@ -0,0 +1,45 @@
+---
+
+- name: cluster/systemd | Ensure mount points exist prior to vault.service startup
+  file:
+    mode: 0750
+    path: "{{ item }}"
+    state: directory
+  with_items:
+    - "{{ vault_config_dir }}"
+    - "{{ vault_log_dir }}"
+    - "{{ vault_secrets_dir }}"
+    - /var/lib/vault/
+
+- name: cluster/systemd | Ensure the vault user has access to needed directories
+  file:
+    owner: vault
+    path: "{{ item }}"
+    recurse: true
+  with_items:
+    - "{{ vault_base_dir }}"
+    - "{{ vault_log_dir }}"
+    - /var/lib/vault
+
+- name: cluster/systemd | Copy down vault.service systemd file
+  template:
+    src: "{{ vault_deployment_type }}.service.j2"
+    dest: /etc/systemd/system/vault.service
+    backup: yes
+  register: vault_systemd_placement
+
+- name: cluster/systemd | Enable vault.service
+  systemd:
+    daemon_reload: true
+    enabled: yes
+    name: vault
+    state: started
+
+- name: cluster/systemd | Query local vault until service is up
+  uri:
+    url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
+    headers: "{{ vault_client_headers }}"
+    status_code: 200,429,500,501
+  register: vault_health_check
+  until: vault_health_check|succeeded
+  retries: 10

roles/vault/tasks/cluster/unseal.yml

@@ -11,16 +11,12 @@
   with_items: "{{ vault_unseal_keys|default([]) }}"
   when: vault_is_sealed
 
-- name: cluster/unseal | Find the current leader
+- name: cluster/unseal | Wait until server is ready
   uri:
     url: "https://localhost:{{ vault_port }}/v1/sys/health"
     headers: "{{ vault_headers }}"
     method: HEAD
     status_code: 200, 429
-  register: vault_leader_check
-
-- name: cluster/unseal | Set fact for current leader
-  set_fact:
-    vault_leader: "{{ item }}"
-  with_items: "{{ groups.vault }}"
-  when: 'hostvars[item]["vault_leader_check"]["status"] == 200'
+  register: vault_node_ready
+  until: vault_node_ready|succeeded
+  retries: 5

roles/vault/tasks/config_ca.yml

@@ -1,19 +0,0 @@
----
-
-- name: config_ca | Read root CA cert for Vault
-  command: cat /etc/vault/ssl/ca.pem
-  register: vault_ca_cert_cat
-
-- name: config_ca | Read root CA key for Vault
-  command: cat /etc/vault/ssl/ca-key.pem
-  register: vault_ca_key_cat
-
-- name: config_ca | Configure pki mount to use the found root CA cert and key
-  uri:
-    url: "{{ vault_url }}/v1/pki/config/ca"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body:
-      pem_bundle: "{{ vault_ca_cert_cat.stdout + '\n' + vault_ca_key_cat.stdout }}"
-    status_code: 204

roles/vault/tasks/gen_cert.yml

@@ -1,50 +0,0 @@
----
-
-# This could be a role or custom module
-
-- name: gen_cert | Ensure target directory exists
-  file:
-    path: "{{ gen_cert_path | dirname }}" 
-    state: directory
-
-- name: gen_cert | Generate the cert
-  uri:
-    url: "{{ gen_cert_vault_url}}/v1/pki/issue/{{ gen_cert_vault_role }}"
-    headers: "{{ gen_cert_vault_headers }}"
-    method: POST
-    body_format: json
-    body:
-      alt_names: "{{ gen_cert_alt_names|default([]) }}"
-      common_name: "{{ gen_cert_path.rsplit('/', 1)[1].rsplit('.', 1)[0] }}"
-      format: "{{ gen_cert_format|default('pem') }}"
-      ip_sans: "{{ gen_cert_ip_sans|default([]) }}"
-  register: gen_cert_result
-  when: inventory_hostname == gen_cert_hosts|first
-
-- name: gen_cert | Copy the cert to all hosts
-  copy:
-    content: "{{ hostvars[gen_cert_hosts|first]['gen_cert_result']['json']['data']['certificate'] }}"
-    dest: "{{ gen_cert_path }}"
-
-- name: gen_cert | Copy the key to all hosts
-  copy:
-    content: "{{ hostvars[gen_cert_hosts|first]['gen_cert_result']['json']['data']['private_key'] }}"
-    dest: "{{ gen_cert_path.rsplit('.', 1)|first + '-key.' + gen_cert_path.rsplit('.', 1)|last }}"
-
-- name: gen_cert | Copy issuing CA cert
-  copy:
-    content: "{{ hostvars[gen_cert_hosts|first]['gen_cert_result']['json']['data']['issuing_ca'] }}"
-    dest: "{{ gen_cert_path | dirname }}/ca.pem"
-  when: gen_cert_copy_ca|default(false)|bool
-
-- name: gen_cert | Unset common variables to avoid bleed over
-  set_fact:
-    gen_cert_copy_ca: false
-    gen_cert_alt_names: []
-    gen_cert_format: pem
-    gen_cert_hosts: []
-    gen_cert_ip_sans: []
-    gen_cert_path: ''
-    gen_cert_vault_headers: ''
-    gen_cert_vault_role: ''
-    gen_cert_vault_url: ''

roles/vault/tasks/main.yml

@@ -1,13 +1,19 @@
 ---
+# The Vault role is typically a two step process:
+# 1. Bootstrap
+#    This starts a temporary Vault to generate certs for Vault itself. This
+#    includes a Root CA for the cluster, assuming one doesn't exist already.
+#    The temporary instance will remain running after Bootstrap, to provide a
+#    running Vault for the Etcd role to generate certs against.
+# 2. Cluster
+#    Once Etcd is started, then the Cluster tasks can start up a long-term
+#    Vault cluster using Etcd as the backend. The same Root CA is mounted as
+#    used during step 1, allowing all certs to have the same chain of trust.
 
-- include: check_vault.yml
-  when: inventory_hostname in groups.vault
-
-# bootstrap.yml's sole purpose is to ensure certs exist for Vault and Etcd
-# prior to startup, so TLS can be enabled.
+## Bootstrap
 - include: bootstrap/main.yml
-  when: vault_bootstrap|bool
+  when: vault_bootstrap | d()
 
-# cluster.yml should only run after the backend service is ready (default etcd)
+## Cluster
 - include: cluster/main.yml
-  when: not vault_bootstrap|bool
+  when: not vault_bootstrap | d()

roles/vault/tasks/shared/auth_backend.yml

@@ -0,0 +1,21 @@
+---
+
+- name: shared/auth_backend | Test if the auth backend exists
+  uri:
+    url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}/tune"
+    headers: "{{ vault_headers }}"
+    validate_certs: false
+  ignore_errors: true 
+  register: vault_auth_backend_check
+
+- name: shared/auth_backend | Add the cert auth backend if needed
+  uri:
+    url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body:
+      description: "{{ auth_backend_description|d('') }}"
+      type: "{{ auth_backend_type }}"
+    status_code: 204
+  when: vault_auth_backend_check|failed

roles/vault/tasks/shared/cert_auth_mount.yml

@@ -0,0 +1,21 @@
+---
+
+- include: ../shared/mount.yml
+  vars:
+    mount_name: auth-pki
+    mount_options:
+      description: PKI mount to generate certs for the Cert Auth Backend
+      config:
+        default_lease_ttl: "{{ vault_default_lease_ttl }}"
+        max_lease_ttl: "{{ vault_max_lease_ttl }}"
+      type: pki
+
+- name: shared/auth_mount | Create a dummy role for issuing certs from auth-pki
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth-pki/roles/dummy"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: POST
+    body_format: json
+    body:
+      {'allow_any_name': true}
+    status_code: 204

roles/vault/tasks/shared/check_etcd.yml

@@ -0,0 +1,19 @@
+---
+
+- name: check_etcd | Check if etcd is up an reachable
+  uri:
+    url: "{{ vault_etcd_url }}/health"
+    validate_certs: no
+  failed_when: false
+  register: vault_etcd_health_check
+
+- name: check_etcd | Set fact based off the etcd_health_check response
+  set_fact:
+    vault_etcd_available: "{{ vault_etcd_health_check.get('json', {}).get('health')|bool  }}"
+
+- name: check_etcd | Fail if etcd is not available and needed
+  fail:
+    msg: >
+         Unable to start Vault cluster! Etcd is not available at
+         {{ vault_etcd_url }} however it is needed by Vault as a backend.
+  when: vault_etcd_needed|d() and not vault_etcd_available

roles/vault/tasks/shared/check_vault.yml

@@ -0,0 +1,31 @@
+---
+
+# Stop temporary Vault if it's running (can linger if playbook fails out)
+- name: stop vault-temp container
+  shell: docker stop {{ vault_temp_container_name }} || rkt stop {{ vault_temp_container_name }}
+  failed_when: false
+  register: vault_temp_stop
+  changed_when: vault_temp_stop|succeeded
+
+# Check if vault is reachable on the localhost
+- name: check_vault | Attempt to pull local https Vault health
+  uri:
+    url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
+    headers: "{{ vault_client_headers }}"
+    status_code: 200,429,500,501
+    validate_certs: no
+  failed_when: false
+  register: vault_local_service_health
+
+- name: check_vault | Set facts about local Vault health
+  set_fact:
+    vault_is_running: "{{ vault_local_service_health|succeeded }}"
+    vault_is_initialized: "{{ vault_local_service_health.get('json', {}).get('initialized', false) }}"
+    vault_is_sealed: "{{ vault_local_service_health.get('json', {}).get('sealed', true) }}"
+    #vault_in_standby: "{{ vault_local_service_health.get('json', {}).get('standby', true) }}"
+    #vault_run_version: "{{ vault_local_service_health.get('json', {}).get('version', '') }}"
+
+- name: check_vault | Set fact about the Vault cluster's initialization state
+  set_fact:
+    vault_cluster_is_initialized: "{{ vault_is_initialized or hostvars[item]['vault_is_initialized'] }}"
+  with_items: "{{ groups.vault }}"

roles/vault/tasks/shared/config_ca.yml

@@ -0,0 +1,30 @@
+---
+
+- name: config_ca | Read root CA cert for Vault
+  command: "cat /etc/vault/ssl/{{ ca_name }}.pem"
+  register: vault_ca_cert_cat
+
+- name: config_ca | Pull current CA cert from Vault
+  uri:
+    url: "{{ vault_leader_url }}/v1/{{ mount_name }}/ca/pem"
+    headers: "{{ vault_headers }}"
+    return_content: true
+    status_code: 200,204
+    validate_certs: no
+  register: vault_pull_current_ca
+
+- name: config_ca | Read root CA key for Vault
+  command: "cat /etc/vault/ssl/{{ ca_name }}-key.pem"
+  register: vault_ca_key_cat
+  when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.content.strip()
+
+- name: config_ca | Configure pki mount to use the found root CA cert and key
+  uri:
+    url: "{{ vault_leader_url }}/v1/{{ mount_name }}/config/ca"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body:
+      pem_bundle: "{{ vault_ca_cert_cat.stdout + '\n' + vault_ca_key_cat.stdout }}"
+    status_code: 204
+  when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.get("content","").strip()

roles/vault/tasks/shared/create_role.yml

@@ -0,0 +1,74 @@
+---
+
+# The JSON inside JSON here is intentional (Vault API wants it)
+- name: create_role | Create a policy for the new role allowing issuing
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/sys/policy/{{ create_role_name }}"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: PUT
+    body_format: json
+    body:
+      rules: >-
+             {%- if create_role_policy_rules|d("default") == "default" -%}
+             {{
+             { 'path': {
+                 'pki/issue/' + create_role_name: {'policy': 'write'},
+                 'pki/roles/' + create_role_name: {'policy': 'read'}
+             }} | to_json + '\n'
+             }}
+             {%- else -%}
+             {{ create_role_policy_rules | to_json + '\n' }}
+             {%- endif -%}
+    status_code: 204
+  when: inventory_hostname == groups[create_role_group]|first
+
+- name: create_role | Create the new role in the pki mount
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/pki/roles/{{ create_role_name }}"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: POST
+    body_format: json
+    body: >-
+          {%- if create_role_options|d("default") == "default" -%}
+          {'allow_any_name': true}
+          {%- else -%}
+          {{ create_role_options }}
+          {%- endif -%}
+    status_code: 204
+  when: inventory_hostname == groups[create_role_group]|first
+
+## Cert based auth method
+
+- include: gen_cert.yml
+  vars:
+    gen_cert_copy_ca: true
+    gen_cert_hosts: "{{ groups[create_role_group] }}"
+    gen_cert_mount: "auth-pki"
+    gen_cert_path: "{{ vault_roles_dir }}/{{ create_role_name }}/issuer.pem"
+    gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    gen_cert_vault_role: "dummy"
+    gen_cert_vault_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  when: vault_role_auth_method == "cert" and inventory_hostname in groups[create_role_group]
+
+- name: create_role | Insert the auth-pki CA as the authenticating CA for that role
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/cert/certs/{{ create_role_name }}"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: POST
+    body_format: json
+    body:
+      certificate: "{{ hostvars[groups[create_role_group]|first]['gen_cert_result']['json']['data']['issuing_ca'] }}"
+      policies: "{{ create_role_name }}"
+    status_code: 204
+  when: vault_role_auth_method == "cert" and inventory_hostname == groups[create_role_group]|first
+
+## Userpass based auth method
+
+- include: gen_userpass.yml
+  vars:
+    gen_userpass_group: "{{ create_role_group }}"
+    gen_userpass_password: "{{ create_role_password|d(''|to_uuid) }}"
+    gen_userpass_policies: "{{ create_role_name }}"
+    gen_userpass_role: "{{ create_role_name }}"
+    gen_userpass_username: "{{ create_role_name }}"
+  when: vault_role_auth_method == "userpass" and inventory_hostname in groups[create_role_group]

roles/vault/tasks/shared/find_leader.yml

@@ -0,0 +1,17 @@
+---
+
+- name: find_leader | Find the current http Vault leader
+  uri:
+    url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: HEAD
+    status_code: 200,429
+  register: vault_leader_check
+  until: "vault_leader_check|succeeded"
+  retries: 10
+
+- name: find_leader | Set fact for current http leader
+  set_fact:
+    vault_leader_url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://{{ item }}:{{ vault_port }}"
+  with_items: "{{ groups.vault }}"
+  when: "hostvars[item]['vault_leader_check'].get('status') == 200"

roles/vault/tasks/shared/gen_userpass.yml

@@ -0,0 +1,30 @@
+---
+
+- name: shared/gen_userpass | Create the Username/Password combo for the role
+  uri:
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/users/{{ gen_userpass_username }}"
+    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
+    method: POST
+    body_format: json
+    body:
+      username: "{{ gen_userpass_username }}"
+      password: "{{ gen_userpass_password }}"
+      policies: "{{ gen_userpass_role }}"
+    status_code: 204
+  when: inventory_hostname == groups[gen_userpass_group]|first
+
+- name: shared/gen_userpass | Ensure destination directory exists
+  file:
+    path: "{{ vault_roles_dir }}/{{ gen_userpass_role }}"
+    state: directory
+  when: inventory_hostname in groups[gen_userpass_group]
+
+- name: shared/gen_userpass | Copy credentials to all hosts in the group
+  copy:
+    content: >
+             {{ 
+             {'username': gen_userpass_username,
+              'password': gen_userpass_password} | to_nice_json(indent=4)
+             }}
+    dest: "{{ vault_roles_dir }}/{{ gen_userpass_role }}/userpass"
+  when: inventory_hostname in groups[gen_userpass_group]

roles/vault/tasks/shared/issue_cert.yml

@@ -0,0 +1,66 @@
+---
+
+# This could be a role or custom module
+
+# Vars:
+#   issue_cert_alt_name:    Requested Subject Alternative Names, in a list.
+#   issue_cert_common_name: Common Name included in the cert
+#   issue_cert_dir_mode:    Mode of the placed cert directory
+#   issue_cert_file_group:  Group of the placed cert file and directory
+#   issue_cert_file_mode:   Mode of the placed cert file
+#   issue_cert_file_owner:  Owner of the placed cert file and directory
+#   issue_cert_format:      Format for returned data. Can be pem, der, or pem_bundle
+#   issue_cert_headers:     Headers passed into the issue request
+#   issue_cert_hosts:       List of hosts to distribute the cert to
+#   issue_cert_ip_sans:     Requested IP Subject Alternative Names, in a list
+#   issue_cert_mount:       Mount point in Vault to make the request to
+#   issue_cert_path:        Full path to the cert, include its name
+#   issue_cert_role:        The Vault role to issue the cert with
+#   issue_cert_url:         Url to reach Vault, including protocol and port
+
+- name: issue_cert | Ensure target directory exists
+  file:
+    path: "{{ issue_cert_path | dirname }}" 
+    state: directory
+    group: "{{ issue_cert_file_group | d('root' )}}"
+    mode: "{{ issue_cert_dir_mode | d('0755') }}"
+    owner: "{{ issue_cert_file_owner | d('root') }}"
+
+- name: issue_cert | Generate the cert
+  uri:
+    url: "{{ issue_cert_url }}/v1/{{ issue_cert_mount|d('pki') }}/issue/{{ issue_cert_role }}"
+    headers: "{{ issue_cert_headers }}"
+    method: POST
+    body_format: json
+    body:
+      alt_names: "{{ issue_cert_alt_names | d([]) | join(',') }}"
+      common_name: "{{ issue_cert_common_name | d(issue_cert_path.rsplit('/', 1)[1].rsplit('.', 1)[0]) }}"
+      format: "{{ issue_cert_format | d('pem') }}"
+      ip_sans: "{{ issue_cert_ip_sans | default([]) | join(',') }}"
+  register: issue_cert_result
+  when: inventory_hostname == issue_cert_hosts|first
+
+- name: issue_cert | Copy the cert to all hosts
+  copy:
+    content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['certificate'] }}"
+    dest: "{{ issue_cert_path }}"
+    group: "{{ issue_cert_file_group | d('root' )}}"
+    mode: "{{ issue_cert_file_mode | d('0644') }}"
+    owner: "{{ issue_cert_file_owner | d('root') }}"
+
+- name: issue_cert | Copy the key to all hosts
+  copy:
+    content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['private_key'] }}"
+    dest: "{{ issue_cert_path.rsplit('.', 1)|first }}-key.{{ issue_cert_path.rsplit('.', 1)|last }}"
+    group: "{{ issue_cert_file_group | d('root' )}}"
+    mode: "{{ issue_cert_file_mode | d('0640') }}"
+    owner: "{{ issue_cert_file_owner | d('root') }}"
+
+- name: issue_cert | Copy issuing CA cert
+  copy:
+    content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['issuing_ca'] }}"
+    dest: "{{ issue_cert_path | dirname }}/ca.pem"
+    group: "{{ issue_cert_file_group | d('root' )}}"
+    mode: "{{ issue_cert_file_mode | d('0644') }}"
+    owner: "{{ issue_cert_file_owner | d('root') }}"
+  when: issue_cert_copy_ca|default(false)

roles/vault/tasks/shared/mount.yml

@@ -0,0 +1,18 @@
+---
+
+- name: shared/mount | Test if PKI mount exists
+  uri:
+    url: "{{ vault_leader_url }}/v1/sys/mounts/{{ mount_name }}/tune"
+    headers: "{{ vault_headers }}"
+  ignore_errors: true
+  register: vault_pki_mount_check
+
+- name: shared/mount | Mount PKI mount if needed
+  uri:
+    url: "{{ vault_leader_url }}/v1/sys/mounts/{{ mount_name }}"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body: "{{ mount_options|d() }}"
+    status_code: 204
+  when: vault_pki_mount_check|failed

roles/vault/tasks/shared/pki_mount.yml

@@ -0,0 +1,11 @@
+---
+
+- include: mount.yml
+  vars:
+    mount_name: pki
+    mount_options:
+      config:
+        default_lease_ttl: "{{ vault_default_lease_ttl }}"
+        max_lease_ttl: "{{ vault_max_lease_ttl }}"
+      description: The default PKI mount for Kubernetes
+      type: pki

roles/vault/tasks/shared/sync.yml

@@ -8,7 +8,7 @@
 - name: "sync_file | Cat the key file"
   command: "cat {{ sync_file_key_path }}"
   register: sync_file_key_cat
-  when: sync_file_is_cert|bool and inventory_hostname == sync_file_srcs|first
+  when: sync_file_is_cert|d() and inventory_hostname == sync_file_srcs|first
 
 - name: "sync_file | Set facts for file contents"
   set_fact:
@@ -17,10 +17,13 @@
 - name: "sync_file | Set fact for key contents"
   set_fact:
     sync_file_key_contents: "{{ hostvars[sync_file_srcs|first]['sync_file_key_cat']['stdout'] }}"
-  when: sync_file_is_cert|bool
+  when: sync_file_is_cert|d()
 
 - name: "sync_file | Ensure the directory exists"
   file:
+    group: "{{ sync_file_group|d('root') }}"
+    mode: "{{ sync_file_dir_mode|default('0750') }}"
+    owner: "{{ sync_file_owner|d('root') }}"
     path: "{{ sync_file_dir }}"
     state: directory
   when: inventory_hostname not in sync_file_srcs
@@ -29,10 +32,16 @@
   copy:
     content: "{{ sync_file_contents }}"
     dest: "{{ sync_file_path }}"
+    group: "{{ sync_file_group|d('root') }}"
+    mode: "{{ sync_file_mode|default('0640') }}"
+    owner: "{{ sync_file_owner|d('root') }}"
   when: inventory_hostname not in sync_file_srcs
 
 - name: "sync_file | Copy the key file to hosts that don't have it"
   copy:
     content: "{{ sync_file_key_contents }}"
     dest: "{{ sync_file_key_path }}"
-  when: sync_file_is_cert|bool and inventory_hostname not in sync_file_srcs
+    group: "{{ sync_file_group|d('root') }}"
+    mode: "{{ sync_file_mode|default('0640') }}"
+    owner: "{{ sync_file_owner|d('root') }}"
+  when: sync_file_is_cert|d() and inventory_hostname not in sync_file_srcs

roles/vault/tasks/shared/sync_auth_certs.yml

@@ -0,0 +1,17 @@
+---
+
+- include: sync_file.yml
+  vars:
+    sync_file: "auth-ca.pem"
+    sync_file_dir: "{{ vault_cert_dir }}"
+    sync_file_hosts: "{{ groups.vault }}"
+    sync_file_is_cert: true
+
+- name: shared/sync_auth_certs | Set facts for vault sync_file results
+  set_fact:
+    vault_auth_ca_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}"
+
+
+- name: shared/sync_auth_certs | Unset sync_file_results after auth-ca.pem sync
+  set_fact:
+    sync_file_results: []

roles/vault/tasks/shared/sync_file.yml

@@ -6,17 +6,18 @@
   set_fact:
     sync_file_dir: "{{ sync_file_path | dirname }}"
     sync_file: "{{ sync_file_path | basename }}"
-  when: sync_file_path|bool
+  when: sync_file_path is defined and sync_file_path != ''
 
 - name: "sync_file | Set fact for sync_file_path when undefined"
   set_fact:
     sync_file_path: "{{ (sync_file_dir, sync_file)|join('/') }}"
-  when: not sync_file_path|bool
+  when: sync_file_path is not defined or sync_file_path == ''
 
 - name: "sync_file | Set fact for key path name"
   set_fact:
     sync_file_key_path: "{{ sync_file_path.rsplit('.', 1)|first + '-key.' + sync_file_path.rsplit('.', 1)|last }}"
-  when: sync_file_is_cert|bool and not sync_file_key_path|bool
+  when: >-
+        sync_file_is_cert|d() and (sync_file_key_path is not defined or sync_file_key_path == '')
 
 - name: "sync_file | Check if file exists"
   stat:
@@ -27,7 +28,7 @@
   stat:
     path: "{{ sync_file_key_path }}"
   register: sync_file_key_stat
-  when: sync_file_is_cert|bool
+  when: sync_file_is_cert|d()
 
 - name: "sync_file | Combine all possible file sync sources"
   set_fact:
@@ -43,13 +44,13 @@
   with_items: "{{ sync_file_hosts | unique }}"
   loop_control:
     loop_var: host_item
-  when: sync_file_is_cert|bool and hostvars[host_item]["sync_file_key_stat"]["stat"]["exists"]|bool
+  when: sync_file_is_cert|d() and hostvars[host_item]["sync_file_key_stat"]["stat"]["exists"]|bool
 
 - name: "sync_file | Remove sync sources with files that do not match sync_file_srcs|first"
   set_fact:
     _: "{% if inventory_hostname in sync_file_srcs %}{{ sync_file_srcs.remove(inventory_hostname) }}{% endif %}"
   when: >-
-        sync_file_srcs|length > 0 and
+        sync_file_srcs|d([])|length > 1 and
         inventory_hostname != sync_file_srcs|first and
         sync_file_stat.stat.get("checksum") != hostvars[sync_file_srcs|first]["sync_file_stat"]["stat"]["checksum"]
 
@@ -57,20 +58,20 @@
   set_fact:
     _: "{% if inventory_hostname in sync_file_srcs %}{{ sync_file_srcs.remove(inventory_hostname) }}{% endif %}"
   when: >-
-        sync_file_is_cert|bool and
-        sync_file_key_srcs|length > 0 and
-        inventory_hostname != sync_file_srcs|first and
+        sync_file_is_cert|d() and
+        sync_file_key_srcs|d([])|length > 1 and
+        inventory_hostname != sync_file_key_srcs|first and
         sync_file_key_stat.stat.checksum != hostvars[sync_file_srcs|first]["sync_file_key_stat"]["stat"]["checksum"]
 
 - name: "sync_file | Consolidate file and key sources"
   set_fact:
-    sync_file_srcs: "{{ sync_file_srcs | intersect(sync_file_key_srcs) }}"
-  when: sync_file_is_cert|bool
+    sync_file_srcs: "{{ sync_file_srcs|d([]) | intersect(sync_file_key_srcs) }}"
+  when: sync_file_is_cert|d()
 
 - name: "sync_file | Set facts for situations where sync is not needed"
   set_fact:
-    sync_file_no_srcs: "{{ true if sync_file_srcs|length == 0 else false }}"
-    sync_file_unneeded: "{{ true if sync_file_srcs|length == sync_file_hosts|length else false }}"
+    sync_file_no_srcs: "{{ true if sync_file_srcs|d([])|length == 0 else false }}"
+    sync_file_unneeded: "{{ true if sync_file_srcs|d([])|length == sync_file_hosts|length else false }}"
 
 - name: "sync_file | Set sync_file_result fact"
   set_fact:
@@ -88,10 +89,9 @@
 
 - name: "Unset local vars to avoid variable bleed into next iteration"
   set_fact:
-    sync_file_dir: ''
     sync_file: ''
+    sync_file_dir: ''
     sync_file_key_path: ''
-    sync_file_hosts: []
+    sync_file_key_srcs: []
     sync_file_path: ''
     sync_file_srcs: []
-    sync_file_key_srcs: []

roles/vault/templates/docker.service.j2

@@ -0,0 +1,32 @@
+[Unit]
+Description=hashicorp vault on docker
+Documentation=https://github.com/hashicorp/vault
+Wants=docker.socket
+After=docker.service
+
+[Service]
+User=root
+Restart=always
+RestartSec=15s
+TimeoutStartSec=5
+LimitNOFILE=10000
+ExecReload={{ docker_bin_dir }}/docker restart {{ vault_container_name }}
+ExecStop={{ docker_bin_dir }}/docker stop {{ vault_container_name }}
+ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ vault_container_name }}
+# Container has the following internal mount points:
+#   /vault/file/    # File backend storage location
+#   /vault/logs/    # Log files
+ExecStart={{ docker_bin_dir }}/docker run \
+--name {{ vault_container_name }} --net=host \
+--cap-add=IPC_LOCK \
+-v {{ vault_cert_dir }}:{{ vault_cert_dir }} \
+-v {{ vault_config_dir }}:{{ vault_config_dir }} \
+-v {{ vault_log_dir }}:/vault/logs \
+-v {{ vault_roles_dir }}:{{ vault_roles_dir }} \
+-v {{ vault_secrets_dir }}:{{ vault_secrets_dir }} \
+--entrypoint=vault \
+{{ vault_image_repo }}:{{ vault_image_tag }} \
+server --config={{ vault_config_dir }}/config.json
+
+[Install]
+WantedBy=multi-user.target

roles/vault/templates/host.service.j2

@@ -0,0 +1,15 @@
+[Unit]
+Description=vault
+After=network.target
+
+[Service]
+AmbientCapabilities=CAP_IPC_LOCK
+ExecStart=/usr/bin/vault server --config={{ vault_config_dir }}/config.json
+LimitNOFILE=40000
+NotifyAccess=all
+Restart=always
+RestartSec=10s
+User={{ vault_adduser_vars.name }}
+
+[Install]
+WantedBy=multi-user.target

roles/vault/templates/rkt.service.j2

@@ -0,0 +1,33 @@
+[Unit]
+Description=hashicorp vault on rkt
+Documentation=https://github.com/hashicorp/vault
+Wants=network.target
+
+[Service]
+User=root
+Restart=on-failure
+RestartSec=10s
+TimeoutStartSec=5
+LimitNOFILE=40000
+# Container has the following internal mount points:
+#   /vault/file/    # File backend storage location
+#   /vault/logs/    # Log files
+ExecStart=/usr/bin/rkt run \
+--insecure-options=image \
+--volume=volume-vault-file,kind=host,source=/var/lib/vault \
+--volume=volume-vault-logs,kind=host,source={{ vault_log_dir }} \
+--volume=vault-cert-dir,kind=host,source={{ vault_cert_dir }} \
+--mount=volume=vault-cert-dir,target={{ vault_cert_dir }} \
+--volume=vault-conf-dir,kind=host,source={{ vault_config_dir }} \
+--mount=volume=vault-conf-dir,target={{ vault_config_dir }} \
+--volume=vault-secrets-dir,kind=host,source={{ vault_secrets_dir }} \
+--mount=volume=vault-secrets-dir,target={{ vault_secrets_dir }} \
+--volume=vault-roles-dir,kind=host,source={{ vault_roles_dir }} \
+--mount=volume=vault-roles-dir,target={{ vault_roles_dir }} \
+docker://{{ vault_image_repo }}:{{ vault_image_tag }} \
+--name={{ vault_container_name }} --net=host \
+--caps-retain=CAP_IPC_LOCK \
+--exec vault -- server --config={{ vault_config_dir }}/config.json
+
+[Install]
+WantedBy=multi-user.target

tests/templates/inventory-aws.j2

@@ -15,6 +15,10 @@ node3
 node1
 node2
 
+[vault]
+node1
+node2
+
 [k8s-cluster:children]
 kube-node
 kube-master

tests/templates/inventory-gce.j2

@@ -13,6 +13,9 @@ node2
 
 [etcd]
 node3
+
+[vault]
+node3
 {% elif mode is defined and mode == "ha" %}
 [kube-master]
 node1
@@ -24,6 +27,10 @@ node3
 [etcd]
 node2
 node3
+
+[vault]
+node2
+node3
 {% else %}
 [kube-master]
 node1
@@ -33,6 +40,9 @@ node2
 
 [etcd]
 node1
+
+[vault]
+node1
 {% endif %}
 
 [k8s-cluster:children]