Change single Vault pki mount to multi pki mounts paths for etcd and kube CA`s (#1552)

* Added update CA trust step for etcd and kube/secrets roles

* Added load_balancer_domain_name to certificate alt names if defined. Reset CA's in RedHat os.

* Rename kube-cluster-ca.crt to vault-ca.crt, we need separated CA`s for vault, etcd and kube.

* Vault role refactoring, remove optional cert vault auth because not not used and worked. Create separate CA`s fro vault and etcd.

* Fixed different certificates set for vault cert_managment

* Update doc/vault.md

* Fixed condition create vault CA, wrong group

* Fixed missing etcd_cert_path mount for rkt deployment type. Distribute vault roles for all vault hosts

* Removed wrong when condition in create etcd role vault tasks.

docs/vault.md

@@ -26,7 +26,6 @@ first task, is to stop any temporary instances of Vault, to free the port for
 the long-term. At the end of this task, the entire Vault cluster should be up
 and read to go.
 
-
 Keys to the Kingdom
 -------------------
 
@@ -50,24 +49,32 @@ Vault by default encrypts all traffic to and from the datastore backend, all
 resting data, and uses TLS for its TCP listener. It is recommended that you
 do not change the Vault config to disable TLS, unless you absolutely have to.
 
-
 Usage
 -----
 
 To get the Vault role running, you must to do two things at a minimum:
 
 1. Assign the ``vault`` group to at least 1 node in your inventory
-2. Change ``cert_management`` to be ``vault`` instead of ``script``
+1. Change ``cert_management`` to be ``vault`` instead of ``script``
 
 Nothing else is required, but customization is possible. Check
 ``roles/vault/defaults/main.yml`` for the different variables that can be
 overridden, most common being ``vault_config``, ``vault_port``, and
 ``vault_deployment_type``.
 
-Also, if you intend to use a Root or Intermediate CA generated elsewhere,
+As a result of the Vault role will be create separated Root CA for `etcd`,
-you'll need to copy the certificate and key to the hosts in the vault group
+`kubernetes` and `vault`. Also, if you intend to use a Root or Intermediate CA
-prior to running the vault role. By default, they'll be located at
+generated elsewhere, you'll need to copy the certificate and key to the hosts in the vault group prior to running the vault role. By default, they'll be located at:
-``/etc/vault/ssl/ca.pem`` and ``/etc/vault/ssl/ca-key.pem``, respectively.
+
+* vault:
+  * ``/etc/vault/ssl/ca.pem``
+  * ``/etc/vault/ssl/ca-key.pem``
+* etcd:
+  * ``/etc/ssl/etcd/ssl/ca.pem``
+  * ``/etc/ssl/etcd/ssl/ca-key.pem``
+* kubernetes:
+  * ``/etc/kubernetes/ssl/ca.pem``
+  * ``/etc/kubernetes/ssl/ca-key.pem``
 
 Additional Notes:
 
@@ -77,7 +84,6 @@ Additional Notes:
   credentials are saved to ``/etc/vault/roles/<role>/``. The service will
   need to read in those credentials, if they want to interact with Vault.
 
-
 Potential Work
 --------------
 
@@ -87,6 +93,3 @@ Potential Work
 - Add the ability to start temp Vault with Host, Rkt, or Docker
 - Add a dynamic way to change out the backend role creation during Bootstrap,
   so other services can be used (such as Consul)
-- Segregate Server Cert generation from Auth Cert generation (separate CAs).
-  This work was partially started with the `auth_cert_backend` tasks, but would
-  need to be further applied to all roles (particularly Etcd and Kubernetes).

roles/etcd/defaults/main.yml

@@ -26,3 +26,5 @@ etcd_memory_limit: 512M
 etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr', [])) }}"
 
 etcd_compaction_retention: "8"
+
+etcd_vault_mount_path: etcd

roles/etcd/tasks/gen_certs_script.yml

@@ -161,30 +161,3 @@
     owner: kube
     mode: "u=rwX,g-rwx,o-rwx"
     recurse: yes
-
-- name: Gen_certs | target ca-certificate store file
-  set_fact:
-    ca_cert_path: |-
-      {% if ansible_os_family == "Debian" -%}
-      /usr/local/share/ca-certificates/etcd-ca.crt
-      {%- elif ansible_os_family == "RedHat" -%}
-      /etc/pki/ca-trust/source/anchors/etcd-ca.crt
-      {%- elif ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] -%}
-      /etc/ssl/certs/etcd-ca.pem
-      {%- endif %}
-  tags: facts
-
-- name: Gen_certs | add CA to trusted CA dir
-  copy:
-    src: "{{ etcd_cert_dir }}/ca.pem"
-    dest: "{{ ca_cert_path }}"
-    remote_src: true
-  register: etcd_ca_cert
-
-- name: Gen_certs | update ca-certificates (Debian/Ubuntu/Container Linux by CoreOS)
-  command: update-ca-certificates
-  when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS", "Container Linux by CoreOS"]
-
-- name: Gen_certs | update ca-certificates (RedHat)
-  command: update-ca-trust extract
-  when: etcd_ca_cert.changed and ansible_os_family == "RedHat"

roles/etcd/tasks/gen_certs_vault.yml

@@ -66,6 +66,7 @@
     issue_cert_path: "{{ item }}"
     issue_cert_role: etcd
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+    issue_cert_mount_path: "{{ etcd_vault_mount_path }}"
   with_items: "{{ etcd_master_certs_needed|d([]) }}"
   when: inventory_hostname in groups.etcd
   notify: set etcd_secret_changed
@@ -92,6 +93,7 @@
     issue_cert_path: "{{ item }}"
     issue_cert_role: etcd
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+    issue_cert_mount_path: "{{ etcd_vault_mount_path }}"
   with_items: "{{ etcd_node_certs_needed|d([]) }}"
   when: inventory_hostname in etcd_node_cert_hosts
   notify: set etcd_secret_changed

roles/etcd/tasks/main.yml

@@ -10,6 +10,9 @@
 - include: "gen_certs_{{ cert_management }}.yml"
   tags: etcd-secrets
 
+- include: upd_ca_trust.yml
+  tags: etcd-secrets
+
 - include: "install_{{ etcd_deployment_type }}.yml"
   when: is_etcd_master
   tags: upgrade

roles/etcd/tasks/upd_ca_trust.yml

@@ -0,0 +1,27 @@
+---
+- name: Gen_certs | target ca-certificate store file
+  set_fact:
+    ca_cert_path: |-
+      {% if ansible_os_family == "Debian" -%}
+      /usr/local/share/ca-certificates/etcd-ca.crt
+      {%- elif ansible_os_family == "RedHat" -%}
+      /etc/pki/ca-trust/source/anchors/etcd-ca.crt
+      {%- elif ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] -%}
+      /etc/ssl/certs/etcd-ca.pem
+      {%- endif %}
+  tags: facts
+
+- name: Gen_certs | add CA to trusted CA dir
+  copy:
+    src: "{{ etcd_cert_dir }}/ca.pem"
+    dest: "{{ ca_cert_path }}"
+    remote_src: true
+  register: etcd_ca_cert
+
+- name: Gen_certs | update ca-certificates (Debian/Ubuntu/Container Linux by CoreOS)
+  command: update-ca-certificates
+  when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS", "Container Linux by CoreOS"]
+
+- name: Gen_certs | update ca-certificates (RedHat)
+  command: update-ca-trust extract
+  when: etcd_ca_cert.changed and ansible_os_family == "RedHat"

roles/kubernetes/secrets/defaults/main.yml

@@ -1,2 +1,3 @@
 ---
 kube_cert_group: kube-cert
+kube_vault_mount_path: kube

roles/kubernetes/secrets/tasks/gen_certs_script.yml

@@ -166,30 +166,3 @@
     owner: kube
     mode: "u=rwX,g-rwx,o-rwx"
     recurse: yes
-
-- name: Gen_certs | target ca-certificates path
-  set_fact:
-    ca_cert_path: |-
-      {% if ansible_os_family == "Debian" -%}
-      /usr/local/share/ca-certificates/kube-ca.crt
-      {%- elif ansible_os_family == "RedHat" -%}
-      /etc/pki/ca-trust/source/anchors/kube-ca.crt
-      {%- elif ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] -%}
-      /etc/ssl/certs/kube-ca.pem
-      {%- endif %}
-  tags: facts
-
-- name: Gen_certs | add CA to trusted CA dir
-  copy:
-    src: "{{ kube_cert_dir }}/ca.pem"
-    dest: "{{ ca_cert_path }}"
-    remote_src: true
-  register: kube_ca_cert
-
-- name: Gen_certs | update ca-certificates (Debian/Ubuntu/Container Linux by CoreOS)
-  command: update-ca-certificates
-  when: kube_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS", "Container Linux by CoreOS"]
-
-- name: Gen_certs | update ca-certificates (RedHat)
-  command: update-ca-trust extract
-  when: kube_ca_cert.changed and ansible_os_family == "RedHat"

roles/kubernetes/secrets/tasks/gen_certs_vault.yml

@@ -49,17 +49,29 @@
     issue_cert_path: "{{ item }}"
     issue_cert_role: kube
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+    issue_cert_mount_path: "{{ kube_vault_mount_path }}"
   with_items: "{{ kube_master_certs_needed|d([]) }}"
   when: inventory_hostname in groups['kube-master']
 
+- name: gen_certs_vault | Set fact about certificate alt names
+  set_fact:
+    kube_cert_alt_names: >-
+      {{
+      groups['kube-master'] +
+      ['kubernetes.default.svc.cluster.local', 'kubernetes.default.svc', 'kubernetes.default', 'kubernetes'] +
+      ['localhost']
+      }}
+  run_once: true
+
+- name: gen_certs_vault | Add external load balancer domain name to certificate alt names
+  set_fact:
+    kube_cert_alt_names: "{{ kube_cert_alt_names + [apiserver_loadbalancer_domain_name] }}"
+  when: loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined
+  run_once: true
+
 - include: ../../../vault/tasks/shared/issue_cert.yml
   vars:
-    issue_cert_alt_names: >-
+    issue_cert_alt_names: "{{ kube_cert_alt_names }}"
-        {{
-        groups['kube-master'] +
-        ['kubernetes.default.svc.cluster.local', 'kubernetes.default.svc', 'kubernetes.default', 'kubernetes'] +
-        ['localhost']
-        }}
     issue_cert_file_group: "{{ kube_cert_group }}"
     issue_cert_file_owner: kube
     issue_cert_headers: "{{ kube_vault_headers }}"
@@ -77,8 +89,10 @@
     issue_cert_path: "{{ item }}"
     issue_cert_role: kube
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+    issue_cert_mount_path: "{{ kube_vault_mount_path }}"
   with_items: "{{ kube_master_components_certs_needed|d([]) }}"
   when: inventory_hostname in groups['kube-master']
+  notify: set secret_changed
 
 # Issue node certs to k8s-cluster nodes
 - include: ../../../vault/tasks/shared/issue_cert.yml
@@ -91,6 +105,7 @@
     issue_cert_path: "{{ item }}"
     issue_cert_role: kube
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+    issue_cert_mount_path: "{{ kube_vault_mount_path }}"
   with_items: "{{ kube_node_certs_needed|d([]) }}"
   when: inventory_hostname in groups['k8s-cluster']
 
@@ -104,5 +119,6 @@
     issue_cert_path: "{{ item }}"
     issue_cert_role: kube
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+    issue_cert_mount_path: "{{ kube_vault_mount_path }}"
   with_items: "{{ kube_proxy_certs_needed|d([]) }}"
   when: inventory_hostname in groups['k8s-cluster']

roles/kubernetes/secrets/tasks/main.yml

@@ -72,5 +72,8 @@
 - include: "gen_certs_{{ cert_management }}.yml"
   tags: k8s-secrets
 
+- include: upd_ca_trust.yml
+  tags: k8s-secrets
+
 - include: gen_tokens.yml
   tags: k8s-secrets

roles/kubernetes/secrets/tasks/upd_ca_trust.yml

@@ -0,0 +1,27 @@
+---
+- name: Gen_certs | target ca-certificates path
+  set_fact:
+    ca_cert_path: |-
+      {% if ansible_os_family == "Debian" -%}
+      /usr/local/share/ca-certificates/kube-ca.crt
+      {%- elif ansible_os_family == "RedHat" -%}
+      /etc/pki/ca-trust/source/anchors/kube-ca.crt
+      {%- elif ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] -%}
+      /etc/ssl/certs/kube-ca.pem
+      {%- endif %}
+  tags: facts
+
+- name: Gen_certs | add CA to trusted CA dir
+  copy:
+    src: "{{ kube_cert_dir }}/ca.pem"
+    dest: "{{ ca_cert_path }}"
+    remote_src: true
+  register: kube_ca_cert
+
+- name: Gen_certs | update ca-certificates (Debian/Ubuntu/Container Linux by CoreOS)
+  command: update-ca-certificates
+  when: kube_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS", "Container Linux by CoreOS"]
+
+- name: Gen_certs | update ca-certificates (RedHat)
+  command: update-ca-trust extract
+  when: kube_ca_cert.changed and ansible_os_family == "RedHat"

roles/reset/tasks/main.yml

@@ -17,6 +17,7 @@
   with_items:
     - kubelet
     - etcd
+    - vault
   register: services_removed
   tags: ['services']
 
@@ -86,10 +87,15 @@
     - /run/flannel
     - /etc/flannel
     - /run/kubernetes
-    - /usr/local/share/ca-certificates/kube-ca.crt
     - /usr/local/share/ca-certificates/etcd-ca.crt
-    - /etc/ssl/certs/kube-ca.pem
+    - /usr/local/share/ca-certificates/kube-ca.crt
+    - /usr/local/share/ca-certificates/vault-ca.crt
     - /etc/ssl/certs/etcd-ca.pem
+    - /etc/ssl/certs/kube-ca.pem
+    - /etc/ssl/certs/vault-ca.crt
+    - /etc/pki/ca-trust/source/anchors/etcd-ca.crt
+    - /etc/pki/ca-trust/source/anchors/kube-ca.crt
+    - /etc/pki/ca-trust/source/anchors/vault-ca.crt
     - /etc/vault
     - /var/log/pods/
     - "{{ bin_dir }}/kubelet"

roles/vault/defaults/main.yml

@@ -8,10 +8,11 @@ vault_adduser_vars:
   system: yes
 vault_base_dir: /etc/vault
 # https://releases.hashicorp.com/vault/0.6.4/vault_0.6.4_SHA256SUMS
+vault_version: 0.6.4
 vault_binary_checksum: 04d87dd553aed59f3fe316222217a8d8777f40115a115dac4d88fac1611c51a6
 vault_bootstrap: false
 vault_ca_options:
-  common_name: kube-cluster-ca
+  common_name: vault
   format: pem
   ttl: 87600h
 vault_cert_dir: "{{ vault_base_dir }}/ssl"
@@ -24,7 +25,7 @@ vault_config:
       address: "{{ vault_etcd_url }}"
       ha_enabled: "true"
       redirect_addr: "https://{{ ansible_default_ipv4.address }}:{{ vault_port }}"
-      tls_ca_file: "{{ vault_cert_dir }}/ca.pem"
+      tls_ca_file: "{{ vault_etcd_cert_dir }}/ca.pem"
   cluster_name: "kubernetes-vault"
   default_lease_ttl: "{{ vault_default_lease_ttl }}"
   listener:
@@ -61,18 +62,6 @@ vault_log_dir: "/var/log/vault"
 vault_max_lease_ttl: 87600h
 vault_needs_gen: false
 vault_port: 8200
-# Although "cert" is an option, ansible has no way to auth via cert until
-# upstream merges: https://github.com/ansible/ansible/pull/18141
-vault_role_auth_method: userpass
-vault_roles:
-  - name: etcd
-    group: etcd
-    policy_rules: default
-    role_options: default
-  - name: kube
-    group: k8s-cluster
-    policy_rules: default
-    role_options: default
 vault_roles_dir: "{{ vault_base_dir }}/roles"
 vault_secret_shares: 1
 vault_secret_threshold: 1
@@ -88,4 +77,25 @@ vault_temp_config:
       tls_disable: "true"
   max_lease_ttl: "{{ vault_max_lease_ttl }}"
 vault_temp_container_name: vault-temp
-vault_version: 0.6.4
+# etcd pki mount options
+vault_etcd_cert_dir: /etc/ssl/etcd/ssl
+vault_etcd_mount_path: etcd
+vault_etcd_default_lease_ttl: 720h
+vault_etcd_max_lease_ttl: 87600h
+vault_etcd_role:
+  name: etcd
+  group: etcd
+  policy_rules: default
+  role_options: default
+  mount_path: "{{ vault_etcd_mount_path }}"
+# kubernetes pki mount options
+vault_kube_cert_dir: "{{ kube_cert_dir }}"
+vault_kube_mount_path: kube
+vault_kube_default_lease_ttl: 720h
+vault_kube_max_lease_ttl: 87600h
+vault_kube_role:
+  name: kube
+  group: k8s-cluster
+  policy_rules: default
+  role_options: default
+  mount_path: "{{ vault_kube_mount_path }}"

roles/vault/tasks/bootstrap/ca_trust.yml

@@ -10,11 +10,11 @@
   set_fact:
     ca_cert_path: >-
       {% if ansible_os_family == "Debian" -%}
-      /usr/local/share/ca-certificates/kube-cluster-ca.crt
+      /usr/local/share/ca-certificates/vault-ca.crt
       {%- elif ansible_os_family == "RedHat" -%}
-      /etc/pki/ca-trust/source/anchors/kube-cluster-ca.crt
+      /etc/pki/ca-trust/source/anchors/vault-ca.crt
       {%- elif ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] -%}
-      /etc/ssl/certs/kube-cluster-ca.pem
+      /etc/ssl/certs/vault-ca.pem
       {%- endif %}
 
 - name: bootstrap/ca_trust | add CA to trusted CA dir

roles/vault/tasks/bootstrap/create_etcd_role.yml

@@ -1,9 +1,17 @@
 ---
+- include: ../shared/auth_backend.yml
+  vars:
+    auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates
+    auth_backend_path: userpass
+    auth_backend_type: userpass
+  delegate_to: "{{ groups.vault|first }}"
+  run_once: true
+
 - include: ../shared/create_role.yml
   vars:
-    create_role_name: "{{ item.name }}"
+    create_role_name: "{{ vault_etcd_role.name }}"
-    create_role_group: "{{ item.group }}"
+    create_role_group: "{{ vault_etcd_role.group }}"
-    create_role_policy_rules: "{{ item.policy_rules }}"
+    create_role_policy_rules: "{{ vault_etcd_role.policy_rules }}"
-    create_role_options: "{{ item.role_options }}"
+    create_role_options: "{{ vault_etcd_role.role_options }}"
-  with_items: "{{ vault_roles }}"
+    create_role_mount_path: "{{ vault_etcd_role.mount_path }}"
-  when: item.name == "etcd"
+  when: inventory_hostname in groups.etcd

roles/vault/tasks/bootstrap/gen_auth_ca.yml

@@ -1,21 +0,0 @@
----
-
-- name: bootstrap/gen_auth_ca | Generate Root CA
-  uri:
-    url: "{{ vault_leader_url }}/v1/auth-pki/root/generate/exported"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body: "{{ vault_ca_options }}"
-  register: vault_auth_ca_gen
-  when: inventory_hostname == groups.vault|first
-
-- name: bootstrap/gen_auth_ca | Copy auth CA cert to Vault nodes
-  copy:
-    content: "{{ hostvars[groups.vault|first]['vault_auth_ca_gen']['json']['data']['certificate'] }}"
-    dest: "{{ vault_cert_dir }}/auth-ca.pem"
-
-- name: bootstrap/gen_auth_ca | Copy auth CA key to Vault nodes
-  copy:
-    content: "{{ hostvars[groups.vault|first]['vault_auth_ca_gen']['json']['data']['private_key'] }}"
-    dest: "{{ vault_cert_dir }}/auth-ca-key.pem"

roles/vault/tasks/bootstrap/gen_ca.yml

@@ -1,31 +0,0 @@
----
-
-- name: bootstrap/gen_ca | Ensure vault_cert_dir exists
-  file:
-    mode: 0755
-    path: "{{ vault_cert_dir }}"
-    state: directory
-
-- name: bootstrap/gen_ca | Generate Root CA in vault-temp
-  uri:
-    url: "{{ vault_leader_url }}/v1/pki/root/generate/exported"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body: "{{ vault_ca_options }}"
-  register: vault_ca_gen
-  when: inventory_hostname == groups.vault|first and vault_ca_cert_needed
-
-- name: bootstrap/gen_ca | Copy root CA cert locally
-  copy:
-    content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['certificate'] }}"
-    dest: "{{ vault_cert_dir }}/ca.pem"
-    mode: 0644
-  when: vault_ca_cert_needed
-
-- name: bootstrap/gen_ca | Copy root CA key locally
-  copy:
-    content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['private_key'] }}"
-    dest: "{{ vault_cert_dir }}/ca-key.pem"
-    mode: 0640
-  when: vault_ca_cert_needed

roles/vault/tasks/bootstrap/gen_vault_certs.yml

@@ -2,7 +2,7 @@
 
 - name: boostrap/gen_vault_certs | Add the vault role
   uri:
-    url: "{{ vault_leader_url }}/v1/pki/roles/vault"
+    url: "{{ vault_leader_url }}/v1/{{ vault_ca_options.common_name }}/roles/vault"
     headers: "{{ vault_headers }}"
     method: POST
     body_format: json
@@ -21,6 +21,7 @@
         {%- endfor -%}
         "127.0.0.1","::1"
         ]
+    issue_cert_mount_path: "{{ vault_ca_options.common_name }}"
     issue_cert_path: "{{ vault_cert_dir }}/api.pem"
     issue_cert_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
     issue_cert_role: vault

roles/vault/tasks/bootstrap/main.yml

@@ -14,6 +14,9 @@
 - include: sync_vault_certs.yml
   when: inventory_hostname in groups.vault
 
+- include: sync_etcd_certs.yml
+  when: inventory_hostname in groups.etcd
+
 ## Generate Certs
 
 # Start a temporary instance of Vault
@@ -28,24 +31,22 @@
     vault_leader_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
   when: not vault_cluster_is_initialized
 
-# NOTE: The next 2 steps run against temp Vault and long-term Vault
+# Ensure vault PKI mounts exists
-
+- include: ../shared/create_mount.yml
-# Ensure PKI mount exists
-- include: ../shared/pki_mount.yml
-  when: >-
-        inventory_hostname == groups.vault|first
-
-# If the Root CA already exists, ensure Vault's PKI is using it
-- include: ../shared/config_ca.yml
   vars:
-    ca_name: ca
+    create_mount_path: "{{ vault_ca_options.common_name }}"
-    mount_name: pki
+    create_mount_default_lease_ttl: "{{ vault_default_lease_ttl }}"
-  when: >-
+    create_mount_max_lease_ttl: "{{ vault_max_lease_ttl }}"
-        inventory_hostname == groups.vault|first and
+    create_mount_description: "Vault Root CA"
-        not vault_ca_cert_needed
+    create_mount_cert_dir: "{{ vault_cert_dir }}"
+    create_mount_config_ca_needed: "{{ not vault_ca_cert_needed }}"
+  when: inventory_hostname == groups.vault|first
 
 # Generate root CA certs for Vault if none exist
-- include: gen_ca.yml
+- include: ../shared/gen_ca.yml
+  vars:
+    gen_ca_cert_dir: "{{ vault_cert_dir }}"
+    gen_ca_mount_path: "{{ vault_ca_options.common_name }}"
   when: >-
         inventory_hostname in groups.vault and
         not vault_cluster_is_initialized and
@@ -55,13 +56,25 @@
 - include: gen_vault_certs.yml
   when: inventory_hostname in groups.vault and vault_api_cert_needed
 
-# Update all host's CA bundle
+# Ensure etcd PKI mounts exists
+- include: ../shared/create_mount.yml
+  vars:
+    create_mount_path: "{{ vault_etcd_mount_path }}"
+    create_mount_default_lease_ttl: "{{ vault_etcd_default_lease_ttl }}"
+    create_mount_max_lease_ttl: "{{ vault_etcd_max_lease_ttl }}"
+    create_mount_description: "Etcd Root CA"
+    create_mount_cert_dir: "{{ vault_etcd_cert_dir }}"
+    create_mount_config_ca_needed: "{{ not vault_etcd_ca_cert_needed }}"
+  when: inventory_hostname == groups.vault|first
+
+# Generate root CA certs for etcd if none exist
+- include: ../shared/gen_ca.yml
+  vars:
+    gen_ca_cert_dir: "{{ vault_etcd_cert_dir }}"
+    gen_ca_mount_path: "{{ vault_etcd_mount_path }}"
+  when: inventory_hostname in groups.etcd and vault_etcd_ca_cert_needed
+
+- include: create_etcd_role.yml
+
+# Update all host's CA bundle, etcd CA will be added in etcd role
 - include: ca_trust.yml
-
-## Add Etcd Role to Vault (if needed)
-
-- include: role_auth_cert.yml
-  when: vault_role_auth_method == "cert"
-
-- include: role_auth_userpass.yml
-  when: vault_role_auth_method == "userpass"

roles/vault/tasks/bootstrap/role_auth_cert.yml

@@ -1,26 +0,0 @@
----
-
-- include: ../shared/sync_auth_certs.yml
-  when: inventory_hostname in groups.vault
-
-- include: ../shared/cert_auth_mount.yml
-  when: inventory_hostname == groups.vault|first
-
-- include: ../shared/auth_backend.yml
-  vars:
-    auth_backend_description: A Cert-based Auth primarily for services needing to issue certificates
-    auth_backend_name: cert
-    auth_backend_type: cert
-  when: inventory_hostname == groups.vault|first
-
-- include: gen_auth_ca.yml
-  when: inventory_hostname in groups.vault and vault_auth_ca_cert_needed
-
-- include: ../shared/config_ca.yml
-  vars:
-    ca_name: auth-ca
-    mount_name: auth-pki
-  when: inventory_hostname == groups.vault|first and not vault_auth_ca_cert_needed
-
-- include: create_etcd_role.yml
-  when: inventory_hostname in groups.etcd

roles/vault/tasks/bootstrap/role_auth_userpass.yml

@@ -1,11 +0,0 @@
----
-
-- include: ../shared/auth_backend.yml
-  vars:
-    auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates
-    auth_backend_path: userpass
-    auth_backend_type: userpass
-  when: inventory_hostname == groups.vault|first
-
-- include: create_etcd_role.yml
-  when: inventory_hostname in groups.etcd

roles/vault/tasks/bootstrap/sync_etcd_certs.yml

@@ -0,0 +1,16 @@
+---
+
+- include: ../shared/sync_file.yml
+  vars:
+    sync_file: "ca.pem"
+    sync_file_dir: "{{ vault_etcd_cert_dir }}"
+    sync_file_hosts: "{{ groups.etcd }}"
+    sync_file_is_cert: true
+
+- name: bootstrap/sync_etcd_certs | Set facts for etcd sync_file results
+  set_fact:
+    vault_etcd_ca_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}"
+
+- name: bootstrap/sync_etcd_certs | Unset sync_file_results after ca.pem sync
+  set_fact:
+    sync_file_results: []

roles/vault/tasks/cluster/create_roles.yml

@@ -1,4 +1,10 @@
 ---
+- include: ../shared/auth_backend.yml
+  vars:
+    auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates
+    auth_backend_path: userpass
+    auth_backend_type: userpass
+  when: inventory_hostname == groups.vault|first
 
 - include: ../shared/create_role.yml
   vars:
@@ -6,4 +12,7 @@
     create_role_group: "{{ item.group }}"
     create_role_policy_rules: "{{ item.policy_rules }}"
     create_role_options: "{{ item.role_options }}"
-  with_items: "{{ vault_roles|d([]) }}"
+    create_role_mount_path: "{{ item.mount_path }}"
+  with_items:
+    - "{{ vault_etcd_role }}"
+    - "{{ vault_kube_role }}"

roles/vault/tasks/cluster/main.yml

@@ -25,19 +25,42 @@
 - include: ../shared/find_leader.yml
   when: inventory_hostname in groups.vault
 
-- include: ../shared/pki_mount.yml
+- include: ../shared/create_mount.yml
+  vars:
+    create_mount_path: "{{ vault_ca_options.common_name }}"
+    create_mount_default_lease_ttl: "{{ vault_default_lease_ttl }}"
+    create_mount_max_lease_ttl: "{{ vault_max_lease_ttl }}"
+    create_mount_description: "Vault Root CA"
+    create_mount_cert_dir: "{{ vault_cert_dir }}"
+    create_mount_config_ca_needed: true
   when: inventory_hostname == groups.vault|first
 
-- include: ../shared/config_ca.yml
+- include: ../shared/create_mount.yml
   vars:
-    ca_name: ca
+    create_mount_path: "{{ vault_etcd_mount_path }}"
-    mount_name: pki
+    create_mount_default_lease_ttl: "{{ vault_etcd_default_lease_ttl }}"
+    create_mount_max_lease_ttl: "{{ vault_etcd_max_lease_ttl }}"
+    create_mount_description: "Etcd Root CA"
+    create_mount_cert_dir: "{{ vault_etcd_cert_dir }}"
+    create_mount_config_ca_needed: true
   when: inventory_hostname == groups.vault|first
 
+- include: ../shared/create_mount.yml
+  vars:
+    create_mount_path: "{{ vault_kube_mount_path }}"
+    create_mount_default_lease_ttl: "{{ vault_kube_default_lease_ttl }}"
+    create_mount_max_lease_ttl: "{{ vault_kube_max_lease_ttl }}"
+    create_mount_description: "Kubernetes Root CA"
+    create_mount_cert_dir: "{{ vault_kube_cert_dir }}"
+    create_mount_config_ca_needed: false
+  when: inventory_hostname == groups.vault|first
+
+- include: ../shared/gen_ca.yml
+  vars:
+    gen_ca_cert_dir: "{{ vault_kube_cert_dir }}"
+    gen_ca_mount_path: "{{ vault_kube_mount_path }}"
+  when: inventory_hostname in groups.vault
+
 ## Vault Policies, Roles, and Auth Backends
 
-- include: role_auth_cert.yml
+- include: create_roles.yml
-  when: vault_role_auth_method == "cert"
-
-- include: role_auth_userpass.yml
-  when: vault_role_auth_method == "userpass"

roles/vault/tasks/cluster/role_auth_cert.yml

@@ -1,19 +0,0 @@
----
-
-- include: ../shared/cert_auth_mount.yml
-  when: inventory_hostname == groups.vault|first
-
-- include: ../shared/auth_backend.yml
-  vars:
-    auth_backend_description: A Cert-based Auth primarily for services needing to issue certificates
-    auth_backend_name: cert
-    auth_backend_type: cert
-  when: inventory_hostname == groups.vault|first
-
-- include: ../shared/config_ca.yml
-  vars:
-    ca_name: auth-ca
-    mount_name: auth-pki
-  when: inventory_hostname == groups.vault|first
-
-- include: create_roles.yml

roles/vault/tasks/cluster/role_auth_userpass.yml

@@ -1,10 +0,0 @@
----
-
-- include: ../shared/auth_backend.yml
-  vars:
-    auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates
-    auth_backend_path: userpass
-    auth_backend_type: userpass
-  when: inventory_hostname == groups.vault|first
-
-- include: create_roles.yml

roles/vault/tasks/shared/cert_auth_mount.yml

@@ -1,14 +1,13 @@
 ---
 
-- include: ../shared/mount.yml
+- include: ../shared/pki_mount.yml
   vars:
-    mount_name: auth-pki
+    pki_mount_path: auth-pki
-    mount_options:
+    pki_mount_options:
       description: PKI mount to generate certs for the Cert Auth Backend
       config:
         default_lease_ttl: "{{ vault_default_lease_ttl }}"
         max_lease_ttl: "{{ vault_max_lease_ttl }}"
-      type: pki
 
 - name: shared/auth_mount | Create a dummy role for issuing certs from auth-pki
   uri:

roles/vault/tasks/shared/config_ca.yml

@@ -1,12 +1,11 @@
 ---
-
 - name: config_ca | Read root CA cert for Vault
-  command: "cat /etc/vault/ssl/{{ ca_name }}.pem"
+  command: "cat {{ config_ca_ca_pem }}"
   register: vault_ca_cert_cat
 
 - name: config_ca | Pull current CA cert from Vault
   uri:
-    url: "{{ vault_leader_url }}/v1/{{ mount_name }}/ca/pem"
+    url: "{{ vault_leader_url }}/v1/{{ config_ca_mount_path }}/ca/pem"
     headers: "{{ vault_headers }}"
     return_content: true
     status_code: 200,204
@@ -14,13 +13,13 @@
   register: vault_pull_current_ca
 
 - name: config_ca | Read root CA key for Vault
-  command: "cat /etc/vault/ssl/{{ ca_name }}-key.pem"
+  command: "cat {{ config_ca_ca_key }}"
   register: vault_ca_key_cat
   when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.content.strip()
 
 - name: config_ca | Configure pki mount to use the found root CA cert and key
   uri:
-    url: "{{ vault_leader_url }}/v1/{{ mount_name }}/config/ca"
+    url: "{{ vault_leader_url }}/v1/{{ config_ca_mount_path }}/config/ca"
     headers: "{{ vault_headers }}"
     method: POST
     body_format: json

roles/vault/tasks/shared/create_mount.yml

@@ -0,0 +1,16 @@
+---
+- include: ../shared/pki_mount.yml
+  vars:
+    pki_mount_path: "{{ create_mount_path }}"
+    pki_mount_options:
+      config:
+        default_lease_ttl: "{{ create_mount_default_lease_ttl }}"
+        max_lease_ttl: "{{ create_mount_max_lease_ttl }}"
+      description: "{{ create_mount_description }}"
+
+- include: ../shared/config_ca.yml
+  vars:
+    config_ca_ca_pem: "{{ create_mount_cert_dir }}/ca.pem"
+    config_ca_ca_key: "{{ create_mount_cert_dir }}/ca-key.pem"
+    config_ca_mount_path: "{{ create_mount_path }}"
+  when: create_mount_config_ca_needed

roles/vault/tasks/shared/create_role.yml

@@ -12,8 +12,8 @@
              {%- if create_role_policy_rules|d("default") == "default" -%}
              {{
              { 'path': {
-                 'pki/issue/' + create_role_name: {'policy': 'write'},
+                 create_role_mount_path + '/issue/' + create_role_name: {'policy': 'write'},
-                 'pki/roles/' + create_role_name: {'policy': 'read'}
+                 create_role_mount_path + '/roles/' + create_role_name: {'policy': 'read'}
              }} | to_json + '\n'
              }}
              {%- else -%}
@@ -22,9 +22,9 @@
     status_code: 204
   when: inventory_hostname == groups[create_role_group]|first
 
-- name: create_role | Create the new role in the pki mount
+- name: create_role | Create the new role in the {{ create_role_mount_path }} pki mount
   uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/pki/roles/{{ create_role_name }}"
+    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/{{ create_role_mount_path }}/roles/{{ create_role_name }}"
     headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
     method: POST
     body_format: json
@@ -37,31 +37,6 @@
     status_code: 204
   when: inventory_hostname == groups[create_role_group]|first
 
-## Cert based auth method
-
-- include: gen_cert.yml
-  vars:
-    gen_cert_copy_ca: true
-    gen_cert_hosts: "{{ groups[create_role_group] }}"
-    gen_cert_mount: "auth-pki"
-    gen_cert_path: "{{ vault_roles_dir }}/{{ create_role_name }}/issuer.pem"
-    gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    gen_cert_vault_role: "dummy"
-    gen_cert_vault_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
-  when: vault_role_auth_method == "cert" and inventory_hostname in groups[create_role_group]
-
-- name: create_role | Insert the auth-pki CA as the authenticating CA for that role
-  uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/cert/certs/{{ create_role_name }}"
-    headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
-    method: POST
-    body_format: json
-    body:
-      certificate: "{{ hostvars[groups[create_role_group]|first]['gen_cert_result']['json']['data']['issuing_ca'] }}"
-      policies: "{{ create_role_name }}"
-    status_code: 204
-  when: vault_role_auth_method == "cert" and inventory_hostname == groups[create_role_group]|first
-
 ## Userpass based auth method
 
 - include: gen_userpass.yml
@@ -71,4 +46,4 @@
     gen_userpass_policies: "{{ create_role_name }}"
     gen_userpass_role: "{{ create_role_name }}"
     gen_userpass_username: "{{ create_role_name }}"
-  when: vault_role_auth_method == "userpass" and inventory_hostname in groups[create_role_group]
+  when: inventory_hostname in groups[create_role_group]

roles/vault/tasks/shared/gen_ca.yml

@@ -0,0 +1,29 @@
+---
+- name: "bootstrap/gen_ca | Ensure cert_dir {{ gen_ca_cert_dir }} exists"
+  file:
+    mode: 0755
+    path: "{{ gen_ca_cert_dir }}"
+    state: directory
+
+- name: "bootstrap/gen_ca | Generate {{ gen_ca_mount_path }} root CA"
+  uri:
+    url: "{{ vault_leader_url }}/v1/{{ gen_ca_mount_path }}/root/generate/exported"
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body: "{{ vault_ca_options }}"
+  register: vault_ca_gen
+  delegate_to: "{{ groups.vault|first }}"
+  run_once: true
+
+- name: "bootstrap/gen_ca | Copy {{ gen_ca_mount_path }} root CA cert locally"
+  copy:
+    content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['certificate'] }}"
+    dest: "{{ gen_ca_cert_dir }}/ca.pem"
+    mode: 0644
+
+- name: "bootstrap/gen_ca | Copy {{ gen_ca_mount_path }} root CA key locally"
+  copy:
+    content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['private_key'] }}"
+    dest: "{{ gen_ca_cert_dir }}/ca-key.pem"
+    mode: 0640

roles/vault/tasks/shared/issue_cert.yml

@@ -14,16 +14,11 @@
 #   issue_cert_headers:     Headers passed into the issue request
 #   issue_cert_hosts:       List of hosts to distribute the cert to
 #   issue_cert_ip_sans:     Requested IP Subject Alternative Names, in a list
-#   issue_cert_mount:       Mount point in Vault to make the request to
+#   issue_cert_mount_path:  Mount point in Vault to make the request to
 #   issue_cert_path:        Full path to the cert, include its name
 #   issue_cert_role:        The Vault role to issue the cert with
 #   issue_cert_url:         Url to reach Vault, including protocol and port
 
-- name: issue_cert | debug who issues certs
-  debug:
-    msg: "{{ issue_cert_hosts }} issues certs"
-
-
 - name: issue_cert | Ensure target directory exists
   file:
     path: "{{ issue_cert_path | dirname }}"
@@ -34,7 +29,7 @@
 
 - name: "issue_cert | Generate the cert for {{ issue_cert_role }}"
   uri:
-    url: "{{ issue_cert_url }}/v1/{{ issue_cert_mount|d('pki') }}/issue/{{ issue_cert_role }}"
+    url: "{{ issue_cert_url }}/v1/{{ issue_cert_mount_path|d('pki') }}/issue/{{ issue_cert_role }}"
     headers: "{{ issue_cert_headers }}"
     method: POST
     body_format: json
@@ -45,11 +40,7 @@
       ip_sans: "{{ issue_cert_ip_sans | default([]) | join(',') }}"
   register: issue_cert_result
   delegate_to: "{{ issue_cert_hosts|first }}"
-
+  run_once: true
-- name: issue_cert | results
-  debug:
-    msg: "{{ issue_cert_result }}"
-
 
 - name: "issue_cert | Copy {{ issue_cert_path }} cert to all hosts"
   copy:

roles/vault/tasks/shared/mount.yml

@@ -1,18 +0,0 @@
----
-
-- name: shared/mount | Test if PKI mount exists
-  uri:
-    url: "{{ vault_leader_url }}/v1/sys/mounts/{{ mount_name }}/tune"
-    headers: "{{ vault_headers }}"
-  ignore_errors: true
-  register: vault_pki_mount_check
-
-- name: shared/mount | Mount PKI mount if needed
-  uri:
-    url: "{{ vault_leader_url }}/v1/sys/mounts/{{ mount_name }}"
-    headers: "{{ vault_headers }}"
-    method: POST
-    body_format: json
-    body: "{{ mount_options|d() }}"
-    status_code: 204
-  when: vault_pki_mount_check|failed

roles/vault/tasks/shared/pki_mount.yml

@@ -1,11 +1,27 @@
 ---
+- name: "shared/mount | Test if {{ pki_mount_path }} PKI mount exists"
+  uri:
+    url: "{{ vault_leader_url }}/v1/sys/mounts/{{ pki_mount_path }}/tune"
+    headers: "{{ vault_headers }}"
+  ignore_errors: true
+  register: vault_pki_mount_check
 
-- include: mount.yml
+- name: shared/mount | Set pki mount type
-  vars:
+  set_fact:
-    mount_name: pki
+    mount_options: "{{ pki_mount_options | combine({'type': 'pki'}) }}"
-    mount_options:
+  when: vault_pki_mount_check|failed
-      config:
+
-        default_lease_ttl: "{{ vault_default_lease_ttl }}"
+- name: shared/mount | Mount {{ pki_mount_path }} PKI mount if needed
-        max_lease_ttl: "{{ vault_max_lease_ttl }}"
+  uri:
-      description: The default PKI mount for Kubernetes
+    url: "{{ vault_leader_url }}/v1/sys/mounts/{{ pki_mount_path }}"
-      type: pki
+    headers: "{{ vault_headers }}"
+    method: POST
+    body_format: json
+    body: "{{ mount_options|d() }}"
+    status_code: 204
+  when: vault_pki_mount_check|failed
+
+- name: shared/mount | Unset mount options
+  set_fact:
+    mount_options: {}
+  when: vault_pki_mount_check|failed

roles/vault/templates/docker.service.j2

@@ -21,6 +21,7 @@ ExecStart={{ docker_bin_dir }}/docker run \
 --cap-add=IPC_LOCK \
 -v {{ vault_cert_dir }}:{{ vault_cert_dir }} \
 -v {{ vault_config_dir }}:{{ vault_config_dir }} \
+-v {{ vault_etcd_cert_dir }}:{{ vault_etcd_cert_dir }} \
 -v {{ vault_log_dir }}:/vault/logs \
 -v {{ vault_roles_dir }}:{{ vault_roles_dir }} \
 -v {{ vault_secrets_dir }}:{{ vault_secrets_dir }} \

roles/vault/templates/rkt.service.j2

@@ -24,6 +24,8 @@ ExecStart=/usr/bin/rkt run \
 --mount=volume=vault-secrets-dir,target={{ vault_secrets_dir }} \
 --volume=vault-roles-dir,kind=host,source={{ vault_roles_dir }} \
 --mount=volume=vault-roles-dir,target={{ vault_roles_dir }} \
+--volume=vault-etcd-cert-dir,kind=host,source={{ vault_etcd_cert_dir }} \
+--mount=volume=vault-etcd-cert-dir,target={{ vault_etcd_cert_dir }} \
 docker://{{ vault_image_repo }}:{{ vault_image_tag }} \
 --name={{ vault_container_name }} --net=host \
 --caps-retain=CAP_IPC_LOCK \