diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9aec99788..72f8de7e9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,6 +54,7 @@ before_script: LOG_LEVEL: "-vv" ETCD_DEPLOYMENT: "docker" KUBELET_DEPLOYMENT: "docker" + VAULT_DEPLOYMENT: "docker" WEAVE_CPU_LIMIT: "100m" MAGIC: "ci check this" @@ -106,6 +107,7 @@ before_script: -e ansible_python_interpreter=${PYPATH} -e ansible_ssh_user=${SSH_USER} -e bootstrap_os=${BOOTSTRAP_OS} + -e cert_management=${CERT_MGMT:-script} -e cloud_provider=gce -e deploy_netchecker=true -e download_localhost=true @@ -115,6 +117,7 @@ before_script: -e kubelet_deployment_type=${KUBELET_DEPLOYMENT} -e local_release_dir=${PWD}/downloads -e resolvconf_mode=${RESOLVCONF_MODE} + -e vault_deployment_type=${VAULT_DEPLOYMENT} cluster.yml @@ -292,6 +295,14 @@ before_script: ETCD_DEPLOYMENT: rkt KUBELET_DEPLOYMENT: rkt +.ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables +# stage: deploy-gce-part1 + KUBE_NETWORK_PLUGIN: canal + CERT_MGMT: vault + CLOUD_IMAGE: ubuntu-1604-xenial + CLOUD_REGION: us-central1-b + CLUSTER_MODE: separate + # Builds for PRs only (premoderated by unit-tests step) and triggers (auto) coreos-calico-sep: stage: deploy-gce-part1 @@ -506,6 +517,17 @@ ubuntu-rkt-sep: except: ['triggers'] only: ['master', /^pr-.*$/] +ubuntu-vault-sep: + stage: deploy-gce-part1 + <<: *job + <<: *gce + variables: + <<: *gce_variables + <<: *ubuntu_vault_sep_variables + when: manual + except: ['triggers'] + only: ['master', /^pr-.*$/] + # Premoderated with manual actions ci-authorized: <<: *job diff --git a/cluster.yml b/cluster.yml index 01b2df105..d1115a57d 100644 --- a/cluster.yml +++ b/cluster.yml @@ -28,14 +28,21 @@ roles: - { role: kubernetes/preinstall, tags: preinstall } - { role: docker, tags: docker } - - { role: rkt, tags: rkt, when: "'rkt' in [ etcd_deployment_type, kubelet_deployment_type ]" } + - role: rkt + tags: rkt + when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]" -- hosts: all +- hosts: etcd:k8s-cluster:vault any_errors_fatal: true roles: - { role: vault, tags: vault, vault_bootstrap: true, when: "cert_management == 'vault'" } -- hosts: etcd:k8s-cluster +- hosts: etcd:!k8s-cluster + any_errors_fatal: true + roles: + - { role: etcd, tags: etcd } + +- hosts: k8s-cluster any_errors_fatal: true roles: - { role: etcd, tags: etcd } diff --git a/docs/vault.md b/docs/vault.md new file mode 100644 index 000000000..446d914c9 --- /dev/null +++ b/docs/vault.md @@ -0,0 +1,92 @@ +Hashicorp Vault Role +==================== + +Overview +-------- + +The Vault role is a two-step process: + +1. Bootstrap + +You cannot start your certificate management service securely with SSL (and +the datastore behind it) without having the certificates in-hand already. This +presents an unfortunate chicken and egg scenario, with one requiring the other. +To solve for this, the Bootstrap step was added. + +This step spins up a temporary instance of Vault to issue certificates for +Vault itself. It then leaves the temporary instance running, so that the Etcd +role can generate certs for itself as well. Eventually, this may be improved +to allow alternate backends (such as Consul), but currently the tasks are +hardcoded to only create a Vault role for Etcd. + +2. Cluster + +This step is where the long-term Vault cluster is started and configured. Its +first task, is to stop any temporary instances of Vault, to free the port for +the long-term. At the end of this task, the entire Vault cluster should be up +and read to go. + + +Keys to the Kingdom +------------------- + +The two most important security pieces of Vault are the ``root_token`` +and ``unsealing_keys``. Both of these values are given exactly once, during +the initialization of the Vault cluster. For convenience, they are saved +to the ``vault_secret_dir`` (default: /etc/vault/secrets) of every host in the +vault group. + +It is *highly* recommended that these secrets are removed from the servers after +your cluster has been deployed, and kept in a safe location of your choosing. +Naturally, the seriousness of the situation depends on what you're doing with +your Kargo cluster, but with these secrets, an attacker will have the ability +to authenticate to almost everything in Kubernetes and decode all private +(HTTPS) traffic on your network signed by Vault certificates. + +For even greater security, you may want to remove and store elsewhere any +CA keys generated as well (e.g. /etc/vault/ssl/ca-key.pem). + +Vault by default encrypts all traffic to and from the datastore backend, all +resting data, and uses TLS for its TCP listener. It is recommended that you +do not change the Vault config to disable TLS, unless you absolutely have to. + + +Usage +----- + +To get the Vault role running, you must to do two things at a minimum: + +1. Assign the ``vault`` group to at least 1 node in your inventory +2. Change ``cert_management`` to be ``vault`` instead of ``script`` + +Nothing else is required, but customization is possible. Check +``roles/vault/defaults/main.yml`` for the different variables that can be +overridden, most common being ``vault_config``, ``vault_port``, and +``vault_deployment_type``. + +Also, if you intend to use a Root or Intermediate CA generated elsewhere, +you'll need to copy the certificate and key to the hosts in the vault group +prior to running the vault role. By default, they'll be located at +``/etc/vault/ssl/ca.pem`` and ``/etc/vault/ssl/ca-key.pem``, respectively. + +Additional Notes: + +- ``groups.vault|first`` is considered the source of truth for Vault variables +- ``vault_leader_url`` is used as pointer for the current running Vault +- Each service should have its own role and credentials. Currently those + credentials are saved to ``/etc/vault/roles//``. The service will + need to read in those credentials, if they want to interact with Vault. + + +Potential Work +-------------- + +- Change the Vault role to not run certain tasks when ``root_token`` and + ``unseal_keys`` are not present. Alternatively, allow user input for these + values when missing. +- Add the ability to start temp Vault with Host, Rkt, or Docker +- Add a dynamic way to change out the backend role creation during Bootstrap, + so other services can be used (such as Consul) +- Segregate Server Cert generation from Auth Cert generation (separate CAs). + This work was partially started with the `auth_cert_backend` tasks, but would + need to be further applied to all roles (particularly Etcd and Kubernetes). diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index f69efd1db..20bbeef42 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -204,6 +204,7 @@ kpm_packages: [] rkt_version: 1.21.0 etcd_deployment_type: docker kubelet_deployment_type: docker +vault_deployment_type: docker efk_enabled: false diff --git a/roles/adduser/defaults/main.yml b/roles/adduser/defaults/main.yml index bd8611d3b..b3a69229c 100644 --- a/roles/adduser/defaults/main.yml +++ b/roles/adduser/defaults/main.yml @@ -14,14 +14,6 @@ addusers: system: yes group: "{{ kube_cert_group }}" createhome: no - vault: - comment: "Hashicorp Vault user" - createhome: no - name: vault - shell: /sbin/nologin - system: yes - - adduser: name: "{{ user.name }}" diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 1e1f2f6e8..6fc594a49 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -26,7 +26,6 @@ calico_cni_version: "v1.5.5" weave_version: 1.8.2 flannel_version: v0.6.2 pod_infra_version: 3.0 -vault_version: 0.6.3 # Download URL's etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd" diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index 2fb9434bf..7b49f4f0e 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -94,7 +94,7 @@ - name: "Set default value for 'container_changed' to false" set_fact: - container_changed: "{{pull_required|bool|default(false)}}" + container_changed: "{{pull_required|default(false)|bool}}" - name: "Update the 'container_changed' fact" set_fact: diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index a5ba5a1b3..a81670fd3 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -2,7 +2,6 @@ etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/" etcd_config_dir: /etc/ssl/etcd -# Role vault.boostrap has an implicit requirement on this var. It should be set at a higher level (inventory+) etcd_cert_dir: "{{ etcd_config_dir }}/ssl" etcd_cert_group: root @@ -16,3 +15,5 @@ etcd_memory_limit: 512M # Uncomment to set CPU share for etcd #etcd_cpu_limit: 300m + +etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr', [])) }}" diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml index addd81053..bff76a129 100644 --- a/roles/etcd/meta/main.yml +++ b/roles/etcd/meta/main.yml @@ -6,3 +6,5 @@ dependencies: - role: download file: "{{ downloads.etcd }}" tags: download + +# NOTE: Dynamic task dependency on Vault Role if cert_management == "vault" diff --git a/roles/etcd/tasks/gen_certs.yml b/roles/etcd/tasks/gen_certs_script.yml similarity index 100% rename from roles/etcd/tasks/gen_certs.yml rename to roles/etcd/tasks/gen_certs_script.yml diff --git a/roles/etcd/tasks/gen_certs_vault.yml b/roles/etcd/tasks/gen_certs_vault.yml new file mode 100644 index 000000000..144e3b658 --- /dev/null +++ b/roles/etcd/tasks/gen_certs_vault.yml @@ -0,0 +1,77 @@ +--- + +- name: gen_certs_vault | Read in the local credentials + command: cat /etc/vault/roles/etcd/userpass + register: etcd_vault_creds_cat + when: inventory_hostname == groups.etcd|first + +- name: gen_certs_vault | Set facts for read Vault Creds + set_fact: + etcd_vault_creds: "{{ hostvars[groups.etcd|first]['etcd_vault_creds_cat']['stdout']|from_json }}" + when: inventory_hostname == groups.etcd|first + +- name: gen_certs_vault | Log into Vault and obtain an token + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ etcd_vault_creds.username }}" + headers: + Accept: application/json + Content-Type: application/json + method: POST + body_format: json + body: + password: "{{ etcd_vault_creds.password }}" + register: etcd_vault_login_result + when: inventory_hostname == groups.etcd|first + +- name: gen_certs_vault | Set fact for Vault API token + set_fact: + etcd_vault_headers: + Accept: application/json + Content-Type: application/json + X-Vault-Token: "{{ hostvars[groups.etcd|first]['etcd_vault_login_result']['json']['auth']['client_token'] }}" + +# Issue master certs to Etcd nodes +- include: ../../vault/tasks/shared/issue_cert.yml + vars: + issue_cert_alt_names: "{{ groups.etcd + ['localhost'] }}" + issue_cert_copy_ca: "{{ item == etcd_master_certs_needed|first }}" + issue_cert_file_group: "{{ etcd_cert_group }}" + issue_cert_file_owner: kube + issue_cert_headers: "{{ etcd_vault_headers }}" + issue_cert_hosts: "{{ groups.etcd }}" + issue_cert_ip_sans: >- + [ + {%- for host in groups.etcd -%} + "{{ hostvars[host]['ansible_default_ipv4']['address'] }}", + {%- endfor -%} + "127.0.0.1","::1" + ] + issue_cert_path: "{{ item }}" + issue_cert_role: etcd + issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + with_items: "{{ etcd_master_certs_needed|d([]) }}" + when: inventory_hostname in groups.etcd + notify: set etcd_secret_changed + +# Issue node certs to everyone else +- include: ../../vault/tasks/shared/issue_cert.yml + vars: + issue_cert_alt_names: "{{ etcd_node_cert_hosts }}" + issue_cert_copy_ca: "{{ item == etcd_node_certs_needed|first }}" + issue_cert_file_group: "{{ etcd_cert_group }}" + issue_cert_file_owner: kube + issue_cert_headers: "{{ etcd_vault_headers }}" + issue_cert_hosts: "{{ etcd_node_cert_hosts }}" + issue_cert_ip_sans: >- + [ + {%- for host in etcd_node_cert_hosts -%} + "{{ hostvars[host]['ansible_default_ipv4']['address'] }}", + {%- endfor -%} + "127.0.0.1","::1" + ] + issue_cert_path: "{{ item }}" + issue_cert_role: etcd + issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + with_items: "{{ etcd_node_certs_needed|d([]) }}" + when: inventory_hostname in etcd_node_cert_hosts + notify: set etcd_secret_changed diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index 394e5de64..6e952cd33 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -1,10 +1,24 @@ --- - include: pre_upgrade.yml tags: etcd-pre-upgrade + - include: check_certs.yml + when: cert_management == "script" tags: [etcd-secrets, facts] -- include: gen_certs.yml +- include: gen_certs_script.yml + when: cert_management == "script" tags: etcd-secrets + +- include: sync_etcd_master_certs.yml + when: cert_management == "vault" and inventory_hostname in groups.etcd + tags: etcd-secrets +- include: sync_etcd_node_certs.yml + when: cert_management == "vault" and inventory_hostname in etcd_node_cert_hosts + tags: etcd-secrets +- include: gen_certs_vault.yml + when: cert_management == "vault" and (etcd_master_certs_needed|d() or etcd_node_certs_needed|d()) + tags: etcd-secrets + - include: "install_{{ etcd_deployment_type }}.yml" when: is_etcd_master tags: upgrade diff --git a/roles/etcd/tasks/sync_etcd_master_certs.yml b/roles/etcd/tasks/sync_etcd_master_certs.yml new file mode 100644 index 000000000..c6badd3c2 --- /dev/null +++ b/roles/etcd/tasks/sync_etcd_master_certs.yml @@ -0,0 +1,38 @@ +--- + +- name: sync_etcd_master_certs | Create list of master certs needing creation + set_fact: + etcd_master_cert_list: >- + {{ etcd_master_cert_list|default([]) + [ + "admin-" + item + ".pem", + "member-" + item + ".pem" + ] }} + with_items: "{{ groups.etcd }}" + +- include: ../../vault/tasks/shared/sync_file.yml + vars: + sync_file: "{{ item }}" + sync_file_dir: "{{ etcd_cert_dir }}" + sync_file_hosts: "{{ groups.etcd }}" + sync_file_is_cert: true + with_items: "{{ etcd_master_cert_list|d([]) }}" + +- name: sync_etcd_certs | Set facts for etcd sync_file results + set_fact: + etcd_master_certs_needed: "{{ etcd_master_certs_needed|default([]) + [item.path] }}" + with_items: "{{ sync_file_results|d([]) }}" + when: item.no_srcs|bool + +- name: sync_etcd_certs | Unset sync_file_results after etcd certs sync + set_fact: + sync_file_results: [] + +- include: ../../vault/tasks/shared/sync_file.yml + vars: + sync_file: ca.pem + sync_file_dir: "{{ etcd_cert_dir }}" + sync_file_hosts: "{{ groups.etcd }}" + +- name: sync_etcd_certs | Unset sync_file_results after ca.pem sync + set_fact: + sync_file_results: [] diff --git a/roles/etcd/tasks/sync_etcd_node_certs.yml b/roles/etcd/tasks/sync_etcd_node_certs.yml new file mode 100644 index 000000000..2f82dcffd --- /dev/null +++ b/roles/etcd/tasks/sync_etcd_node_certs.yml @@ -0,0 +1,34 @@ +--- + +- name: sync_etcd_node_certs | Create list of node certs needing creation + set_fact: + etcd_node_cert_list: "{{ etcd_node_cert_list|default([]) + ['node-' + item + '.pem'] }}" + with_items: "{{ etcd_node_cert_hosts }}" + +- include: ../../vault/tasks/shared/sync_file.yml + vars: + sync_file: "{{ item }}" + sync_file_dir: "{{ etcd_cert_dir }}" + sync_file_hosts: "{{ etcd_node_cert_hosts }}" + sync_file_is_cert: true + with_items: "{{ etcd_node_cert_list|d([]) }}" + +- name: sync_etcd_node_certs | Set facts for etcd sync_file results + set_fact: + etcd_node_certs_needed: "{{ etcd_node_certs_needed|default([]) + [item.path] }}" + with_items: "{{ sync_file_results|d([]) }}" + when: item.no_srcs|bool + +- name: sync_etcd_node_certs | Unset sync_file_results after etcd node certs + set_fact: + sync_file_results: [] + +- include: ../../vault/tasks/shared/sync_file.yml + vars: + sync_file: ca.pem + sync_file_dir: "{{ etcd_cert_dir }}" + sync_file_hosts: "{{ etcd_node_cert_hosts }}" + +- name: sync_etcd_node_certs | Unset sync_file_results after ca.pem + set_fact: + sync_file_results: [] diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index bfe4a8cc8..e949e87de 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -25,6 +25,20 @@ template: "src=kubelet.{{ kubelet_deployment_type }}.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes" notify: restart kubelet +- name: install | Set SSL CA directories + set_fact: + ssl_ca_dirs: "[ + {% if ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] -%} + '/usr/share/ca-certificates', + {% elif ansible_os_family == 'RedHat' -%} + '/etc/pki/tls', + '/etc/pki/ca-trust', + {% elif ansible_os_family == 'Debian' -%} + '/usr/share/ca-certificates', + {% endif -%} + ]" + tags: facts + - name: install | Install kubelet launch script template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes notify: restart kubelet diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index 610f74dcd..6aa7d7e7c 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -10,6 +10,7 @@ common_required_pkgs: - rsync - bash-completion - socat + - unzip # Set to true if your network does not support IPv6 # This maybe necessary for pulling Docker images from diff --git a/roles/kubernetes/secrets/meta/main.yml b/roles/kubernetes/secrets/meta/main.yml new file mode 100644 index 000000000..dca734575 --- /dev/null +++ b/roles/kubernetes/secrets/meta/main.yml @@ -0,0 +1,2 @@ +--- +# NOTE: Dynamic task dependency on Vault Role if cert_management == "vault" diff --git a/roles/kubernetes/secrets/tasks/gen_certs.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml similarity index 93% rename from roles/kubernetes/secrets/tasks/gen_certs.yml rename to roles/kubernetes/secrets/tasks/gen_certs_script.yml index 484afff63..ebcfb7d81 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml @@ -160,20 +160,6 @@ {%- endif %} tags: facts -- name: SSL CA directories | Set SSL CA directories - set_fact: - ssl_ca_dirs: "[ - {% if ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] -%} - '/usr/share/ca-certificates', - {% elif ansible_os_family == 'RedHat' -%} - '/etc/pki/tls', - '/etc/pki/ca-trust', - {% elif ansible_os_family == 'Debian' -%} - '/usr/share/ca-certificates', - {% endif -%} - ]" - tags: facts - - name: Gen_certs | add CA to trusted CA dir copy: src: "{{ kube_cert_dir }}/ca.pem" diff --git a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml new file mode 100644 index 000000000..5a7c4827b --- /dev/null +++ b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml @@ -0,0 +1,84 @@ +--- + +- name: gen_certs_vault | Read in the local credentials + command: cat /etc/vault/roles/kube/userpass + register: kube_vault_creds_cat + when: inventory_hostname == groups['k8s-cluster']|first + +- name: gen_certs_vault | Set facts for read Vault Creds + set_fact: + kube_vault_creds: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_creds_cat']['stdout'] | from_json }}" + when: inventory_hostname == groups['k8s-cluster']|first + +- name: gen_certs_vault | Log into Vault and obtain an token + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ kube_vault_creds.username }}" + headers: + Accept: application/json + Content-Type: application/json + method: POST + body_format: json + body: + password: "{{ kube_vault_creds.password }}" + register: kube_vault_login_result + when: inventory_hostname == groups['k8s-cluster']|first + +- name: gen_certs_vault | Set fact for Vault API token + set_fact: + kube_vault_headers: + Accept: application/json + Content-Type: application/json + X-Vault-Token: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_login_result']['json']['auth']['client_token'] }}" + +# Issue certs to kube-master nodes +- include: ../../../vault/tasks/shared/issue_cert.yml + vars: + issue_cert_copy_ca: "{{ item == kube_master_certs_needed|first }}" + issue_cert_file_group: "{{ kube_cert_group }}" + issue_cert_file_owner: kube + issue_cert_headers: "{{ kube_vault_headers }}" + issue_cert_hosts: "{{ groups['kube-master'] }}" + issue_cert_path: "{{ item }}" + issue_cert_role: kube + issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + with_items: "{{ kube_master_certs_needed|d([]) }}" + when: inventory_hostname in groups['kube-master'] + +- include: ../../../vault/tasks/shared/issue_cert.yml + vars: + issue_cert_alt_names: >- + {{ + groups['kube-master'] + + ['kubernetes.default.svc.cluster.local', 'kubernetes.default.svc', 'kubernetes.default', 'kubernetes'] + + ['localhost'] + }} + issue_cert_file_group: "{{ kube_cert_group }}" + issue_cert_file_owner: kube + issue_cert_headers: "{{ kube_vault_headers }}" + issue_cert_hosts: "{{ groups['kube-master'] }}" + issue_cert_ip_sans: >- + [ + {%- for host in groups['kube-master'] -%} + "{{ hostvars[host]['ansible_default_ipv4']['address'] }}", + {%- endfor -%} + "127.0.0.1","::1","{{ kube_apiserver_ip }}" + ] + issue_cert_path: "{{ item }}" + issue_cert_role: kube + issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + with_items: "{{ kube_api_certs_needed|d([]) }}" + when: inventory_hostname in groups['kube-master'] + +# Issue node certs to k8s-cluster nodes +- include: ../../../vault/tasks/shared/issue_cert.yml + vars: + issue_cert_copy_ca: "{{ item == kube_node_certs_needed|first }}" + issue_cert_file_group: "{{ kube_cert_group }}" + issue_cert_file_owner: kube + issue_cert_headers: "{{ kube_vault_headers }}" + issue_cert_hosts: "{{ groups['k8s-cluster'] }}" + issue_cert_path: "{{ item }}" + issue_cert_role: kube + issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + with_items: "{{ kube_node_certs_needed|d([]) }}" + when: inventory_hostname in groups['k8s-cluster'] diff --git a/roles/kubernetes/secrets/tasks/main.yml b/roles/kubernetes/secrets/tasks/main.yml index 4d25a94af..f442b62b3 100644 --- a/roles/kubernetes/secrets/tasks/main.yml +++ b/roles/kubernetes/secrets/tasks/main.yml @@ -70,7 +70,19 @@ delegate_to: "{{groups['kube-master'][0]}}" when: gen_tokens|default(false) -- include: gen_certs.yml +- include: gen_certs_script.yml + when: cert_management == "script" tags: k8s-secrets + +- include: sync_kube_master_certs.yml + when: cert_management == "vault" and inventory_hostname in groups['kube-master'] + tags: k8s-secrets +- include: sync_kube_node_certs.yml + when: cert_management == "vault" and inventory_hostname in groups['k8s-cluster'] + tags: k8s-secrets +- include: gen_certs_vault.yml + when: cert_management == "vault" + tags: k8s-secrets + - include: gen_tokens.yml tags: k8s-secrets diff --git a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml new file mode 100644 index 000000000..0561d6581 --- /dev/null +++ b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml @@ -0,0 +1,58 @@ +--- + +- name: sync_kube_master_certs | Create list of needed kube admin certs + set_fact: + kube_master_cert_list: "{{ kube_master_cert_list|d([]) + ['admin-' + item + '.pem'] }}" + with_items: "{{ groups['kube-master'] }}" + +- include: ../../../vault/tasks/shared/sync_file.yml + vars: + sync_file: "{{ item }}" + sync_file_dir: "{{ kube_cert_dir }}" + sync_file_group: "{{ kube_cert_group }}" + sync_file_hosts: "{{ groups['kube-master'] }}" + sync_file_is_cert: true + sync_file_owner: kube + with_items: "{{ kube_master_cert_list|d([]) }}" + +- name: sync_kube_master_certs | Set facts for kube admin sync_file results + set_fact: + kube_master_certs_needed: "{{ kube_master_certs_needed|default([]) + [item.path] }}" + with_items: "{{ sync_file_results|d([]) }}" + when: item.no_srcs|bool + +- name: sync_kube_master_certs | Unset sync_file_results after kube admin certs + set_fact: + sync_file_results: [] + +- include: ../../../vault/tasks/shared/sync_file.yml + vars: + sync_file: "apiserver.pem" + sync_file_dir: "{{ kube_cert_dir }}" + sync_file_group: "{{ kube_cert_group }}" + sync_file_hosts: "{{ groups['kube-master'] }}" + sync_file_is_cert: true + sync_file_owner: kube + +- name: sync_kube_master_certs | Set facts for apiserver sync_file results + set_fact: + kube_api_certs_needed: "{{ item.path }}" + with_items: "{{ sync_file_results|d([]) }}" + when: "{{ item.no_srcs }}" + +- name: sync_kube_master_certs | Unset sync_file_results after apiserver cert + set_fact: + sync_file_results: [] + + +- include: ../../../vault/tasks/shared/sync_file.yml + vars: + sync_file: ca.pem + sync_file_dir: "{{ kube_cert_dir }}" + sync_file_group: "{{ kube_cert_group }}" + sync_file_hosts: "{{ groups['kube-master'] }}" + sync_file_owner: kube + +- name: sync_kube_master_certs | Unset sync_file_results after ca.pem + set_fact: + sync_file_results: [] diff --git a/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml b/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml new file mode 100644 index 000000000..9d6deb563 --- /dev/null +++ b/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml @@ -0,0 +1,38 @@ +--- + +- name: sync_kube_node_certs | Create list of needed certs + set_fact: + kube_node_cert_list: "{{ kube_node_cert_list|default([]) + ['node-' + item + '.pem'] }}" + with_items: "{{ groups['k8s-cluster'] }}" + +- include: ../../../vault/tasks/shared/sync_file.yml + vars: + sync_file: "{{ item }}" + sync_file_dir: "{{ kube_cert_dir }}" + sync_file_group: "{{ kuber_cert_group }}" + sync_file_hosts: "{{ groups['k8s-cluster'] }}" + sync_file_is_cert: true + sync_file_owner: kube + with_items: "{{ kube_node_cert_list|default([]) }}" + +- name: sync_kube_node_certs | Set facts for kube-master sync_file results + set_fact: + kube_node_certs_needed: "{{ kube_node_certs_needed|default([]) + [item.path] }}" + with_items: "{{ sync_file_results|d([]) }}" + when: item.no_srcs|bool + +- name: sync_kube_node_certs | Unset sync_file_results after kube node certs + set_fact: + sync_file_results: [] + +- include: ../../../vault/tasks/shared/sync_file.yml + vars: + sync_file: ca.pem + sync_file_dir: "{{ kube_cert_dir }}" + sync_file_group: "{{ kuber_cert_group }}" + sync_file_hosts: "{{ groups['k8s-cluster'] }}" + sync_file_owner: kube + +- name: sync_kube_node_certs | Unset sync_file_results after ca.pem + set_fact: + sync_file_results: [] diff --git a/roles/vault/defaults/main.yml b/roles/vault/defaults/main.yml index b42345a18..4a0a60982 100644 --- a/roles/vault/defaults/main.yml +++ b/roles/vault/defaults/main.yml @@ -1,18 +1,27 @@ --- +vault_adduser_vars: + comment: "Hashicorp Vault User" + createhome: no + name: vault + shell: /sbin/nologin + system: yes +vault_base_dir: /etc/vault +# https://releases.hashicorp.com/vault/0.6.4/vault_0.6.4_SHA256SUMS +vault_binary_checksum: 04d87dd553aed59f3fe316222217a8d8777f40115a115dac4d88fac1611c51a6 vault_bootstrap: false vault_ca_options: common_name: kube-cluster-ca format: pem ttl: 87600h -vault_cert_dir: "{{ vault_config_dir }}/ssl" +vault_cert_dir: "{{ vault_base_dir }}/ssl" vault_client_headers: Accept: "application/json" Content-Type: "application/json" vault_config: backend: etcd: - address: "https://{{ hostvars[groups.etcd[0]]['ansible_default_ipv4']['address'] }}:2379" + address: "{{ vault_etcd_url }}" ha_enabled: "true" redirect_addr: "https://{{ ansible_default_ipv4.address }}:{{ vault_port }}" tls_ca_file: "{{ vault_cert_dir }}/ca.pem" @@ -23,21 +32,51 @@ vault_config: address: "0.0.0.0:{{ vault_port }}" tls_cert_file: "{{ vault_cert_dir }}/api.pem" tls_key_file: "{{ vault_cert_dir }}/api-key.pem" - max_lease_ttl: 720h -vault_config_dir: /etc/vault + max_lease_ttl: "{{ vault_max_lease_ttl }}" +vault_config_dir: "{{ vault_base_dir }}/config" vault_container_name: kube-hashicorp-vault +# This variable is meant to match the GID of vault inside Hashicorp's official Vault Container vault_default_lease_ttl: 720h vault_default_role_permissions: allow_any_name: true vault_deployment_type: docker -vault_etcd_needs_gen: false -vault_etcd_sync_hosts: [] -vault_max_lease_ttl: 87600h +vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip" +vault_download_vars: + container: "{{ vault_deployment_type != 'host' }}" + dest: "vault/vault_{{ vault_version }}_linux_amd64.zip" + enabled: true + mode: "0755" + owner: "vault" + repo: "{{ vault_image_repo }}" + sha256: "{{ vault_binary_checksum if vault_deployment_type == 'host' else vault_digest_checksum|d(none) }}" + source_url: "{{ vault_download_url }}" + tag: "{{ vault_image_tag }}" + unarchive: true + url: "{{ vault_download_url }}" + version: "{{ vault_version }}" +vault_etcd_url: "https://{{ hostvars[groups.etcd[0]]['ansible_default_ipv4']['address'] }}:2379" +vault_image_repo: "vault" +vault_image_tag: "{{ vault_version }}" +vault_log_dir: "/var/log/vault" +vault_max_lease_ttl: 87600h vault_needs_gen: false vault_port: 8200 +# Although "cert" is an option, ansible has no way to auth via cert until +# upstream merges: https://github.com/ansible/ansible/pull/18141 +vault_role_auth_method: userpass +vault_roles: + - name: etcd + group: etcd + policy_rules: default + role_options: default + - name: kube + group: k8s-cluster + policy_rules: default + role_options: default +vault_roles_dir: "{{ vault_base_dir }}/roles" vault_secret_shares: 1 vault_secret_threshold: 1 -vault_secrets_dir: "{{ vault_config_dir }}/secrets" +vault_secrets_dir: "{{ vault_base_dir }}/secrets" vault_temp_config: default_lease_ttl: "{{ vault_default_lease_ttl }}" backend: @@ -45,21 +84,7 @@ vault_temp_config: path: /vault/file listener: tcp: - address: "0.0.0.0:{{ vault_temp_port }}" + address: "0.0.0.0:{{ vault_port }}" tls_disable: "true" -vault_temp_port: 8201 - -# This should be set higher up, but setting defaults here to avoid issues -etcd_cert_dir: /etc/ssl/etcd/ssl -kube_cert_dir: /etc/kubernetes/ssl - -# Sync cert defaults (should be role, once include_role is fixed) -sync_file: '' -sync_file_dir: '' -sync_file_host_count: 0 -sync_file_is_cert: false -sync_file_key_path: '' -sync_file_key_srcs: [] -sync_file_path: '' -sync_file_results: [] -sync_file_srcs: [] +vault_temp_container_name: vault-temp +vault_version: 0.6.4 diff --git a/roles/vault/meta/main.yml b/roles/vault/meta/main.yml index 747c3ad0d..ba559e361 100644 --- a/roles/vault/meta/main.yml +++ b/roles/vault/meta/main.yml @@ -1,6 +1,8 @@ --- -# Implicit requirement on sync_cert role (include_role used in tasks) dependencies: + - role: adduser + user: "{{ vault_adduser_vars }}" - role: download - file: "{{ downloads.vault }}" + file: "{{ vault_download_vars }}" + tags: download diff --git a/roles/vault/tasks/bootstrap/ca_trust.yml b/roles/vault/tasks/bootstrap/ca_trust.yml index 4ba877aac..2bcfcc2fa 100644 --- a/roles/vault/tasks/bootstrap/ca_trust.yml +++ b/roles/vault/tasks/bootstrap/ca_trust.yml @@ -1,12 +1,12 @@ --- -- name: trust_ca | pull CA from cert from groups.vault|first +- name: bootstrap/ca_trust | pull CA from cert from groups.vault|first command: "cat {{ vault_cert_dir }}/ca.pem" register: vault_cert_file_cat when: inventory_hostname == groups.vault|first # This part is mostly stolen from the etcd role -- name: trust_ca | target ca-certificate store file +- name: bootstrap/ca_trust | target ca-certificate store file set_fact: ca_cert_path: >- {% if ansible_os_family == "Debian" -%} @@ -17,16 +17,16 @@ /etc/ssl/certs/kube-cluster-ca.pem {%- endif %} -- name: trust_ca | add CA to trusted CA dir +- name: bootstrap/ca_trust | add CA to trusted CA dir copy: content: "{{ hostvars[groups.vault|first]['vault_cert_file_cat']['stdout'] }}" dest: "{{ ca_cert_path }}" register: vault_ca_cert -- name: trust_ca | update ca-certificates (Debian/Ubuntu/CoreOS) +- name: bootstrap/ca_trust | update ca-certificates (Debian/Ubuntu/CoreOS) command: update-ca-certificates when: vault_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"] -- name: trust_ca | update ca-certificates (RedHat) +- name: bootstrap/ca_trust | update ca-certificates (RedHat) command: update-ca-trust extract when: vault_ca_cert.changed and ansible_os_family == "RedHat" diff --git a/roles/vault/tasks/bootstrap/create_etcd_role.yml b/roles/vault/tasks/bootstrap/create_etcd_role.yml new file mode 100644 index 000000000..57518f944 --- /dev/null +++ b/roles/vault/tasks/bootstrap/create_etcd_role.yml @@ -0,0 +1,10 @@ +--- + +- include: ../shared/create_role.yml + vars: + create_role_name: "{{ item.name }}" + create_role_group: "{{ item.group }}" + create_role_policy_rules: "{{ item.policy_rules }}" + create_role_options: "{{ item.role_options }}" + with_items: "{{ vault_roles }}" + when: item.name == "etcd" diff --git a/roles/vault/tasks/bootstrap/gen_auth_ca.yml b/roles/vault/tasks/bootstrap/gen_auth_ca.yml new file mode 100644 index 000000000..10313ecea --- /dev/null +++ b/roles/vault/tasks/bootstrap/gen_auth_ca.yml @@ -0,0 +1,21 @@ +--- + +- name: bootstrap/gen_auth_ca | Generate Root CA + uri: + url: "{{ vault_leader_url }}/v1/auth-pki/root/generate/exported" + headers: "{{ vault_headers }}" + method: POST + body_format: json + body: "{{ vault_ca_options }}" + register: vault_auth_ca_gen + when: inventory_hostname == groups.vault|first + +- name: bootstrap/gen_auth_ca | Copy auth CA cert to Vault nodes + copy: + content: "{{ hostvars[groups.vault|first]['vault_auth_ca_gen']['json']['data']['certificate'] }}" + dest: "{{ vault_cert_dir }}/auth-ca.pem" + +- name: bootstrap/gen_auth_ca | Copy auth CA key to Vault nodes + copy: + content: "{{ hostvars[groups.vault|first]['vault_auth_ca_gen']['json']['data']['private_key'] }}" + dest: "{{ vault_cert_dir }}/auth-ca-key.pem" diff --git a/roles/vault/tasks/bootstrap/gen_ca.yml b/roles/vault/tasks/bootstrap/gen_ca.yml new file mode 100644 index 000000000..ab1cb6345 --- /dev/null +++ b/roles/vault/tasks/bootstrap/gen_ca.yml @@ -0,0 +1,31 @@ +--- + +- name: bootstrap/gen_ca | Ensure vault_cert_dir exists + file: + mode: 0755 + path: "{{ vault_cert_dir }}" + state: directory + +- name: bootstrap/gen_ca | Generate Root CA in vault-temp + uri: + url: "{{ vault_leader_url }}/v1/pki/root/generate/exported" + headers: "{{ vault_headers }}" + method: POST + body_format: json + body: "{{ vault_ca_options }}" + register: vault_ca_gen + when: inventory_hostname == groups.vault|first and vault_ca_cert_needed + +- name: bootstrap/gen_ca | Copy root CA cert locally + copy: + content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['certificate'] }}" + dest: "{{ vault_cert_dir }}/ca.pem" + mode: 0644 + when: vault_ca_cert_needed + +- name: bootstrap/gen_ca | Copy root CA key locally + copy: + content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['private_key'] }}" + dest: "{{ vault_cert_dir }}/ca-key.pem" + mode: 0640 + when: vault_ca_cert_needed diff --git a/roles/vault/tasks/bootstrap/gen_etcd_certs.yml b/roles/vault/tasks/bootstrap/gen_etcd_certs.yml deleted file mode 100644 index 5619aaa93..000000000 --- a/roles/vault/tasks/bootstrap/gen_etcd_certs.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- - -- name: bootstrap/gen_etcd_certs | Add the etcd role - uri: - url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}/v1/pki/roles/etcd" - headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" - method: POST - body_format: json - body: - allow_any_name: true - status_code: 204 - when: inventory_hostname == groups.etcd|first - -- include: ../gen_cert.yml - vars: - gen_cert_alt_names: "{{ groups.etcd | join(',') }},localhost" - gen_cert_copy_ca: "{{ true if item == vault_etcd_certs_needed|first else false }}" - gen_cert_hosts: "{{ groups.etcd }}" - gen_cert_ip_sans: >- - {%- for host in groups.etcd -%} - {{ hostvars[host]["ansible_default_ipv4"]["address"] }} - {%- if not loop.last -%},{%- endif -%} - {%- endfor -%} - ,127.0.0.1,::1 - gen_cert_path: "{{ item }}" - gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" - gen_cert_vault_role: etcd - gen_cert_vault_url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}" - with_items: "{{ vault_etcd_certs_needed|default([]) }}" diff --git a/roles/vault/tasks/bootstrap/gen_etcd_node_certs.yml b/roles/vault/tasks/bootstrap/gen_etcd_node_certs.yml deleted file mode 100644 index 3ef48fd57..000000000 --- a/roles/vault/tasks/bootstrap/gen_etcd_node_certs.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- - -- name: bootstrap/gen_etcd_node_certs | Add the etcd role - uri: - url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}/v1/pki/roles/etcd" - headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" - method: POST - body_format: json - body: - allow_any_name: true - status_code: 204 - when: inventory_hostname == groups["k8s-cluster"]|first - -- include: ../gen_cert.yml - vars: - gen_cert_alt_names: "{{ groups['k8s-cluster'] | union(groups.etcd) | join(',') }},localhost" - gen_cert_copy_ca: "{{ true if item == vault_etcd_node_certs_needed|first else false }}" - gen_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.etcd) }}" - gen_cert_ip_sans: >- - {%- for host in groups["k8s-cluster"] | union(groups.etcd) -%} - {{ hostvars[host]["ansible_default_ipv4"]["address"] }} - {%- if not loop.last -%},{%- endif -%} - {%- endfor -%} - ,127.0.0.1,::1 - gen_cert_path: "{{ item }}" - gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" - gen_cert_vault_role: etcd - gen_cert_vault_url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}" - with_items: "{{ vault_etcd_node_certs_needed|default([]) }}" diff --git a/roles/vault/tasks/bootstrap/gen_vault_certs.yml b/roles/vault/tasks/bootstrap/gen_vault_certs.yml index 79b60e541..4a7f4ed31 100644 --- a/roles/vault/tasks/bootstrap/gen_vault_certs.yml +++ b/roles/vault/tasks/bootstrap/gen_vault_certs.yml @@ -1,47 +1,8 @@ --- -- name: bootstrap/gen_vault_certs | Ensure vault_cert_dir exists - file: - path: "{{ vault_cert_dir }}" - state: directory - -- name: bootstrap/gen_vault_certs | Generate Root CA in vault-temp - uri: - url: "http://localhost:{{ vault_temp_port }}/v1/pki/root/generate/exported" - headers: "{{ vault_headers }}" - method: POST - body_format: json - body: "{{ vault_ca_options }}" - register: vault_ca_gen - when: inventory_hostname == groups.vault|first and vault_ca_cert_needed - -- name: bootstrap/gen_vault_certs | Set facts for ca cert and key - set_fact: - vault_ca_cert: "{{ vault_ca_gen.json.data.certificate }}" - vault_ca_key: "{{ vault_ca_gen.json.data.private_key }}" - when: inventory_hostname == groups.vault|first and vault_ca_cert_needed - -- name: bootstrap/gen_vault_certs | Set cert and key facts for all hosts other than groups.vault|first - set_fact: - vault_ca_cert: "{{ hostvars[groups.vault|first]['vault_ca_cert'] }}" - vault_ca_key: "{{ hostvars[groups.vault|first]['vault_ca_key'] }}" - when: inventory_hostname != groups.vault|first and vault_ca_cert_needed - -- name: bootstrap/gen_vault_certs | Copy root CA cert locally - copy: - content: "{{ vault_ca_cert }}" - dest: "{{ vault_cert_dir }}/ca.pem" - when: vault_ca_cert_needed - -- name: bootstrap/gen_vault_certs | Copy root CA key locally - copy: - content: "{{vault_ca_key}}" - dest: "{{vault_cert_dir}}/ca-key.pem" - when: vault_ca_cert_needed - - name: boostrap/gen_vault_certs | Add the vault role uri: - url: "http://localhost:{{ vault_temp_port }}/v1/pki/roles/vault" + url: "{{ vault_leader_url }}/v1/pki/roles/vault" headers: "{{ vault_headers }}" method: POST body_format: json @@ -49,18 +10,19 @@ status_code: 204 when: inventory_hostname == groups.vault|first and vault_api_cert_needed -- include: ../gen_cert.yml +- include: ../shared/issue_cert.yml vars: - gen_cert_alt_names: "{{ groups.vault | join(',') }},localhost" - gen_cert_hosts: "{{ groups.vault }}" - gen_cert_ip_sans: >- + issue_cert_alt_names: "{{ groups.vault + ['localhost'] }}" + issue_cert_hosts: "{{ groups.vault }}" + issue_cert_ip_sans: >- + [ {%- for host in groups.vault -%} - {{ hostvars[host]["ansible_default_ipv4"]["address"] }} - {%- if not loop.last -%},{%- endif -%} + "{{ hostvars[host]['ansible_default_ipv4']['address'] }}", {%- endfor -%} - ,127.0.0.1,::1 - gen_cert_path: "{{ vault_cert_dir }}/api.pem" - gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" - gen_cert_vault_role: vault - gen_cert_vault_url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}" + "127.0.0.1","::1" + ] + issue_cert_path: "{{ vault_cert_dir }}/api.pem" + issue_cert_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + issue_cert_role: vault + issue_cert_url: "{{ vault_leader_url }}" when: vault_api_cert_needed diff --git a/roles/vault/tasks/bootstrap/main.yml b/roles/vault/tasks/bootstrap/main.yml index 4f73bd78c..edd2912d3 100644 --- a/roles/vault/tasks/bootstrap/main.yml +++ b/roles/vault/tasks/bootstrap/main.yml @@ -1,60 +1,58 @@ --- +- include: ../shared/check_vault.yml + when: inventory_hostname in groups.vault +- include: sync_secrets.yml + when: inventory_hostname in groups.vault +- include: ../shared/find_leader.yml + when: inventory_hostname in groups.vault and vault_cluster_is_initialized|d() + ## Sync Certs -- include: bootstrap/sync_vault_certs.yml +- include: sync_vault_certs.yml when: inventory_hostname in groups.vault -- include: bootstrap/sync_etcd_certs.yml - when: inventory_hostname in groups.etcd - -- include: bootstrap/sync_etcd_node_certs.yml - when: inventory_hostname in groups["k8s-cluster"] | union(groups.etcd) - ## Generate Certs # Start a temporary instance of Vault -- include: bootstrap/start_vault_temp.yml - when: >- - ( hostvars[groups.etcd|first].get("vault_etcd_certs_needed", [])|length > 0 or - hostvars[groups.etcd|first].get("vault_etcd_node_certs_needed", [])|length > 0 or - hostvars[groups.vault|first]["vault_ca_cert_needed"] ) and - inventory_hostname == groups.vault|first - -# Generate root CA certs for Vault if none exist -- include: bootstrap/gen_vault_certs.yml - when: >- - ( hostvars[groups.vault|first]["vault_ca_cert_needed"] or - hostvars[groups.vault|first]["vault_api_cert_needed"] ) and - inventory_hostname in groups.vault - -# Change vault-temp's issuing CA to use existing ca.pem/ca-key.pem -- include: config_ca.yml - vars: - vault_url: "http://{{ groups.vault|first }}:{{ vault_temp_port }}" - when: >- - ( hostvars[groups.etcd|first].get("vault_etcd_certs_needed", [])|length > 0 or - hostvars[groups["k8s-cluster"]|first].get("vault_etcd_node_certs_needed", [])|length > 0 or - hostvars[groups.vault|first]["vault_api_cert_needed"] ) and - not hostvars[groups.vault|first]["vault_ca_cert_needed"] and - inventory_hostname == groups.vault|first - -# Generate etcd certs for etcd cluster members -- include: bootstrap/gen_etcd_certs.yml - when: >- - hostvars[groups.etcd|first].get("vault_etcd_certs_needed", [])|length > 0 and - inventory_hostname in groups.etcd - -# Generate etcd node certs for all k8s-cluster -- include: bootstrap/gen_etcd_node_certs.yml - when: >- - hostvars[groups["k8s-cluster"]|first].get("vault_etcd_node_certs_needed", [])|length > 0 and - inventory_hostname in groups["k8s-cluster"] | union(groups.etcd) - -# Stop temporary vault -- include: bootstrap/stop_vault_temp.yml +- include: start_vault_temp.yml when: >- inventory_hostname == groups.vault|first and - hostvars[groups.vault|first]["vault_temp_start"]|succeeded + not vault_cluster_is_initialized +# NOTE: The next 2 steps run against temp Vault and long-term Vault + +# Ensure PKI mount exists +- include: ../shared/pki_mount.yml + when: >- + inventory_hostname == groups.vault|first + +# If the Root CA already exists, ensure Vault's PKI is using it +- include: ../shared/config_ca.yml + vars: + ca_name: ca + mount_name: pki + when: >- + inventory_hostname == groups.vault|first and + not vault_ca_cert_needed + +# Generate root CA certs for Vault if none exist +- include: gen_ca.yml + when: >- + inventory_hostname in groups.vault and + not vault_cluster_is_initialized and + vault_ca_cert_needed + +# Generate Vault API certs +- include: gen_vault_certs.yml + when: inventory_hostname in groups.vault and vault_api_cert_needed + +# Update all host's CA bundle - include: ca_trust.yml + +## Add Etcd Role to Vault (if needed) + +- include: role_auth_cert.yml + when: vault_role_auth_method == "cert" +- include: role_auth_userpass.yml + when: vault_role_auth_method == "userpass" diff --git a/roles/vault/tasks/bootstrap/role_auth_cert.yml b/roles/vault/tasks/bootstrap/role_auth_cert.yml new file mode 100644 index 000000000..7bbf58e86 --- /dev/null +++ b/roles/vault/tasks/bootstrap/role_auth_cert.yml @@ -0,0 +1,25 @@ +--- + +- include: ../shared/sync_auth_certs.yml + when: inventory_hostname in groups.vault + +- include: ../shared/cert_auth_mount.yml + when: inventory_hostname == groups.vault|first + +- include: ../shared/auth_backend.yml + vars: + auth_backend_description: A Cert-based Auth primarily for services needing to issue certificates + auth_backend_name: cert + auth_backend_type: cert + when: inventory_hostname == groups.vault|first + +- include: gen_auth_ca.yml + when: inventory_hostname in groups.vault and vault_auth_ca_cert_needed + +- include: ../shared/config_ca.yml + vars: + ca_name: auth-ca + mount_name: auth-pki + when: inventory_hostname == groups.vault|first and not vault_auth_ca_cert_needed +- include: create_etcd_role.yml + when: inventory_hostname in groups.etcd diff --git a/roles/vault/tasks/bootstrap/role_auth_userpass.yml b/roles/vault/tasks/bootstrap/role_auth_userpass.yml new file mode 100644 index 000000000..ad09ab05b --- /dev/null +++ b/roles/vault/tasks/bootstrap/role_auth_userpass.yml @@ -0,0 +1,10 @@ +--- + +- include: ../shared/auth_backend.yml + vars: + auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates + auth_backend_path: userpass + auth_backend_type: userpass + when: inventory_hostname == groups.vault|first +- include: create_etcd_role.yml + when: inventory_hostname in groups.etcd diff --git a/roles/vault/tasks/bootstrap/start_vault_temp.yml b/roles/vault/tasks/bootstrap/start_vault_temp.yml index 9fbc9719e..eeaaad535 100644 --- a/roles/vault/tasks/bootstrap/start_vault_temp.yml +++ b/roles/vault/tasks/bootstrap/start_vault_temp.yml @@ -1,21 +1,21 @@ --- -- name: boostrap/start_vault_temp | Ensure vault-temp isn't already running - shell: if docker rm -f vault-temp 2>&1 1>/dev/null;then echo true;else echo false;fi +- name: bootstrap/start_vault_temp | Ensure vault-temp isn't already running + shell: if docker rm -f {{ vault_temp_container_name }} 2>&1 1>/dev/null;then echo true;else echo false;fi register: vault_temp_stop_check changed_when: "{{ 'true' in vault_temp_stop_check.stdout }}" - name: bootstrap/start_vault_temp | Start single node Vault with file backend command: > - docker run -d --cap-add=IPC_LOCK --name vault-temp -p {{ vault_temp_port }}:{{ vault_temp_port }} + docker run -d --cap-add=IPC_LOCK --name {{ vault_temp_container_name }} + -p {{ vault_port }}:{{ vault_port }} -e 'VAULT_LOCAL_CONFIG={{ vault_temp_config|to_json }}' -v /etc/vault:/etc/vault {{ vault_image_repo }}:{{ vault_version }} server - register: vault_temp_start - name: bootstrap/start_vault_temp | Initialize vault-temp uri: - url: "http://localhost:{{ vault_temp_port }}/v1/sys/init" + url: "http://localhost:{{ vault_port }}/v1/sys/init" headers: "{{ vault_client_headers }}" method: PUT body_format: json @@ -24,32 +24,20 @@ secret_threshold: 1 register: vault_temp_init -# NOTE: vault_headers and vault_url are used by subsequent gen_cert calls +# NOTE: vault_headers and vault_url are used by subsequent issue calls - name: bootstrap/start_vault_temp | Set needed vault facts set_fact: + vault_leader_url: "http://{{ inventory_hostname }}:{{ vault_port }}" vault_temp_unseal_keys: "{{ vault_temp_init.json['keys'] }}" vault_temp_root_token: "{{ vault_temp_init.json.root_token }}" vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_temp_init.json.root_token}) }}" - name: bootstrap/start_vault_temp | Unseal vault-temp uri: - url: "http://localhost:{{ vault_temp_port }}/v1/sys/unseal" + url: "http://localhost:{{ vault_port }}/v1/sys/unseal" headers: "{{ vault_headers }}" method: POST body_format: json body: key: "{{ item }}" with_items: "{{ vault_temp_unseal_keys|default([]) }}" - -- name: bootstrap/start_vault_temp | Create new PKI mount - uri: - url: "http://localhost:{{ vault_temp_port }}/v1/sys/mounts/pki" - headers: "{{ vault_headers }}" - method: POST - body_format: json - body: - config: - default_lease_ttl: "{{ vault_default_lease_ttl }}" - max_lease_ttl: "{{ vault_max_lease_ttl }}" - type: pki - status_code: 204 diff --git a/roles/vault/tasks/bootstrap/stop_vault_temp.yml b/roles/vault/tasks/bootstrap/stop_vault_temp.yml deleted file mode 100644 index 1699eebc7..000000000 --- a/roles/vault/tasks/bootstrap/stop_vault_temp.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -- name: stop vault-temp container - command: docker stop vault-temp diff --git a/roles/vault/tasks/bootstrap/sync_etcd_certs.yml b/roles/vault/tasks/bootstrap/sync_etcd_certs.yml deleted file mode 100644 index d6ae8e4cc..000000000 --- a/roles/vault/tasks/bootstrap/sync_etcd_certs.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- - -- name: bootstrap/sync_etcd_certs | Create list of certs needing creation - set_fact: - vault_etcd_cert_list: >- - {{ vault_etcd_cert_list|default([]) + [ - "admin-" + item + ".pem", - "member-" + item + ".pem" - ] }} - with_items: "{{ groups.etcd }}" - -- include: ../sync_file.yml - vars: - sync_file: "{{ item }}" - sync_file_dir: "{{ etcd_cert_dir }}" - sync_file_hosts: "{{ groups.etcd }}" - sync_file_is_cert: true - with_items: "{{ vault_etcd_cert_list|default([]) }}" - -- name: bootstrap/sync_etcd_certs | Set facts for etcd sync_file results - set_fact: - vault_etcd_certs_needed: "{{ vault_etcd_certs_needed|default([]) + [item.path] }}" - with_items: "{{ sync_file_results }}" - when: item.no_srcs|bool - -- name: bootstrap/sync_etcd_certs | Unset sync_file_results after etcd certs sync - set_fact: - sync_file_results: [] - -- include: ../sync_file.yml - vars: - sync_file: ca.pem - sync_file_dir: "{{ etcd_cert_dir }}" - sync_file_hosts: "{{ groups.etcd }}" - -- name: bootstrap/sync_etcd_certs | Unset sync_file_results after ca.pem sync - set_fact: - sync_file_results: [] diff --git a/roles/vault/tasks/bootstrap/sync_etcd_node_certs.yml b/roles/vault/tasks/bootstrap/sync_etcd_node_certs.yml deleted file mode 100644 index 8a50a5208..000000000 --- a/roles/vault/tasks/bootstrap/sync_etcd_node_certs.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- - -- name: bootstrap/sync_etcd_node_certs | Create list of certs needing creation - set_fact: - vault_etcd_node_cert_list: "{{ vault_etcd_node_cert_list|default([]) + ['node-' + item + '.pem'] }}" - with_items: "{{ groups['k8s-cluster'] | union(groups.etcd) }}" - -- include: ../sync_file.yml - vars: - sync_file: "{{ item }}" - sync_file_dir: "{{ etcd_cert_dir }}" - sync_file_hosts: "{{ groups['k8s-cluster'] | union(groups.etcd) }}" - sync_file_is_cert: true - with_items: "{{ vault_etcd_node_cert_list|default([]) }}" - -- name: bootstrap/sync_etcd_node_certs | Set facts for etcd sync_file results - set_fact: - vault_etcd_node_certs_needed: "{{ vault_etcd_node_certs_needed|default([]) + [item.path] }}" - with_items: "{{ sync_file_results }}" - when: item.no_srcs|bool - -- name: bootstrap/sync_etcd_node_certs | Unset sync_file_results after etcd node certs - set_fact: - sync_file_results: [] - -- include: ../sync_file.yml - vars: - sync_file: ca.pem - sync_file_dir: "{{ etcd_cert_dir }}" - sync_file_hosts: "{{ groups['k8s-cluster']| union(groups.etcd) }}" - -- name: bootstrap/sync_etcd_node_certs | Unset sync_file_results after ca.pem - set_fact: - sync_file_results: [] diff --git a/roles/vault/tasks/bootstrap/sync_secrets.yml b/roles/vault/tasks/bootstrap/sync_secrets.yml new file mode 100644 index 000000000..3b5af775a --- /dev/null +++ b/roles/vault/tasks/bootstrap/sync_secrets.yml @@ -0,0 +1,48 @@ +--- + +- include: ../shared/sync_file.yml + vars: + sync_file: "{{ item }}" + sync_file_dir: "{{ vault_secrets_dir }}" + sync_file_hosts: "{{ groups.vault }}" + with_items: + - root_token + - unseal_keys + +- name: bootstrap/sync_secrets | Set fact based on sync_file_results + set_fact: + vault_secrets_available: "{{ vault_secrets_available|default(true) and not item.no_srcs }}" + with_items: "{{ sync_file_results|d([]) }}" + +- name: bootstrap/sync_secrets | Reset sync_file_results to avoid variable bleed + set_fact: + sync_file_results: [] + +- name: bootstrap/sync_secrets | Print out warning message if secrets are not available and vault is initialized + pause: + prompt: > + Vault orchestration may not be able to proceed. The Vault cluster is initialzed, but + 'root_token' or 'unseal_keys' were not found in {{ vault_secrets_dir }}. These are + needed for many vault orchestration steps. + when: vault_cluster_is_initialized and not vault_secrets_available + +- name: bootstrap/sync_secrets | Cat root_token from a vault host + command: "cat {{ vault_secrets_dir }}/root_token" + register: vault_root_token_cat + when: vault_secrets_available and inventory_hostname == groups.vault|first + +- name: bootstrap/sync_secrets | Cat unseal_keys from a vault host + command: "cat {{ vault_secrets_dir }}/unseal_keys" + register: vault_unseal_keys_cat + when: vault_secrets_available and inventory_hostname == groups.vault|first + +- name: bootstrap/sync_secrets | Set needed facts for Vault API interaction when Vault is already running + set_fact: + vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token_cat']['stdout'] }}" + vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys_cat']['stdout_lines'] }}" + when: vault_secrets_available + +- name: bootstrap/sync_secrets | Update vault_headers if we have the root_token + set_fact: + vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token}) }}" + when: vault_secrets_available diff --git a/roles/vault/tasks/bootstrap/sync_vault_certs.yml b/roles/vault/tasks/bootstrap/sync_vault_certs.yml index ad2c49853..ab088753f 100644 --- a/roles/vault/tasks/bootstrap/sync_vault_certs.yml +++ b/roles/vault/tasks/bootstrap/sync_vault_certs.yml @@ -1,21 +1,21 @@ --- -- include: ../sync_file.yml +- include: ../shared/sync_file.yml vars: sync_file: "ca.pem" sync_file_dir: "{{ vault_cert_dir }}" sync_file_hosts: "{{ groups.vault }}" sync_file_is_cert: true -- name: "bootstrap/sync_vault_certs | Set facts for vault sync_file results" +- name: bootstrap/sync_vault_certs | Set facts for vault sync_file results set_fact: - vault_ca_cert_needed: "{{ true if sync_file_results|length > 0 else false }}" + vault_ca_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}" - name: bootstrap/sync_vault_certs | Unset sync_file_results after ca.pem sync set_fact: sync_file_results: [] -- include: ../sync_file.yml +- include: ../shared/sync_file.yml vars: sync_file: "api.pem" sync_file_dir: "{{ vault_cert_dir }}" @@ -24,7 +24,7 @@ - name: bootstrap/sync_vault_certs | Set fact if Vault's API cert is needed set_fact: - vault_api_cert_needed: "{{ true if sync_file_results|length > 0 else false }}" + vault_api_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}" - name: bootstrap/sync_vault_certs | Unset sync_file_results after api.pem sync set_fact: diff --git a/roles/vault/tasks/check_vault.yml b/roles/vault/tasks/check_vault.yml deleted file mode 100644 index 575d1207f..000000000 --- a/roles/vault/tasks/check_vault.yml +++ /dev/null @@ -1,77 +0,0 @@ ---- - -# Check if vault is reachable on the localhost -- name: check_vault | Attempt to pull local vault health - uri: - url: "https://localhost:{{ vault_port }}/v1/sys/health" - headers: "{{ vault_client_headers }}" - validate_certs: no - ignore_errors: true - register: vault_local_service_health - -- name: check_vault | Set facts about local Vault health - set_fact: - vault_is_running: "{{ vault_local_service_health|succeeded }}" - vault_is_initialized: "{{ vault_local_service_health.get('json', {}).get('initialized', false) }}" - vault_is_sealed: "{{ vault_local_service_health.get('json', {}).get('sealed', true) }}" - vault_in_standby: "{{ vault_local_service_health.get('json', {}).get('standby', true) }}" - vault_run_version: "{{ vault_local_service_health.get('json', {}).get('version', '') }}" - -- name: check_vault | Set fact about the Vault cluster's initialization state - set_fact: - vault_cluster_is_initialized: "{{ vault_is_initialized or hostvars[item]['vault_is_initialized'] }}" - with_items: "{{ groups.vault }}" - -- name: check_vault | Set fact about the Vault Cluster's available hosts - set_fact: - vault_available_hosts: "{{ vault_available_hosts|default([]) + [item] }}" - with_items: "{{ groups.vault }}" - when: "hostvars[item]['vault_is_running'] and not hostvars[item]['vault_is_sealed']" - -- include: sync_file.yml - vars: - sync_file: "{{ item }}" - sync_file_dir: "{{ vault_secrets_dir }}" - sync_file_hosts: "{{ groups.vault }}" - with_items: - - root_token - - unseal_keys - -# Logic is hard to follow on this one, probably need to simplify somehow -- name: "check_vault | Set fact based on sync_file_results" - set_fact: - vault_secrets_available: "{{ vault_secrets_available|default(true) and not item.no_srcs }}" - with_items: "{{ sync_file_results }}" - -- name: "check_vault | Reset sync_file_results to avoid variable bleed" - set_fact: - sync_file_results: [] - -- name: "check_vault | Print out warning message if secrets are not available" - pause: - prompt: > - Vault orchestration may not be able to proceed. The Vault cluster is initialzed, but - 'root_token' or 'unseal_keys' were not found in {{ vault_secrets_dir }}. These are - needed for many orchestration steps. - when: vault_cluster_is_initialized and not vault_secrets_available - -- name: "check_vault | Cat root_token from a vault host" - command: "cat {{ vault_secrets_dir }}/root_token" - register: vault_root_token_cat - when: vault_secrets_available and inventory_hostname == groups.vault|first - -- name: "check_vault | Cat unseal_keys from a vault host" - command: "cat {{ vault_secrets_dir }}/unseal_keys" - register: vault_unseal_keys_cat - when: vault_secrets_available and inventory_hostname == groups.vault|first - -- name: "check_vault | Set needed facts for Vault API interaction when Vault is already running" - set_fact: - vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token_cat']['stdout'] }}" - vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys_cat']['stdout_lines'] }}" - when: vault_secrets_available - -- name: "check-vault | Update vault_headers if we have the root_token" - set_fact: - vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token}) }}" - when: vault_secrets_available diff --git a/roles/vault/tasks/cluster/binary.yml b/roles/vault/tasks/cluster/binary.yml new file mode 100644 index 000000000..41024dd54 --- /dev/null +++ b/roles/vault/tasks/cluster/binary.yml @@ -0,0 +1,9 @@ +--- + +- name: cluster/binary | Copy vault binary from downloaddir + copy: + src: "{{ local_release_dir }}/vault/vault" + dest: "/usr/bin/vault" + remote_src: true + mode: "0755" + owner: vault diff --git a/roles/vault/tasks/cluster/configure.yml b/roles/vault/tasks/cluster/configure.yml new file mode 100644 index 000000000..7ac8f5f9e --- /dev/null +++ b/roles/vault/tasks/cluster/configure.yml @@ -0,0 +1,14 @@ +--- + +- name: cluster/configure | Ensure the vault/config directory exists + file: + dest: "{{ vault_config_dir }}" + mode: 0750 + state: directory + +- name: cluster/configure | Lay down the configuration file + copy: + content: "{{ vault_config | to_nice_json(indent=4) }}" + dest: "{{ vault_config_dir }}/config.json" + mode: 0640 + register: vault_config_change diff --git a/roles/vault/tasks/cluster/create_roles.yml b/roles/vault/tasks/cluster/create_roles.yml new file mode 100644 index 000000000..a135137da --- /dev/null +++ b/roles/vault/tasks/cluster/create_roles.yml @@ -0,0 +1,9 @@ +--- + +- include: ../shared/create_role.yml + vars: + create_role_name: "{{ item.name }}" + create_role_group: "{{ item.group }}" + create_role_policy_rules: "{{ item.policy_rules }}" + create_role_options: "{{ item.role_options }}" + with_items: "{{ vault_roles|d([]) }}" diff --git a/roles/vault/tasks/cluster/docker.yml b/roles/vault/tasks/cluster/docker.yml deleted file mode 100644 index ba7d3a3aa..000000000 --- a/roles/vault/tasks/cluster/docker.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- - -- name: docker | Check on state of docker instance - command: "docker inspect {{ vault_container_name }}" - ignore_errors: true - register: vault_container_inspect - -- name: docker | Set fact on container status - set_fact: - vault_container_inspect_json: "{{ vault_container_inspect.stdout|from_json }}" - when: vault_container_inspect|succeeded - -# Not sure if State.Running is the best check here... -- name: docker | Remove old container if it's not currently running - command: "docker rm {{ vault_container_name }}" - when: vault_container_inspect|succeeded and not vault_container_inspect_json[0]["State"]["Running"]|bool - -- name: docker | Start a new Vault instance - command: > - docker run -d --cap-add=IPC_LOCK --name {{vault_container_name}} -p {{vault_port}}:{{vault_port}} - -e 'VAULT_LOCAL_CONFIG={{ vault_config|to_json }}' - -v /etc/vault:/etc/vault - {{vault_image_repo}}:{{vault_version}} server - register: vault_docker_start - when: vault_container_inspect|failed or not vault_container_inspect_json[0]["State"]["Running"]|bool diff --git a/roles/vault/tasks/cluster/gen_kube_master_certs.yml b/roles/vault/tasks/cluster/gen_kube_master_certs.yml deleted file mode 100644 index bf886af66..000000000 --- a/roles/vault/tasks/cluster/gen_kube_master_certs.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- - -- name: "cluster/gen_kube_node_certs | Ensure kube_cert_dir exists" - file: - path: "{{ kube_cert_dir }}" - state: directory - -- name: gen_kube_master_certs | Add the kube role - uri: - url: "https://{{ hostvars[groups.vault|first]['vault_leader'] }}:{{ vault_port }}/v1/pki/roles/kubernetes" - headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" - method: POST - body_format: json - body: "{{ vault_default_role_permissions }}" - status_code: 204 - when: inventory_hostname == groups["kube-master"]|first - -- include: ../gen_cert.yml - vars: - gen_cert_alt_names: "{{ groups['kube-master'] | join(',') }},localhost" - gen_cert_copy_ca: "{{ true if item == vault_kube_master_certs_needed|first else false }}" - gen_cert_hosts: "{{ groups['kube-master'] }}" - gen_cert_ip_sans: >- - {%- for host in groups["kube-master"] -%} - {{ hostvars[host]["ansible_default_ipv4"]["address"] }} - {%- if not loop.last -%},{%- endif -%} - {%- endfor -%} - ,127.0.0.1,::1 - gen_cert_path: "{{ item }}" - gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" - gen_cert_vault_role: kubernetes - gen_cert_vault_url: "https://{{ hostvars[groups.vault|first]['vault_leader'] }}:{{ vault_port }}" - with_items: "{{ vault_kube_master_certs_needed|default([]) }}" diff --git a/roles/vault/tasks/cluster/gen_kube_node_certs.yml b/roles/vault/tasks/cluster/gen_kube_node_certs.yml deleted file mode 100644 index ea48d4c43..000000000 --- a/roles/vault/tasks/cluster/gen_kube_node_certs.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- - -- name: "cluster/gen_kube_node_certs | Ensure kube_cert_dir exists" - file: - path: "{{ kube_cert_dir }}" - state: directory - -- name: "cluster/gen_kube_node_certs | Add the kubernetes role" - uri: - url: "https://{{ hostvars[groups.vault|first]['vault_leader'] }}:{{ vault_port }}/v1/pki/roles/kubernetes" - headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" - method: POST - body_format: json - body: "{{ vault_default_role_permissions }}" - status_code: 204 - when: inventory_hostname == groups["k8s-cluster"]|first - -- include: ../gen_cert.yml - vars: - gen_cert_alt_names: "{{ groups['k8s-cluster'] | join(',') }},localhost" - gen_cert_copy_ca: "{{ true if item == vault_kube_node_certs_needed|first else false }}" - gen_cert_hosts: "{{ groups['k8s-cluster'] }}" - gen_cert_ip_sans: >- - {%- for host in groups["k8s-cluster"] -%} - {{ hostvars[host]["ansible_default_ipv4"]["address"] }} - {%- if not loop.last -%},{%- endif -%} - {%- endfor -%} - ,127.0.0.1,::1 - gen_cert_path: "{{ item }}" - gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" - gen_cert_vault_role: kubernetes - gen_cert_vault_url: "https://{{ hostvars[groups.vault|first]['vault_leader'] }}:{{ vault_port }}" - with_items: "{{ vault_kube_node_certs_needed|default([]) }}" diff --git a/roles/vault/tasks/cluster/init.yml b/roles/vault/tasks/cluster/init.yml index 9f124869a..60aaf9591 100644 --- a/roles/vault/tasks/cluster/init.yml +++ b/roles/vault/tasks/cluster/init.yml @@ -28,6 +28,7 @@ - name: cluster/init | Ensure the vault_secrets_dir exists file: + mode: 0750 path: "{{ vault_secrets_dir }}" state: directory @@ -35,12 +36,14 @@ copy: content: "{{ vault_unseal_keys|join('\n') }}" dest: "{{ vault_secrets_dir }}/unseal_keys" + mode: 0640 when: not vault_cluster_is_initialized - name: cluster/init | Ensure all in groups.vault have the root_token locally copy: content: "{{ vault_root_token }}" dest: "{{ vault_secrets_dir }}/root_token" + mode: 0640 when: not vault_cluster_is_initialized - name: cluster/init | Ensure vault_headers and vault statuses are updated diff --git a/roles/vault/tasks/cluster/main.yml b/roles/vault/tasks/cluster/main.yml index 14c3adaaf..5dab550aa 100644 --- a/roles/vault/tasks/cluster/main.yml +++ b/roles/vault/tasks/cluster/main.yml @@ -1,30 +1,35 @@ --- +- include: ../shared/check_vault.yml + when: inventory_hostname in groups.vault +- include: ../shared/check_etcd.yml + when: inventory_hostname in groups.vault + ## Vault Cluster Setup -- include: docker.yml - when: inventory_hostname in groups.vault and vault_deployment_type == "docker" +- include: configure.yml + when: inventory_hostname in groups.vault +- include: binary.yml + when: inventory_hostname in groups.vault and vault_deployment_type == "host" +- include: systemd.yml + when: inventory_hostname in groups.vault - include: init.yml when: inventory_hostname in groups.vault - include: unseal.yml when: inventory_hostname in groups.vault -- include: pki_mount.yml - when: 'inventory_hostname == hostvars[groups.vault|first]["vault_leader"]' -- include: config_ca.yml +- include: ../shared/find_leader.yml + when: inventory_hostname in groups.vault +- include: ../shared/pki_mount.yml + when: inventory_hostname == groups.vault|first +- include: ../shared/config_ca.yml vars: - vault_url: "https://{{ vault_leader }}:{{ vault_port }}" - when: 'inventory_hostname == hostvars[groups.vault|first]["vault_leader"]' + ca_name: ca + mount_name: pki + when: inventory_hostname == groups.vault|first -## Sync Kubernetes Certs +## Vault Policies, Roles, and Auth Backends -- include: sync_kube_master_certs.yml - when: inventory_hostname in groups["kube-master"] -- include: sync_kube_node_certs.yml - when: inventory_hostname in groups["k8s-cluster"] - -## Generate Kubernetes Certs - -- include: gen_kube_master_certs.yml - when: inventory_hostname in groups["kube-master"] -- include: gen_kube_node_certs.yml - when: inventory_hostname in groups["k8s-cluster"] +- include: role_auth_cert.yml + when: vault_role_auth_method == "cert" +- include: role_auth_userpass.yml + when: vault_role_auth_method == "userpass" diff --git a/roles/vault/tasks/cluster/pki_mount.yml b/roles/vault/tasks/cluster/pki_mount.yml deleted file mode 100644 index 266c5f666..000000000 --- a/roles/vault/tasks/cluster/pki_mount.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- - -- name: cluster/pki_mount | Test if default PKI mount exists - uri: - url: "https://localhost:{{ vault_port }}/v1/sys/mounts/pki/tune" - headers: "{{ vault_headers }}" - validate_certs: false - ignore_errors: true - register: vault_pki_mount_check - -- name: cluster/pki_mount | Mount default PKI mount if needed - uri: - url: "https://localhost:{{ vault_port }}/v1/sys/mounts/pki" - headers: "{{ vault_headers }}" - method: POST - body_format: json - body: - config: - default_lease_ttl: "{{ vault_default_lease_ttl }}" - max_lease_ttl: "{{ vault_max_lease_ttl }}" - type: pki - status_code: 204 - when: vault_pki_mount_check | failed diff --git a/roles/vault/tasks/cluster/role_auth_cert.yml b/roles/vault/tasks/cluster/role_auth_cert.yml new file mode 100644 index 000000000..9f186e3ff --- /dev/null +++ b/roles/vault/tasks/cluster/role_auth_cert.yml @@ -0,0 +1,19 @@ +--- + +- include: ../shared/cert_auth_mount.yml + when: inventory_hostname == groups.vault|first + +- include: ../shared/auth_backend.yml + vars: + auth_backend_description: A Cert-based Auth primarily for services needing to issue certificates + auth_backend_name: cert + auth_backend_type: cert + when: inventory_hostname == groups.vault|first + +- include: ../shared/config_ca.yml + vars: + ca_name: auth-ca + mount_name: auth-pki + when: inventory_hostname == groups.vault|first + +- include: create_roles.yml diff --git a/roles/vault/tasks/cluster/role_auth_userpass.yml b/roles/vault/tasks/cluster/role_auth_userpass.yml new file mode 100644 index 000000000..ac3b2c6c1 --- /dev/null +++ b/roles/vault/tasks/cluster/role_auth_userpass.yml @@ -0,0 +1,10 @@ +--- + +- include: ../shared/auth_backend.yml + vars: + auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates + auth_backend_path: userpass + auth_backend_type: userpass + when: inventory_hostname == groups.vault|first + +- include: create_roles.yml diff --git a/roles/vault/tasks/cluster/sync_kube_master_certs.yml b/roles/vault/tasks/cluster/sync_kube_master_certs.yml deleted file mode 100644 index db201a911..000000000 --- a/roles/vault/tasks/cluster/sync_kube_master_certs.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- - -- name: cluster/sync_kube_master_certs | Create list of needed certs - set_fact: - vault_kube_master_cert_list: >- - {{ vault_kube_master_cert_list|default([]) + [ - "admin-" + item + ".pem", - "apiserver-" + item + ".pem" - ] }} - with_items: "{{ groups['kube-master'] }}" - -- include: ../sync_file.yml - vars: - sync_file: "{{ item }}" - sync_file_dir: "{{ kube_cert_dir }}" - sync_file_hosts: "{{ groups['kube-master'] }}" - sync_file_is_cert: true - with_items: "{{ vault_kube_master_cert_list|default([]) }}" - -- name: cluster/sync_kube_master_certs | Set facts for kube-master sync_file results - set_fact: - vault_kube_master_certs_needed: "{{ vault_kube_master_certs_needed|default([]) + [item.path] }}" - with_items: "{{ sync_file_results }}" - when: item.no_srcs|bool - -- name: cluster/sync_kube_master_certs | Unset sync_file_results after kube master certs - set_fact: - sync_file_results: [] - -- include: ../sync_file.yml - vars: - sync_file: ca.pem - sync_file_dir: "{{ kube_cert_dir }}" - sync_file_hosts: "{{ groups['kube-master'] }}" - -- name: cluster/sync_kube_master_certs | Unset sync_file_results after ca.pem - set_fact: - sync_file_results: [] diff --git a/roles/vault/tasks/cluster/sync_kube_node_certs.yml b/roles/vault/tasks/cluster/sync_kube_node_certs.yml deleted file mode 100644 index 9a73b7870..000000000 --- a/roles/vault/tasks/cluster/sync_kube_node_certs.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- - -- name: cluster/sync_kube_node_certs | Create list of needed certs - set_fact: - vault_kube_node_cert_list: "{{ vault_kube_node_cert_list|default([]) + ['node-' + item + '.pem'] }}" - with_items: "{{ groups['k8s-cluster'] }}" - -- include: ../sync_file.yml - vars: - sync_file: "{{ item }}" - sync_file_dir: "{{ kube_cert_dir }}" - sync_file_hosts: "{{ groups['k8s-cluster'] }}" - sync_file_is_cert: true - with_items: "{{ vault_kube_node_cert_list|default([]) }}" - -- name: cluster/sync_kube_node_certs | Set facts for kube-master sync_file results - set_fact: - vault_kube_node_certs_needed: "{{ vault_kube_node_certs_needed|default([]) + [item.path] }}" - with_items: "{{ sync_file_results }}" - when: item.no_srcs|bool - -- name: cluster/sync_kube_node_certs | Unset sync_file_results after kube node certs - set_fact: - sync_file_results: [] - -- include: ../sync_file.yml - vars: - sync_file: ca.pem - sync_file_dir: "{{ kube_cert_dir }}" - sync_file_hosts: "{{ groups['k8s-cluster'] }}" - -- name: cluster/sync_kube_node_certs | Unset sync_file_results after ca.pem - set_fact: - sync_file_results: [] diff --git a/roles/vault/tasks/cluster/systemd.yml b/roles/vault/tasks/cluster/systemd.yml new file mode 100644 index 000000000..4d2a1da58 --- /dev/null +++ b/roles/vault/tasks/cluster/systemd.yml @@ -0,0 +1,45 @@ +--- + +- name: cluster/systemd | Ensure mount points exist prior to vault.service startup + file: + mode: 0750 + path: "{{ item }}" + state: directory + with_items: + - "{{ vault_config_dir }}" + - "{{ vault_log_dir }}" + - "{{ vault_secrets_dir }}" + - /var/lib/vault/ + +- name: cluster/systemd | Ensure the vault user has access to needed directories + file: + owner: vault + path: "{{ item }}" + recurse: true + with_items: + - "{{ vault_base_dir }}" + - "{{ vault_log_dir }}" + - /var/lib/vault + +- name: cluster/systemd | Copy down vault.service systemd file + template: + src: "{{ vault_deployment_type }}.service.j2" + dest: /etc/systemd/system/vault.service + backup: yes + register: vault_systemd_placement + +- name: cluster/systemd | Enable vault.service + systemd: + daemon_reload: true + enabled: yes + name: vault + state: started + +- name: cluster/systemd | Query local vault until service is up + uri: + url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health" + headers: "{{ vault_client_headers }}" + status_code: 200,429,500,501 + register: vault_health_check + until: vault_health_check|succeeded + retries: 10 diff --git a/roles/vault/tasks/cluster/unseal.yml b/roles/vault/tasks/cluster/unseal.yml index ea74694ea..2fbdbce56 100644 --- a/roles/vault/tasks/cluster/unseal.yml +++ b/roles/vault/tasks/cluster/unseal.yml @@ -11,16 +11,12 @@ with_items: "{{ vault_unseal_keys|default([]) }}" when: vault_is_sealed -- name: cluster/unseal | Find the current leader +- name: cluster/unseal | Wait until server is ready uri: url: "https://localhost:{{ vault_port }}/v1/sys/health" headers: "{{ vault_headers }}" method: HEAD - status_code: 200,429 - register: vault_leader_check - -- name: cluster/unseal | Set fact for current leader - set_fact: - vault_leader: "{{ item }}" - with_items: "{{ groups.vault }}" - when: 'hostvars[item]["vault_leader_check"]["status"] == 200' + status_code: 200, 429 + register: vault_node_ready + until: vault_node_ready|succeeded + retries: 5 diff --git a/roles/vault/tasks/config_ca.yml b/roles/vault/tasks/config_ca.yml deleted file mode 100644 index ef3c08cf1..000000000 --- a/roles/vault/tasks/config_ca.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- - -- name: config_ca | Read root CA cert for Vault - command: cat /etc/vault/ssl/ca.pem - register: vault_ca_cert_cat - -- name: config_ca | Read root CA key for Vault - command: cat /etc/vault/ssl/ca-key.pem - register: vault_ca_key_cat - -- name: config_ca | Configure pki mount to use the found root CA cert and key - uri: - url: "{{ vault_url }}/v1/pki/config/ca" - headers: "{{ vault_headers }}" - method: POST - body_format: json - body: - pem_bundle: "{{ vault_ca_cert_cat.stdout + '\n' + vault_ca_key_cat.stdout }}" - status_code: 204 diff --git a/roles/vault/tasks/gen_cert.yml b/roles/vault/tasks/gen_cert.yml deleted file mode 100644 index 9dc5d30f4..000000000 --- a/roles/vault/tasks/gen_cert.yml +++ /dev/null @@ -1,50 +0,0 @@ ---- - -# This could be a role or custom module - -- name: gen_cert | Ensure target directory exists - file: - path: "{{ gen_cert_path | dirname }}" - state: directory - -- name: gen_cert | Generate the cert - uri: - url: "{{ gen_cert_vault_url}}/v1/pki/issue/{{ gen_cert_vault_role }}" - headers: "{{ gen_cert_vault_headers }}" - method: POST - body_format: json - body: - alt_names: "{{ gen_cert_alt_names|default([]) }}" - common_name: "{{ gen_cert_path.rsplit('/', 1)[1].rsplit('.', 1)[0] }}" - format: "{{ gen_cert_format|default('pem') }}" - ip_sans: "{{ gen_cert_ip_sans|default([]) }}" - register: gen_cert_result - when: inventory_hostname == gen_cert_hosts|first - -- name: gen_cert | Copy the cert to all hosts - copy: - content: "{{ hostvars[gen_cert_hosts|first]['gen_cert_result']['json']['data']['certificate'] }}" - dest: "{{ gen_cert_path }}" - -- name: gen_cert | Copy the key to all hosts - copy: - content: "{{ hostvars[gen_cert_hosts|first]['gen_cert_result']['json']['data']['private_key'] }}" - dest: "{{ gen_cert_path.rsplit('.', 1)|first + '-key.' + gen_cert_path.rsplit('.', 1)|last }}" - -- name: gen_cert | Copy issuing CA cert - copy: - content: "{{ hostvars[gen_cert_hosts|first]['gen_cert_result']['json']['data']['issuing_ca'] }}" - dest: "{{ gen_cert_path | dirname }}/ca.pem" - when: gen_cert_copy_ca|default(false)|bool - -- name: gen_cert | Unset common variables to avoid bleed over - set_fact: - gen_cert_copy_ca: false - gen_cert_alt_names: [] - gen_cert_format: pem - gen_cert_hosts: [] - gen_cert_ip_sans: [] - gen_cert_path: '' - gen_cert_vault_headers: '' - gen_cert_vault_role: '' - gen_cert_vault_url: '' diff --git a/roles/vault/tasks/main.yml b/roles/vault/tasks/main.yml index 59c8bb396..f7414b74f 100644 --- a/roles/vault/tasks/main.yml +++ b/roles/vault/tasks/main.yml @@ -1,13 +1,19 @@ --- +# The Vault role is typically a two step process: +# 1. Bootstrap +# This starts a temporary Vault to generate certs for Vault itself. This +# includes a Root CA for the cluster, assuming one doesn't exist already. +# The temporary instance will remain running after Bootstrap, to provide a +# running Vault for the Etcd role to generate certs against. +# 2. Cluster +# Once Etcd is started, then the Cluster tasks can start up a long-term +# Vault cluster using Etcd as the backend. The same Root CA is mounted as +# used during step 1, allowing all certs to have the same chain of trust. -- include: check_vault.yml - when: inventory_hostname in groups.vault - -# bootstrap.yml's sole purpose is to ensure certs exist for Vault and Etcd -# prior to startup, so TLS can be enabled. +## Bootstrap - include: bootstrap/main.yml - when: vault_bootstrap|bool + when: vault_bootstrap | d() -# cluster.yml should only run after the backend service is ready (default etcd) +## Cluster - include: cluster/main.yml - when: not vault_bootstrap|bool + when: not vault_bootstrap | d() diff --git a/roles/vault/tasks/shared/auth_backend.yml b/roles/vault/tasks/shared/auth_backend.yml new file mode 100644 index 000000000..ad5b191c9 --- /dev/null +++ b/roles/vault/tasks/shared/auth_backend.yml @@ -0,0 +1,21 @@ +--- + +- name: shared/auth_backend | Test if the auth backend exists + uri: + url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}/tune" + headers: "{{ vault_headers }}" + validate_certs: false + ignore_errors: true + register: vault_auth_backend_check + +- name: shared/auth_backend | Add the cert auth backend if needed + uri: + url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}" + headers: "{{ vault_headers }}" + method: POST + body_format: json + body: + description: "{{ auth_backend_description|d('') }}" + type: "{{ auth_backend_type }}" + status_code: 204 + when: vault_auth_backend_check|failed diff --git a/roles/vault/tasks/shared/cert_auth_mount.yml b/roles/vault/tasks/shared/cert_auth_mount.yml new file mode 100644 index 000000000..9710aa7ca --- /dev/null +++ b/roles/vault/tasks/shared/cert_auth_mount.yml @@ -0,0 +1,21 @@ +--- + +- include: ../shared/mount.yml + vars: + mount_name: auth-pki + mount_options: + description: PKI mount to generate certs for the Cert Auth Backend + config: + default_lease_ttl: "{{ vault_default_lease_ttl }}" + max_lease_ttl: "{{ vault_max_lease_ttl }}" + type: pki + +- name: shared/auth_mount | Create a dummy role for issuing certs from auth-pki + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth-pki/roles/dummy" + headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + method: POST + body_format: json + body: + {'allow_any_name': true} + status_code: 204 diff --git a/roles/vault/tasks/shared/check_etcd.yml b/roles/vault/tasks/shared/check_etcd.yml new file mode 100644 index 000000000..83c8b29e9 --- /dev/null +++ b/roles/vault/tasks/shared/check_etcd.yml @@ -0,0 +1,19 @@ +--- + +- name: check_etcd | Check if etcd is up an reachable + uri: + url: "{{ vault_etcd_url }}/health" + validate_certs: no + failed_when: false + register: vault_etcd_health_check + +- name: check_etcd | Set fact based off the etcd_health_check response + set_fact: + vault_etcd_available: "{{ vault_etcd_health_check.get('json', {}).get('health')|bool }}" + +- name: check_etcd | Fail if etcd is not available and needed + fail: + msg: > + Unable to start Vault cluster! Etcd is not available at + {{ vault_etcd_url }} however it is needed by Vault as a backend. + when: vault_etcd_needed|d() and not vault_etcd_available diff --git a/roles/vault/tasks/shared/check_vault.yml b/roles/vault/tasks/shared/check_vault.yml new file mode 100644 index 000000000..8a81ae5f2 --- /dev/null +++ b/roles/vault/tasks/shared/check_vault.yml @@ -0,0 +1,31 @@ +--- + +# Stop temporary Vault if it's running (can linger if playbook fails out) +- name: stop vault-temp container + shell: docker stop {{ vault_temp_container_name }} || rkt stop {{ vault_temp_container_name }} + failed_when: false + register: vault_temp_stop + changed_when: vault_temp_stop|succeeded + +# Check if vault is reachable on the localhost +- name: check_vault | Attempt to pull local https Vault health + uri: + url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health" + headers: "{{ vault_client_headers }}" + status_code: 200,429,500,501 + validate_certs: no + failed_when: false + register: vault_local_service_health + +- name: check_vault | Set facts about local Vault health + set_fact: + vault_is_running: "{{ vault_local_service_health|succeeded }}" + vault_is_initialized: "{{ vault_local_service_health.get('json', {}).get('initialized', false) }}" + vault_is_sealed: "{{ vault_local_service_health.get('json', {}).get('sealed', true) }}" + #vault_in_standby: "{{ vault_local_service_health.get('json', {}).get('standby', true) }}" + #vault_run_version: "{{ vault_local_service_health.get('json', {}).get('version', '') }}" + +- name: check_vault | Set fact about the Vault cluster's initialization state + set_fact: + vault_cluster_is_initialized: "{{ vault_is_initialized or hostvars[item]['vault_is_initialized'] }}" + with_items: "{{ groups.vault }}" diff --git a/roles/vault/tasks/shared/config_ca.yml b/roles/vault/tasks/shared/config_ca.yml new file mode 100644 index 000000000..79c972b4d --- /dev/null +++ b/roles/vault/tasks/shared/config_ca.yml @@ -0,0 +1,30 @@ +--- + +- name: config_ca | Read root CA cert for Vault + command: "cat /etc/vault/ssl/{{ ca_name }}.pem" + register: vault_ca_cert_cat + +- name: config_ca | Pull current CA cert from Vault + uri: + url: "{{ vault_leader_url }}/v1/{{ mount_name }}/ca/pem" + headers: "{{ vault_headers }}" + return_content: true + status_code: 200,204 + validate_certs: no + register: vault_pull_current_ca + +- name: config_ca | Read root CA key for Vault + command: "cat /etc/vault/ssl/{{ ca_name }}-key.pem" + register: vault_ca_key_cat + when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.content.strip() + +- name: config_ca | Configure pki mount to use the found root CA cert and key + uri: + url: "{{ vault_leader_url }}/v1/{{ mount_name }}/config/ca" + headers: "{{ vault_headers }}" + method: POST + body_format: json + body: + pem_bundle: "{{ vault_ca_cert_cat.stdout + '\n' + vault_ca_key_cat.stdout }}" + status_code: 204 + when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.get("content","").strip() diff --git a/roles/vault/tasks/shared/create_role.yml b/roles/vault/tasks/shared/create_role.yml new file mode 100644 index 000000000..c39fafe8c --- /dev/null +++ b/roles/vault/tasks/shared/create_role.yml @@ -0,0 +1,74 @@ +--- + +# The JSON inside JSON here is intentional (Vault API wants it) +- name: create_role | Create a policy for the new role allowing issuing + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/sys/policy/{{ create_role_name }}" + headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + method: PUT + body_format: json + body: + rules: >- + {%- if create_role_policy_rules|d("default") == "default" -%} + {{ + { 'path': { + 'pki/issue/' + create_role_name: {'policy': 'write'}, + 'pki/roles/' + create_role_name: {'policy': 'read'} + }} | to_json + '\n' + }} + {%- else -%} + {{ create_role_policy_rules | to_json + '\n' }} + {%- endif -%} + status_code: 204 + when: inventory_hostname == groups[create_role_group]|first + +- name: create_role | Create the new role in the pki mount + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/pki/roles/{{ create_role_name }}" + headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + method: POST + body_format: json + body: >- + {%- if create_role_options|d("default") == "default" -%} + {'allow_any_name': true} + {%- else -%} + {{ create_role_options }} + {%- endif -%} + status_code: 204 + when: inventory_hostname == groups[create_role_group]|first + +## Cert based auth method + +- include: gen_cert.yml + vars: + gen_cert_copy_ca: true + gen_cert_hosts: "{{ groups[create_role_group] }}" + gen_cert_mount: "auth-pki" + gen_cert_path: "{{ vault_roles_dir }}/{{ create_role_name }}/issuer.pem" + gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + gen_cert_vault_role: "dummy" + gen_cert_vault_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + when: vault_role_auth_method == "cert" and inventory_hostname in groups[create_role_group] + +- name: create_role | Insert the auth-pki CA as the authenticating CA for that role + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/cert/certs/{{ create_role_name }}" + headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + method: POST + body_format: json + body: + certificate: "{{ hostvars[groups[create_role_group]|first]['gen_cert_result']['json']['data']['issuing_ca'] }}" + policies: "{{ create_role_name }}" + status_code: 204 + when: vault_role_auth_method == "cert" and inventory_hostname == groups[create_role_group]|first + +## Userpass based auth method + +- include: gen_userpass.yml + vars: + gen_userpass_group: "{{ create_role_group }}" + gen_userpass_password: "{{ create_role_password|d(''|to_uuid) }}" + gen_userpass_policies: "{{ create_role_name }}" + gen_userpass_role: "{{ create_role_name }}" + gen_userpass_username: "{{ create_role_name }}" + when: vault_role_auth_method == "userpass" and inventory_hostname in groups[create_role_group] diff --git a/roles/vault/tasks/shared/find_leader.yml b/roles/vault/tasks/shared/find_leader.yml new file mode 100644 index 000000000..0c1a697d3 --- /dev/null +++ b/roles/vault/tasks/shared/find_leader.yml @@ -0,0 +1,17 @@ +--- + +- name: find_leader | Find the current http Vault leader + uri: + url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health" + headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + method: HEAD + status_code: 200,429 + register: vault_leader_check + until: "vault_leader_check|succeeded" + retries: 10 + +- name: find_leader | Set fact for current http leader + set_fact: + vault_leader_url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://{{ item }}:{{ vault_port }}" + with_items: "{{ groups.vault }}" + when: "hostvars[item]['vault_leader_check'].get('status') == 200" diff --git a/roles/vault/tasks/shared/gen_userpass.yml b/roles/vault/tasks/shared/gen_userpass.yml new file mode 100644 index 000000000..ab3d171b8 --- /dev/null +++ b/roles/vault/tasks/shared/gen_userpass.yml @@ -0,0 +1,30 @@ +--- + +- name: shared/gen_userpass | Create the Username/Password combo for the role + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/users/{{ gen_userpass_username }}" + headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + method: POST + body_format: json + body: + username: "{{ gen_userpass_username }}" + password: "{{ gen_userpass_password }}" + policies: "{{ gen_userpass_role }}" + status_code: 204 + when: inventory_hostname == groups[gen_userpass_group]|first + +- name: shared/gen_userpass | Ensure destination directory exists + file: + path: "{{ vault_roles_dir }}/{{ gen_userpass_role }}" + state: directory + when: inventory_hostname in groups[gen_userpass_group] + +- name: shared/gen_userpass | Copy credentials to all hosts in the group + copy: + content: > + {{ + {'username': gen_userpass_username, + 'password': gen_userpass_password} | to_nice_json(indent=4) + }} + dest: "{{ vault_roles_dir }}/{{ gen_userpass_role }}/userpass" + when: inventory_hostname in groups[gen_userpass_group] diff --git a/roles/vault/tasks/shared/issue_cert.yml b/roles/vault/tasks/shared/issue_cert.yml new file mode 100644 index 000000000..0733e86a0 --- /dev/null +++ b/roles/vault/tasks/shared/issue_cert.yml @@ -0,0 +1,66 @@ +--- + +# This could be a role or custom module + +# Vars: +# issue_cert_alt_name: Requested Subject Alternative Names, in a list. +# issue_cert_common_name: Common Name included in the cert +# issue_cert_dir_mode: Mode of the placed cert directory +# issue_cert_file_group: Group of the placed cert file and directory +# issue_cert_file_mode: Mode of the placed cert file +# issue_cert_file_owner: Owner of the placed cert file and directory +# issue_cert_format: Format for returned data. Can be pem, der, or pem_bundle +# issue_cert_headers: Headers passed into the issue request +# issue_cert_hosts: List of hosts to distribute the cert to +# issue_cert_ip_sans: Requested IP Subject Alternative Names, in a list +# issue_cert_mount: Mount point in Vault to make the request to +# issue_cert_path: Full path to the cert, include its name +# issue_cert_role: The Vault role to issue the cert with +# issue_cert_url: Url to reach Vault, including protocol and port + +- name: issue_cert | Ensure target directory exists + file: + path: "{{ issue_cert_path | dirname }}" + state: directory + group: "{{ issue_cert_file_group | d('root' )}}" + mode: "{{ issue_cert_dir_mode | d('0755') }}" + owner: "{{ issue_cert_file_owner | d('root') }}" + +- name: issue_cert | Generate the cert + uri: + url: "{{ issue_cert_url }}/v1/{{ issue_cert_mount|d('pki') }}/issue/{{ issue_cert_role }}" + headers: "{{ issue_cert_headers }}" + method: POST + body_format: json + body: + alt_names: "{{ issue_cert_alt_names | d([]) | join(',') }}" + common_name: "{{ issue_cert_common_name | d(issue_cert_path.rsplit('/', 1)[1].rsplit('.', 1)[0]) }}" + format: "{{ issue_cert_format | d('pem') }}" + ip_sans: "{{ issue_cert_ip_sans | default([]) | join(',') }}" + register: issue_cert_result + when: inventory_hostname == issue_cert_hosts|first + +- name: issue_cert | Copy the cert to all hosts + copy: + content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['certificate'] }}" + dest: "{{ issue_cert_path }}" + group: "{{ issue_cert_file_group | d('root' )}}" + mode: "{{ issue_cert_file_mode | d('0644') }}" + owner: "{{ issue_cert_file_owner | d('root') }}" + +- name: issue_cert | Copy the key to all hosts + copy: + content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['private_key'] }}" + dest: "{{ issue_cert_path.rsplit('.', 1)|first }}-key.{{ issue_cert_path.rsplit('.', 1)|last }}" + group: "{{ issue_cert_file_group | d('root' )}}" + mode: "{{ issue_cert_file_mode | d('0640') }}" + owner: "{{ issue_cert_file_owner | d('root') }}" + +- name: issue_cert | Copy issuing CA cert + copy: + content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['issuing_ca'] }}" + dest: "{{ issue_cert_path | dirname }}/ca.pem" + group: "{{ issue_cert_file_group | d('root' )}}" + mode: "{{ issue_cert_file_mode | d('0644') }}" + owner: "{{ issue_cert_file_owner | d('root') }}" + when: issue_cert_copy_ca|default(false) diff --git a/roles/vault/tasks/shared/mount.yml b/roles/vault/tasks/shared/mount.yml new file mode 100644 index 000000000..b98b45c57 --- /dev/null +++ b/roles/vault/tasks/shared/mount.yml @@ -0,0 +1,18 @@ +--- + +- name: shared/mount | Test if PKI mount exists + uri: + url: "{{ vault_leader_url }}/v1/sys/mounts/{{ mount_name }}/tune" + headers: "{{ vault_headers }}" + ignore_errors: true + register: vault_pki_mount_check + +- name: shared/mount | Mount PKI mount if needed + uri: + url: "{{ vault_leader_url }}/v1/sys/mounts/{{ mount_name }}" + headers: "{{ vault_headers }}" + method: POST + body_format: json + body: "{{ mount_options|d() }}" + status_code: 204 + when: vault_pki_mount_check|failed diff --git a/roles/vault/tasks/shared/pki_mount.yml b/roles/vault/tasks/shared/pki_mount.yml new file mode 100644 index 000000000..31faef434 --- /dev/null +++ b/roles/vault/tasks/shared/pki_mount.yml @@ -0,0 +1,11 @@ +--- + +- include: mount.yml + vars: + mount_name: pki + mount_options: + config: + default_lease_ttl: "{{ vault_default_lease_ttl }}" + max_lease_ttl: "{{ vault_max_lease_ttl }}" + description: The default PKI mount for Kubernetes + type: pki diff --git a/roles/vault/tasks/sync.yml b/roles/vault/tasks/shared/sync.yml similarity index 64% rename from roles/vault/tasks/sync.yml rename to roles/vault/tasks/shared/sync.yml index 0c9272457..02818b5f1 100644 --- a/roles/vault/tasks/sync.yml +++ b/roles/vault/tasks/shared/sync.yml @@ -8,7 +8,7 @@ - name: "sync_file | Cat the key file" command: "cat {{ sync_file_key_path }}" register: sync_file_key_cat - when: sync_file_is_cert|bool and inventory_hostname == sync_file_srcs|first + when: sync_file_is_cert|d() and inventory_hostname == sync_file_srcs|first - name: "sync_file | Set facts for file contents" set_fact: @@ -17,10 +17,13 @@ - name: "sync_file | Set fact for key contents" set_fact: sync_file_key_contents: "{{ hostvars[sync_file_srcs|first]['sync_file_key_cat']['stdout'] }}" - when: sync_file_is_cert|bool + when: sync_file_is_cert|d() - name: "sync_file | Ensure the directory exists" file: + group: "{{ sync_file_group|d('root') }}" + mode: "{{ sync_file_dir_mode|default('0750') }}" + owner: "{{ sync_file_owner|d('root') }}" path: "{{ sync_file_dir }}" state: directory when: inventory_hostname not in sync_file_srcs @@ -29,10 +32,16 @@ copy: content: "{{ sync_file_contents }}" dest: "{{ sync_file_path }}" + group: "{{ sync_file_group|d('root') }}" + mode: "{{ sync_file_mode|default('0640') }}" + owner: "{{ sync_file_owner|d('root') }}" when: inventory_hostname not in sync_file_srcs - name: "sync_file | Copy the key file to hosts that don't have it" copy: content: "{{ sync_file_key_contents }}" dest: "{{ sync_file_key_path }}" - when: sync_file_is_cert|bool and inventory_hostname not in sync_file_srcs + group: "{{ sync_file_group|d('root') }}" + mode: "{{ sync_file_mode|default('0640') }}" + owner: "{{ sync_file_owner|d('root') }}" + when: sync_file_is_cert|d() and inventory_hostname not in sync_file_srcs diff --git a/roles/vault/tasks/shared/sync_auth_certs.yml b/roles/vault/tasks/shared/sync_auth_certs.yml new file mode 100644 index 000000000..7bd9c4b49 --- /dev/null +++ b/roles/vault/tasks/shared/sync_auth_certs.yml @@ -0,0 +1,17 @@ +--- + +- include: sync_file.yml + vars: + sync_file: "auth-ca.pem" + sync_file_dir: "{{ vault_cert_dir }}" + sync_file_hosts: "{{ groups.vault }}" + sync_file_is_cert: true + +- name: shared/sync_auth_certs | Set facts for vault sync_file results + set_fact: + vault_auth_ca_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}" + + +- name: shared/sync_auth_certs | Unset sync_file_results after auth-ca.pem sync + set_fact: + sync_file_results: [] diff --git a/roles/vault/tasks/sync_file.yml b/roles/vault/tasks/shared/sync_file.yml similarity index 77% rename from roles/vault/tasks/sync_file.yml rename to roles/vault/tasks/shared/sync_file.yml index 4db1d4637..484d4aced 100644 --- a/roles/vault/tasks/sync_file.yml +++ b/roles/vault/tasks/shared/sync_file.yml @@ -6,17 +6,18 @@ set_fact: sync_file_dir: "{{ sync_file_path | dirname }}" sync_file: "{{ sync_file_path | basename }}" - when: sync_file_path|bool + when: sync_file_path is defined and sync_file_path != '' - name: "sync_file | Set fact for sync_file_path when undefined" set_fact: sync_file_path: "{{ (sync_file_dir, sync_file)|join('/') }}" - when: not sync_file_path|bool + when: sync_file_path is not defined or sync_file_path == '' - name: "sync_file | Set fact for key path name" set_fact: sync_file_key_path: "{{ sync_file_path.rsplit('.', 1)|first + '-key.' + sync_file_path.rsplit('.', 1)|last }}" - when: sync_file_is_cert|bool and not sync_file_key_path|bool + when: >- + sync_file_is_cert|d() and (sync_file_key_path is not defined or sync_file_key_path == '') - name: "sync_file | Check if file exists" stat: @@ -27,7 +28,7 @@ stat: path: "{{ sync_file_key_path }}" register: sync_file_key_stat - when: sync_file_is_cert|bool + when: sync_file_is_cert|d() - name: "sync_file | Combine all possible file sync sources" set_fact: @@ -43,13 +44,13 @@ with_items: "{{ sync_file_hosts | unique }}" loop_control: loop_var: host_item - when: sync_file_is_cert|bool and hostvars[host_item]["sync_file_key_stat"]["stat"]["exists"]|bool + when: sync_file_is_cert|d() and hostvars[host_item]["sync_file_key_stat"]["stat"]["exists"]|bool - name: "sync_file | Remove sync sources with files that do not match sync_file_srcs|first" set_fact: _: "{% if inventory_hostname in sync_file_srcs %}{{ sync_file_srcs.remove(inventory_hostname) }}{% endif %}" when: >- - sync_file_srcs|length > 0 and + sync_file_srcs|d([])|length > 1 and inventory_hostname != sync_file_srcs|first and sync_file_stat.stat.get("checksum") != hostvars[sync_file_srcs|first]["sync_file_stat"]["stat"]["checksum"] @@ -57,20 +58,20 @@ set_fact: _: "{% if inventory_hostname in sync_file_srcs %}{{ sync_file_srcs.remove(inventory_hostname) }}{% endif %}" when: >- - sync_file_is_cert|bool and - sync_file_key_srcs|length > 0 and - inventory_hostname != sync_file_srcs|first and + sync_file_is_cert|d() and + sync_file_key_srcs|d([])|length > 1 and + inventory_hostname != sync_file_key_srcs|first and sync_file_key_stat.stat.checksum != hostvars[sync_file_srcs|first]["sync_file_key_stat"]["stat"]["checksum"] - name: "sync_file | Consolidate file and key sources" set_fact: - sync_file_srcs: "{{ sync_file_srcs | intersect(sync_file_key_srcs) }}" - when: sync_file_is_cert|bool + sync_file_srcs: "{{ sync_file_srcs|d([]) | intersect(sync_file_key_srcs) }}" + when: sync_file_is_cert|d() - name: "sync_file | Set facts for situations where sync is not needed" set_fact: - sync_file_no_srcs: "{{ true if sync_file_srcs|length == 0 else false }}" - sync_file_unneeded: "{{ true if sync_file_srcs|length == sync_file_hosts|length else false }}" + sync_file_no_srcs: "{{ true if sync_file_srcs|d([])|length == 0 else false }}" + sync_file_unneeded: "{{ true if sync_file_srcs|d([])|length == sync_file_hosts|length else false }}" - name: "sync_file | Set sync_file_result fact" set_fact: @@ -88,10 +89,9 @@ - name: "Unset local vars to avoid variable bleed into next iteration" set_fact: - sync_file_dir: '' sync_file: '' + sync_file_dir: '' sync_file_key_path: '' - sync_file_hosts: [] + sync_file_key_srcs: [] sync_file_path: '' sync_file_srcs: [] - sync_file_key_srcs: [] diff --git a/roles/vault/templates/docker.service.j2 b/roles/vault/templates/docker.service.j2 new file mode 100644 index 000000000..c355b7f01 --- /dev/null +++ b/roles/vault/templates/docker.service.j2 @@ -0,0 +1,32 @@ +[Unit] +Description=hashicorp vault on docker +Documentation=https://github.com/hashicorp/vault +Wants=docker.socket +After=docker.service + +[Service] +User=root +Restart=always +RestartSec=15s +TimeoutStartSec=5 +LimitNOFILE=10000 +ExecReload={{ docker_bin_dir }}/docker restart {{ vault_container_name }} +ExecStop={{ docker_bin_dir }}/docker stop {{ vault_container_name }} +ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ vault_container_name }} +# Container has the following internal mount points: +# /vault/file/ # File backend storage location +# /vault/logs/ # Log files +ExecStart={{ docker_bin_dir }}/docker run \ +--name {{ vault_container_name }} --net=host \ +--cap-add=IPC_LOCK \ +-v {{ vault_cert_dir }}:{{ vault_cert_dir }} \ +-v {{ vault_config_dir }}:{{ vault_config_dir }} \ +-v {{ vault_log_dir }}:/vault/logs \ +-v {{ vault_roles_dir }}:{{ vault_roles_dir }} \ +-v {{ vault_secrets_dir }}:{{ vault_secrets_dir }} \ +--entrypoint=vault \ +{{ vault_image_repo }}:{{ vault_image_tag }} \ +server --config={{ vault_config_dir }}/config.json + +[Install] +WantedBy=multi-user.target diff --git a/roles/vault/templates/host.service.j2 b/roles/vault/templates/host.service.j2 new file mode 100644 index 000000000..11bce2f29 --- /dev/null +++ b/roles/vault/templates/host.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=vault +After=network.target + +[Service] +AmbientCapabilities=CAP_IPC_LOCK +ExecStart=/usr/bin/vault server --config={{ vault_config_dir }}/config.json +LimitNOFILE=40000 +NotifyAccess=all +Restart=always +RestartSec=10s +User={{ vault_adduser_vars.name }} + +[Install] +WantedBy=multi-user.target diff --git a/roles/vault/templates/rkt.service.j2 b/roles/vault/templates/rkt.service.j2 new file mode 100644 index 000000000..42b9458ac --- /dev/null +++ b/roles/vault/templates/rkt.service.j2 @@ -0,0 +1,33 @@ +[Unit] +Description=hashicorp vault on rkt +Documentation=https://github.com/hashicorp/vault +Wants=network.target + +[Service] +User=root +Restart=on-failure +RestartSec=10s +TimeoutStartSec=5 +LimitNOFILE=40000 +# Container has the following internal mount points: +# /vault/file/ # File backend storage location +# /vault/logs/ # Log files +ExecStart=/usr/bin/rkt run \ +--insecure-options=image \ +--volume=volume-vault-file,kind=host,source=/var/lib/vault \ +--volume=volume-vault-logs,kind=host,source={{ vault_log_dir }} \ +--volume=vault-cert-dir,kind=host,source={{ vault_cert_dir }} \ +--mount=volume=vault-cert-dir,target={{ vault_cert_dir }} \ +--volume=vault-conf-dir,kind=host,source={{ vault_config_dir }} \ +--mount=volume=vault-conf-dir,target={{ vault_config_dir }} \ +--volume=vault-secrets-dir,kind=host,source={{ vault_secrets_dir }} \ +--mount=volume=vault-secrets-dir,target={{ vault_secrets_dir }} \ +--volume=vault-roles-dir,kind=host,source={{ vault_roles_dir }} \ +--mount=volume=vault-roles-dir,target={{ vault_roles_dir }} \ +docker://{{ vault_image_repo }}:{{ vault_image_tag }} \ +--name={{ vault_container_name }} --net=host \ +--caps-retain=CAP_IPC_LOCK \ +--exec vault -- server --config={{ vault_config_dir }}/config.json + +[Install] +WantedBy=multi-user.target diff --git a/tests/templates/inventory-aws.j2 b/tests/templates/inventory-aws.j2 index e0771a94e..ee89bb5a4 100644 --- a/tests/templates/inventory-aws.j2 +++ b/tests/templates/inventory-aws.j2 @@ -15,6 +15,10 @@ node3 node1 node2 +[vault] +node1 +node2 + [k8s-cluster:children] kube-node kube-master diff --git a/tests/templates/inventory-gce.j2 b/tests/templates/inventory-gce.j2 index 3ede8f529..015bdb6a4 100644 --- a/tests/templates/inventory-gce.j2 +++ b/tests/templates/inventory-gce.j2 @@ -13,6 +13,9 @@ node2 [etcd] node3 + +[vault] +node3 {% elif mode is defined and mode == "ha" %} [kube-master] node1 @@ -24,6 +27,10 @@ node3 [etcd] node2 node3 + +[vault] +node2 +node3 {% else %} [kube-master] node1 @@ -33,6 +40,9 @@ node2 [etcd] node1 + +[vault] +node1 {% endif %} [k8s-cluster:children]