diff --git a/.gitignore b/.gitignore index 2b4490687..fc2bd5b1f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ temp *.tfstate.backup **/*.sw[pon] /ssh-bastion.conf +**/*.sw[pon] diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9aec99788..72f8de7e9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,6 +54,7 @@ before_script: LOG_LEVEL: "-vv" ETCD_DEPLOYMENT: "docker" KUBELET_DEPLOYMENT: "docker" + VAULT_DEPLOYMENT: "docker" WEAVE_CPU_LIMIT: "100m" MAGIC: "ci check this" @@ -106,6 +107,7 @@ before_script: -e ansible_python_interpreter=${PYPATH} -e ansible_ssh_user=${SSH_USER} -e bootstrap_os=${BOOTSTRAP_OS} + -e cert_management=${CERT_MGMT:-script} -e cloud_provider=gce -e deploy_netchecker=true -e download_localhost=true @@ -115,6 +117,7 @@ before_script: -e kubelet_deployment_type=${KUBELET_DEPLOYMENT} -e local_release_dir=${PWD}/downloads -e resolvconf_mode=${RESOLVCONF_MODE} + -e vault_deployment_type=${VAULT_DEPLOYMENT} cluster.yml @@ -292,6 +295,14 @@ before_script: ETCD_DEPLOYMENT: rkt KUBELET_DEPLOYMENT: rkt +.ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables +# stage: deploy-gce-part1 + KUBE_NETWORK_PLUGIN: canal + CERT_MGMT: vault + CLOUD_IMAGE: ubuntu-1604-xenial + CLOUD_REGION: us-central1-b + CLUSTER_MODE: separate + # Builds for PRs only (premoderated by unit-tests step) and triggers (auto) coreos-calico-sep: stage: deploy-gce-part1 @@ -506,6 +517,17 @@ ubuntu-rkt-sep: except: ['triggers'] only: ['master', /^pr-.*$/] +ubuntu-vault-sep: + stage: deploy-gce-part1 + <<: *job + <<: *gce + variables: + <<: *gce_variables + <<: *ubuntu_vault_sep_variables + when: manual + except: ['triggers'] + only: ['master', /^pr-.*$/] + # Premoderated with manual actions ci-authorized: <<: *job diff --git a/cluster.yml b/cluster.yml index 1a08283b5..d1115a57d 100644 --- a/cluster.yml +++ b/cluster.yml @@ -28,7 +28,14 @@ roles: - { role: kubernetes/preinstall, tags: preinstall } - { role: docker, tags: docker } - - { role: rkt, tags: rkt, when: "'rkt' in [ etcd_deployment_type, kubelet_deployment_type ]" } + - role: rkt + tags: rkt + when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]" + +- hosts: etcd:k8s-cluster:vault + any_errors_fatal: true + roles: + - { role: vault, tags: vault, vault_bootstrap: true, when: "cert_management == 'vault'" } - hosts: etcd:!k8s-cluster any_errors_fatal: true @@ -39,6 +46,15 @@ any_errors_fatal: true roles: - { role: etcd, tags: etcd } + +- hosts: etcd:k8s-cluster:vault + any_errors_fatal: true + roles: + - { role: vault, tags: vault, when: "cert_management == 'vault'"} + +- hosts: k8s-cluster + any_errors_fatal: true + roles: - { role: kubernetes/node, tags: node } - { role: network_plugin, tags: network } diff --git a/docs/vault.md b/docs/vault.md new file mode 100644 index 000000000..446d914c9 --- /dev/null +++ b/docs/vault.md @@ -0,0 +1,92 @@ +Hashicorp Vault Role +==================== + +Overview +-------- + +The Vault role is a two-step process: + +1. Bootstrap + +You cannot start your certificate management service securely with SSL (and +the datastore behind it) without having the certificates in-hand already. This +presents an unfortunate chicken and egg scenario, with one requiring the other. +To solve for this, the Bootstrap step was added. + +This step spins up a temporary instance of Vault to issue certificates for +Vault itself. It then leaves the temporary instance running, so that the Etcd +role can generate certs for itself as well. Eventually, this may be improved +to allow alternate backends (such as Consul), but currently the tasks are +hardcoded to only create a Vault role for Etcd. + +2. Cluster + +This step is where the long-term Vault cluster is started and configured. Its +first task, is to stop any temporary instances of Vault, to free the port for +the long-term. At the end of this task, the entire Vault cluster should be up +and read to go. + + +Keys to the Kingdom +------------------- + +The two most important security pieces of Vault are the ``root_token`` +and ``unsealing_keys``. Both of these values are given exactly once, during +the initialization of the Vault cluster. For convenience, they are saved +to the ``vault_secret_dir`` (default: /etc/vault/secrets) of every host in the +vault group. + +It is *highly* recommended that these secrets are removed from the servers after +your cluster has been deployed, and kept in a safe location of your choosing. +Naturally, the seriousness of the situation depends on what you're doing with +your Kargo cluster, but with these secrets, an attacker will have the ability +to authenticate to almost everything in Kubernetes and decode all private +(HTTPS) traffic on your network signed by Vault certificates. + +For even greater security, you may want to remove and store elsewhere any +CA keys generated as well (e.g. /etc/vault/ssl/ca-key.pem). + +Vault by default encrypts all traffic to and from the datastore backend, all +resting data, and uses TLS for its TCP listener. It is recommended that you +do not change the Vault config to disable TLS, unless you absolutely have to. + + +Usage +----- + +To get the Vault role running, you must to do two things at a minimum: + +1. Assign the ``vault`` group to at least 1 node in your inventory +2. Change ``cert_management`` to be ``vault`` instead of ``script`` + +Nothing else is required, but customization is possible. Check +``roles/vault/defaults/main.yml`` for the different variables that can be +overridden, most common being ``vault_config``, ``vault_port``, and +``vault_deployment_type``. + +Also, if you intend to use a Root or Intermediate CA generated elsewhere, +you'll need to copy the certificate and key to the hosts in the vault group +prior to running the vault role. By default, they'll be located at +``/etc/vault/ssl/ca.pem`` and ``/etc/vault/ssl/ca-key.pem``, respectively. + +Additional Notes: + +- ``groups.vault|first`` is considered the source of truth for Vault variables +- ``vault_leader_url`` is used as pointer for the current running Vault +- Each service should have its own role and credentials. Currently those + credentials are saved to ``/etc/vault/roles//``. The service will + need to read in those credentials, if they want to interact with Vault. + + +Potential Work +-------------- + +- Change the Vault role to not run certain tasks when ``root_token`` and + ``unseal_keys`` are not present. Alternatively, allow user input for these + values when missing. +- Add the ability to start temp Vault with Host, Rkt, or Docker +- Add a dynamic way to change out the backend role creation during Bootstrap, + so other services can be used (such as Consul) +- Segregate Server Cert generation from Auth Cert generation (separate CAs). + This work was partially started with the `auth_cert_backend` tasks, but would + need to be further applied to all roles (particularly Etcd and Kubernetes). diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 40b801dd9..cc26d5847 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -204,5 +204,12 @@ kpm_packages: [] rkt_version: 1.21.0 etcd_deployment_type: docker kubelet_deployment_type: docker +vault_deployment_type: docker efk_enabled: false + +## Certificate Management +## This setting determines whether certs are generated via scripts or whether a +## cluster of Hashicorp's Vault is started to issue certificates (using etcd +## as a backend). Options are "script" or "vault" +cert_management: script diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index 2fb9434bf..7b49f4f0e 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -94,7 +94,7 @@ - name: "Set default value for 'container_changed' to false" set_fact: - container_changed: "{{pull_required|bool|default(false)}}" + container_changed: "{{pull_required|default(false)|bool}}" - name: "Update the 'container_changed' fact" set_fact: diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index 9f117da76..a81670fd3 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -15,3 +15,5 @@ etcd_memory_limit: 512M # Uncomment to set CPU share for etcd #etcd_cpu_limit: 300m + +etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr', [])) }}" diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml index addd81053..bff76a129 100644 --- a/roles/etcd/meta/main.yml +++ b/roles/etcd/meta/main.yml @@ -6,3 +6,5 @@ dependencies: - role: download file: "{{ downloads.etcd }}" tags: download + +# NOTE: Dynamic task dependency on Vault Role if cert_management == "vault" diff --git a/roles/etcd/tasks/gen_certs.yml b/roles/etcd/tasks/gen_certs_script.yml similarity index 100% rename from roles/etcd/tasks/gen_certs.yml rename to roles/etcd/tasks/gen_certs_script.yml diff --git a/roles/etcd/tasks/gen_certs_vault.yml b/roles/etcd/tasks/gen_certs_vault.yml new file mode 100644 index 000000000..144e3b658 --- /dev/null +++ b/roles/etcd/tasks/gen_certs_vault.yml @@ -0,0 +1,77 @@ +--- + +- name: gen_certs_vault | Read in the local credentials + command: cat /etc/vault/roles/etcd/userpass + register: etcd_vault_creds_cat + when: inventory_hostname == groups.etcd|first + +- name: gen_certs_vault | Set facts for read Vault Creds + set_fact: + etcd_vault_creds: "{{ hostvars[groups.etcd|first]['etcd_vault_creds_cat']['stdout']|from_json }}" + when: inventory_hostname == groups.etcd|first + +- name: gen_certs_vault | Log into Vault and obtain an token + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ etcd_vault_creds.username }}" + headers: + Accept: application/json + Content-Type: application/json + method: POST + body_format: json + body: + password: "{{ etcd_vault_creds.password }}" + register: etcd_vault_login_result + when: inventory_hostname == groups.etcd|first + +- name: gen_certs_vault | Set fact for Vault API token + set_fact: + etcd_vault_headers: + Accept: application/json + Content-Type: application/json + X-Vault-Token: "{{ hostvars[groups.etcd|first]['etcd_vault_login_result']['json']['auth']['client_token'] }}" + +# Issue master certs to Etcd nodes +- include: ../../vault/tasks/shared/issue_cert.yml + vars: + issue_cert_alt_names: "{{ groups.etcd + ['localhost'] }}" + issue_cert_copy_ca: "{{ item == etcd_master_certs_needed|first }}" + issue_cert_file_group: "{{ etcd_cert_group }}" + issue_cert_file_owner: kube + issue_cert_headers: "{{ etcd_vault_headers }}" + issue_cert_hosts: "{{ groups.etcd }}" + issue_cert_ip_sans: >- + [ + {%- for host in groups.etcd -%} + "{{ hostvars[host]['ansible_default_ipv4']['address'] }}", + {%- endfor -%} + "127.0.0.1","::1" + ] + issue_cert_path: "{{ item }}" + issue_cert_role: etcd + issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + with_items: "{{ etcd_master_certs_needed|d([]) }}" + when: inventory_hostname in groups.etcd + notify: set etcd_secret_changed + +# Issue node certs to everyone else +- include: ../../vault/tasks/shared/issue_cert.yml + vars: + issue_cert_alt_names: "{{ etcd_node_cert_hosts }}" + issue_cert_copy_ca: "{{ item == etcd_node_certs_needed|first }}" + issue_cert_file_group: "{{ etcd_cert_group }}" + issue_cert_file_owner: kube + issue_cert_headers: "{{ etcd_vault_headers }}" + issue_cert_hosts: "{{ etcd_node_cert_hosts }}" + issue_cert_ip_sans: >- + [ + {%- for host in etcd_node_cert_hosts -%} + "{{ hostvars[host]['ansible_default_ipv4']['address'] }}", + {%- endfor -%} + "127.0.0.1","::1" + ] + issue_cert_path: "{{ item }}" + issue_cert_role: etcd + issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + with_items: "{{ etcd_node_certs_needed|d([]) }}" + when: inventory_hostname in etcd_node_cert_hosts + notify: set etcd_secret_changed diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index 394e5de64..6e952cd33 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -1,10 +1,24 @@ --- - include: pre_upgrade.yml tags: etcd-pre-upgrade + - include: check_certs.yml + when: cert_management == "script" tags: [etcd-secrets, facts] -- include: gen_certs.yml +- include: gen_certs_script.yml + when: cert_management == "script" tags: etcd-secrets + +- include: sync_etcd_master_certs.yml + when: cert_management == "vault" and inventory_hostname in groups.etcd + tags: etcd-secrets +- include: sync_etcd_node_certs.yml + when: cert_management == "vault" and inventory_hostname in etcd_node_cert_hosts + tags: etcd-secrets +- include: gen_certs_vault.yml + when: cert_management == "vault" and (etcd_master_certs_needed|d() or etcd_node_certs_needed|d()) + tags: etcd-secrets + - include: "install_{{ etcd_deployment_type }}.yml" when: is_etcd_master tags: upgrade diff --git a/roles/etcd/tasks/sync_etcd_master_certs.yml b/roles/etcd/tasks/sync_etcd_master_certs.yml new file mode 100644 index 000000000..c6badd3c2 --- /dev/null +++ b/roles/etcd/tasks/sync_etcd_master_certs.yml @@ -0,0 +1,38 @@ +--- + +- name: sync_etcd_master_certs | Create list of master certs needing creation + set_fact: + etcd_master_cert_list: >- + {{ etcd_master_cert_list|default([]) + [ + "admin-" + item + ".pem", + "member-" + item + ".pem" + ] }} + with_items: "{{ groups.etcd }}" + +- include: ../../vault/tasks/shared/sync_file.yml + vars: + sync_file: "{{ item }}" + sync_file_dir: "{{ etcd_cert_dir }}" + sync_file_hosts: "{{ groups.etcd }}" + sync_file_is_cert: true + with_items: "{{ etcd_master_cert_list|d([]) }}" + +- name: sync_etcd_certs | Set facts for etcd sync_file results + set_fact: + etcd_master_certs_needed: "{{ etcd_master_certs_needed|default([]) + [item.path] }}" + with_items: "{{ sync_file_results|d([]) }}" + when: item.no_srcs|bool + +- name: sync_etcd_certs | Unset sync_file_results after etcd certs sync + set_fact: + sync_file_results: [] + +- include: ../../vault/tasks/shared/sync_file.yml + vars: + sync_file: ca.pem + sync_file_dir: "{{ etcd_cert_dir }}" + sync_file_hosts: "{{ groups.etcd }}" + +- name: sync_etcd_certs | Unset sync_file_results after ca.pem sync + set_fact: + sync_file_results: [] diff --git a/roles/etcd/tasks/sync_etcd_node_certs.yml b/roles/etcd/tasks/sync_etcd_node_certs.yml new file mode 100644 index 000000000..2f82dcffd --- /dev/null +++ b/roles/etcd/tasks/sync_etcd_node_certs.yml @@ -0,0 +1,34 @@ +--- + +- name: sync_etcd_node_certs | Create list of node certs needing creation + set_fact: + etcd_node_cert_list: "{{ etcd_node_cert_list|default([]) + ['node-' + item + '.pem'] }}" + with_items: "{{ etcd_node_cert_hosts }}" + +- include: ../../vault/tasks/shared/sync_file.yml + vars: + sync_file: "{{ item }}" + sync_file_dir: "{{ etcd_cert_dir }}" + sync_file_hosts: "{{ etcd_node_cert_hosts }}" + sync_file_is_cert: true + with_items: "{{ etcd_node_cert_list|d([]) }}" + +- name: sync_etcd_node_certs | Set facts for etcd sync_file results + set_fact: + etcd_node_certs_needed: "{{ etcd_node_certs_needed|default([]) + [item.path] }}" + with_items: "{{ sync_file_results|d([]) }}" + when: item.no_srcs|bool + +- name: sync_etcd_node_certs | Unset sync_file_results after etcd node certs + set_fact: + sync_file_results: [] + +- include: ../../vault/tasks/shared/sync_file.yml + vars: + sync_file: ca.pem + sync_file_dir: "{{ etcd_cert_dir }}" + sync_file_hosts: "{{ etcd_node_cert_hosts }}" + +- name: sync_etcd_node_certs | Unset sync_file_results after ca.pem + set_fact: + sync_file_results: [] diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index bfe4a8cc8..e949e87de 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -25,6 +25,20 @@ template: "src=kubelet.{{ kubelet_deployment_type }}.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes" notify: restart kubelet +- name: install | Set SSL CA directories + set_fact: + ssl_ca_dirs: "[ + {% if ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] -%} + '/usr/share/ca-certificates', + {% elif ansible_os_family == 'RedHat' -%} + '/etc/pki/tls', + '/etc/pki/ca-trust', + {% elif ansible_os_family == 'Debian' -%} + '/usr/share/ca-certificates', + {% endif -%} + ]" + tags: facts + - name: install | Install kubelet launch script template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes notify: restart kubelet diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index 610f74dcd..6aa7d7e7c 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -10,6 +10,7 @@ common_required_pkgs: - rsync - bash-completion - socat + - unzip # Set to true if your network does not support IPv6 # This maybe necessary for pulling Docker images from diff --git a/roles/kubernetes/secrets/meta/main.yml b/roles/kubernetes/secrets/meta/main.yml new file mode 100644 index 000000000..dca734575 --- /dev/null +++ b/roles/kubernetes/secrets/meta/main.yml @@ -0,0 +1,2 @@ +--- +# NOTE: Dynamic task dependency on Vault Role if cert_management == "vault" diff --git a/roles/kubernetes/secrets/tasks/gen_certs.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml similarity index 93% rename from roles/kubernetes/secrets/tasks/gen_certs.yml rename to roles/kubernetes/secrets/tasks/gen_certs_script.yml index 484afff63..ebcfb7d81 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml @@ -160,20 +160,6 @@ {%- endif %} tags: facts -- name: SSL CA directories | Set SSL CA directories - set_fact: - ssl_ca_dirs: "[ - {% if ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] -%} - '/usr/share/ca-certificates', - {% elif ansible_os_family == 'RedHat' -%} - '/etc/pki/tls', - '/etc/pki/ca-trust', - {% elif ansible_os_family == 'Debian' -%} - '/usr/share/ca-certificates', - {% endif -%} - ]" - tags: facts - - name: Gen_certs | add CA to trusted CA dir copy: src: "{{ kube_cert_dir }}/ca.pem" diff --git a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml new file mode 100644 index 000000000..5a7c4827b --- /dev/null +++ b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml @@ -0,0 +1,84 @@ +--- + +- name: gen_certs_vault | Read in the local credentials + command: cat /etc/vault/roles/kube/userpass + register: kube_vault_creds_cat + when: inventory_hostname == groups['k8s-cluster']|first + +- name: gen_certs_vault | Set facts for read Vault Creds + set_fact: + kube_vault_creds: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_creds_cat']['stdout'] | from_json }}" + when: inventory_hostname == groups['k8s-cluster']|first + +- name: gen_certs_vault | Log into Vault and obtain an token + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ kube_vault_creds.username }}" + headers: + Accept: application/json + Content-Type: application/json + method: POST + body_format: json + body: + password: "{{ kube_vault_creds.password }}" + register: kube_vault_login_result + when: inventory_hostname == groups['k8s-cluster']|first + +- name: gen_certs_vault | Set fact for Vault API token + set_fact: + kube_vault_headers: + Accept: application/json + Content-Type: application/json + X-Vault-Token: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_login_result']['json']['auth']['client_token'] }}" + +# Issue certs to kube-master nodes +- include: ../../../vault/tasks/shared/issue_cert.yml + vars: + issue_cert_copy_ca: "{{ item == kube_master_certs_needed|first }}" + issue_cert_file_group: "{{ kube_cert_group }}" + issue_cert_file_owner: kube + issue_cert_headers: "{{ kube_vault_headers }}" + issue_cert_hosts: "{{ groups['kube-master'] }}" + issue_cert_path: "{{ item }}" + issue_cert_role: kube + issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + with_items: "{{ kube_master_certs_needed|d([]) }}" + when: inventory_hostname in groups['kube-master'] + +- include: ../../../vault/tasks/shared/issue_cert.yml + vars: + issue_cert_alt_names: >- + {{ + groups['kube-master'] + + ['kubernetes.default.svc.cluster.local', 'kubernetes.default.svc', 'kubernetes.default', 'kubernetes'] + + ['localhost'] + }} + issue_cert_file_group: "{{ kube_cert_group }}" + issue_cert_file_owner: kube + issue_cert_headers: "{{ kube_vault_headers }}" + issue_cert_hosts: "{{ groups['kube-master'] }}" + issue_cert_ip_sans: >- + [ + {%- for host in groups['kube-master'] -%} + "{{ hostvars[host]['ansible_default_ipv4']['address'] }}", + {%- endfor -%} + "127.0.0.1","::1","{{ kube_apiserver_ip }}" + ] + issue_cert_path: "{{ item }}" + issue_cert_role: kube + issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + with_items: "{{ kube_api_certs_needed|d([]) }}" + when: inventory_hostname in groups['kube-master'] + +# Issue node certs to k8s-cluster nodes +- include: ../../../vault/tasks/shared/issue_cert.yml + vars: + issue_cert_copy_ca: "{{ item == kube_node_certs_needed|first }}" + issue_cert_file_group: "{{ kube_cert_group }}" + issue_cert_file_owner: kube + issue_cert_headers: "{{ kube_vault_headers }}" + issue_cert_hosts: "{{ groups['k8s-cluster'] }}" + issue_cert_path: "{{ item }}" + issue_cert_role: kube + issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + with_items: "{{ kube_node_certs_needed|d([]) }}" + when: inventory_hostname in groups['k8s-cluster'] diff --git a/roles/kubernetes/secrets/tasks/main.yml b/roles/kubernetes/secrets/tasks/main.yml index 4d25a94af..f442b62b3 100644 --- a/roles/kubernetes/secrets/tasks/main.yml +++ b/roles/kubernetes/secrets/tasks/main.yml @@ -70,7 +70,19 @@ delegate_to: "{{groups['kube-master'][0]}}" when: gen_tokens|default(false) -- include: gen_certs.yml +- include: gen_certs_script.yml + when: cert_management == "script" tags: k8s-secrets + +- include: sync_kube_master_certs.yml + when: cert_management == "vault" and inventory_hostname in groups['kube-master'] + tags: k8s-secrets +- include: sync_kube_node_certs.yml + when: cert_management == "vault" and inventory_hostname in groups['k8s-cluster'] + tags: k8s-secrets +- include: gen_certs_vault.yml + when: cert_management == "vault" + tags: k8s-secrets + - include: gen_tokens.yml tags: k8s-secrets diff --git a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml new file mode 100644 index 000000000..0561d6581 --- /dev/null +++ b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml @@ -0,0 +1,58 @@ +--- + +- name: sync_kube_master_certs | Create list of needed kube admin certs + set_fact: + kube_master_cert_list: "{{ kube_master_cert_list|d([]) + ['admin-' + item + '.pem'] }}" + with_items: "{{ groups['kube-master'] }}" + +- include: ../../../vault/tasks/shared/sync_file.yml + vars: + sync_file: "{{ item }}" + sync_file_dir: "{{ kube_cert_dir }}" + sync_file_group: "{{ kube_cert_group }}" + sync_file_hosts: "{{ groups['kube-master'] }}" + sync_file_is_cert: true + sync_file_owner: kube + with_items: "{{ kube_master_cert_list|d([]) }}" + +- name: sync_kube_master_certs | Set facts for kube admin sync_file results + set_fact: + kube_master_certs_needed: "{{ kube_master_certs_needed|default([]) + [item.path] }}" + with_items: "{{ sync_file_results|d([]) }}" + when: item.no_srcs|bool + +- name: sync_kube_master_certs | Unset sync_file_results after kube admin certs + set_fact: + sync_file_results: [] + +- include: ../../../vault/tasks/shared/sync_file.yml + vars: + sync_file: "apiserver.pem" + sync_file_dir: "{{ kube_cert_dir }}" + sync_file_group: "{{ kube_cert_group }}" + sync_file_hosts: "{{ groups['kube-master'] }}" + sync_file_is_cert: true + sync_file_owner: kube + +- name: sync_kube_master_certs | Set facts for apiserver sync_file results + set_fact: + kube_api_certs_needed: "{{ item.path }}" + with_items: "{{ sync_file_results|d([]) }}" + when: "{{ item.no_srcs }}" + +- name: sync_kube_master_certs | Unset sync_file_results after apiserver cert + set_fact: + sync_file_results: [] + + +- include: ../../../vault/tasks/shared/sync_file.yml + vars: + sync_file: ca.pem + sync_file_dir: "{{ kube_cert_dir }}" + sync_file_group: "{{ kube_cert_group }}" + sync_file_hosts: "{{ groups['kube-master'] }}" + sync_file_owner: kube + +- name: sync_kube_master_certs | Unset sync_file_results after ca.pem + set_fact: + sync_file_results: [] diff --git a/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml b/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml new file mode 100644 index 000000000..9d6deb563 --- /dev/null +++ b/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml @@ -0,0 +1,38 @@ +--- + +- name: sync_kube_node_certs | Create list of needed certs + set_fact: + kube_node_cert_list: "{{ kube_node_cert_list|default([]) + ['node-' + item + '.pem'] }}" + with_items: "{{ groups['k8s-cluster'] }}" + +- include: ../../../vault/tasks/shared/sync_file.yml + vars: + sync_file: "{{ item }}" + sync_file_dir: "{{ kube_cert_dir }}" + sync_file_group: "{{ kuber_cert_group }}" + sync_file_hosts: "{{ groups['k8s-cluster'] }}" + sync_file_is_cert: true + sync_file_owner: kube + with_items: "{{ kube_node_cert_list|default([]) }}" + +- name: sync_kube_node_certs | Set facts for kube-master sync_file results + set_fact: + kube_node_certs_needed: "{{ kube_node_certs_needed|default([]) + [item.path] }}" + with_items: "{{ sync_file_results|d([]) }}" + when: item.no_srcs|bool + +- name: sync_kube_node_certs | Unset sync_file_results after kube node certs + set_fact: + sync_file_results: [] + +- include: ../../../vault/tasks/shared/sync_file.yml + vars: + sync_file: ca.pem + sync_file_dir: "{{ kube_cert_dir }}" + sync_file_group: "{{ kuber_cert_group }}" + sync_file_hosts: "{{ groups['k8s-cluster'] }}" + sync_file_owner: kube + +- name: sync_kube_node_certs | Unset sync_file_results after ca.pem + set_fact: + sync_file_results: [] diff --git a/roles/vault/defaults/main.yml b/roles/vault/defaults/main.yml new file mode 100644 index 000000000..4a0a60982 --- /dev/null +++ b/roles/vault/defaults/main.yml @@ -0,0 +1,90 @@ +--- + +vault_adduser_vars: + comment: "Hashicorp Vault User" + createhome: no + name: vault + shell: /sbin/nologin + system: yes +vault_base_dir: /etc/vault +# https://releases.hashicorp.com/vault/0.6.4/vault_0.6.4_SHA256SUMS +vault_binary_checksum: 04d87dd553aed59f3fe316222217a8d8777f40115a115dac4d88fac1611c51a6 +vault_bootstrap: false +vault_ca_options: + common_name: kube-cluster-ca + format: pem + ttl: 87600h +vault_cert_dir: "{{ vault_base_dir }}/ssl" +vault_client_headers: + Accept: "application/json" + Content-Type: "application/json" +vault_config: + backend: + etcd: + address: "{{ vault_etcd_url }}" + ha_enabled: "true" + redirect_addr: "https://{{ ansible_default_ipv4.address }}:{{ vault_port }}" + tls_ca_file: "{{ vault_cert_dir }}/ca.pem" + cluster_name: "kubernetes-vault" + default_lease_ttl: "{{ vault_default_lease_ttl }}" + listener: + tcp: + address: "0.0.0.0:{{ vault_port }}" + tls_cert_file: "{{ vault_cert_dir }}/api.pem" + tls_key_file: "{{ vault_cert_dir }}/api-key.pem" + max_lease_ttl: "{{ vault_max_lease_ttl }}" +vault_config_dir: "{{ vault_base_dir }}/config" +vault_container_name: kube-hashicorp-vault +# This variable is meant to match the GID of vault inside Hashicorp's official Vault Container +vault_default_lease_ttl: 720h +vault_default_role_permissions: + allow_any_name: true +vault_deployment_type: docker +vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip" +vault_download_vars: + container: "{{ vault_deployment_type != 'host' }}" + dest: "vault/vault_{{ vault_version }}_linux_amd64.zip" + enabled: true + mode: "0755" + owner: "vault" + repo: "{{ vault_image_repo }}" + sha256: "{{ vault_binary_checksum if vault_deployment_type == 'host' else vault_digest_checksum|d(none) }}" + source_url: "{{ vault_download_url }}" + tag: "{{ vault_image_tag }}" + unarchive: true + url: "{{ vault_download_url }}" + version: "{{ vault_version }}" +vault_etcd_url: "https://{{ hostvars[groups.etcd[0]]['ansible_default_ipv4']['address'] }}:2379" +vault_image_repo: "vault" +vault_image_tag: "{{ vault_version }}" +vault_log_dir: "/var/log/vault" +vault_max_lease_ttl: 87600h +vault_needs_gen: false +vault_port: 8200 +# Although "cert" is an option, ansible has no way to auth via cert until +# upstream merges: https://github.com/ansible/ansible/pull/18141 +vault_role_auth_method: userpass +vault_roles: + - name: etcd + group: etcd + policy_rules: default + role_options: default + - name: kube + group: k8s-cluster + policy_rules: default + role_options: default +vault_roles_dir: "{{ vault_base_dir }}/roles" +vault_secret_shares: 1 +vault_secret_threshold: 1 +vault_secrets_dir: "{{ vault_base_dir }}/secrets" +vault_temp_config: + default_lease_ttl: "{{ vault_default_lease_ttl }}" + backend: + file: + path: /vault/file + listener: + tcp: + address: "0.0.0.0:{{ vault_port }}" + tls_disable: "true" +vault_temp_container_name: vault-temp +vault_version: 0.6.4 diff --git a/roles/vault/meta/main.yml b/roles/vault/meta/main.yml new file mode 100644 index 000000000..ba559e361 --- /dev/null +++ b/roles/vault/meta/main.yml @@ -0,0 +1,8 @@ +--- + +dependencies: + - role: adduser + user: "{{ vault_adduser_vars }}" + - role: download + file: "{{ vault_download_vars }}" + tags: download diff --git a/roles/vault/tasks/bootstrap/ca_trust.yml b/roles/vault/tasks/bootstrap/ca_trust.yml new file mode 100644 index 000000000..2bcfcc2fa --- /dev/null +++ b/roles/vault/tasks/bootstrap/ca_trust.yml @@ -0,0 +1,32 @@ +--- + +- name: bootstrap/ca_trust | pull CA from cert from groups.vault|first + command: "cat {{ vault_cert_dir }}/ca.pem" + register: vault_cert_file_cat + when: inventory_hostname == groups.vault|first + +# This part is mostly stolen from the etcd role +- name: bootstrap/ca_trust | target ca-certificate store file + set_fact: + ca_cert_path: >- + {% if ansible_os_family == "Debian" -%} + /usr/local/share/ca-certificates/kube-cluster-ca.crt + {%- elif ansible_os_family == "RedHat" -%} + /etc/pki/ca-trust/source/anchors/kube-cluster-ca.crt + {%- elif ansible_os_family == "CoreOS" -%} + /etc/ssl/certs/kube-cluster-ca.pem + {%- endif %} + +- name: bootstrap/ca_trust | add CA to trusted CA dir + copy: + content: "{{ hostvars[groups.vault|first]['vault_cert_file_cat']['stdout'] }}" + dest: "{{ ca_cert_path }}" + register: vault_ca_cert + +- name: bootstrap/ca_trust | update ca-certificates (Debian/Ubuntu/CoreOS) + command: update-ca-certificates + when: vault_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"] + +- name: bootstrap/ca_trust | update ca-certificates (RedHat) + command: update-ca-trust extract + when: vault_ca_cert.changed and ansible_os_family == "RedHat" diff --git a/roles/vault/tasks/bootstrap/create_etcd_role.yml b/roles/vault/tasks/bootstrap/create_etcd_role.yml new file mode 100644 index 000000000..57518f944 --- /dev/null +++ b/roles/vault/tasks/bootstrap/create_etcd_role.yml @@ -0,0 +1,10 @@ +--- + +- include: ../shared/create_role.yml + vars: + create_role_name: "{{ item.name }}" + create_role_group: "{{ item.group }}" + create_role_policy_rules: "{{ item.policy_rules }}" + create_role_options: "{{ item.role_options }}" + with_items: "{{ vault_roles }}" + when: item.name == "etcd" diff --git a/roles/vault/tasks/bootstrap/gen_auth_ca.yml b/roles/vault/tasks/bootstrap/gen_auth_ca.yml new file mode 100644 index 000000000..10313ecea --- /dev/null +++ b/roles/vault/tasks/bootstrap/gen_auth_ca.yml @@ -0,0 +1,21 @@ +--- + +- name: bootstrap/gen_auth_ca | Generate Root CA + uri: + url: "{{ vault_leader_url }}/v1/auth-pki/root/generate/exported" + headers: "{{ vault_headers }}" + method: POST + body_format: json + body: "{{ vault_ca_options }}" + register: vault_auth_ca_gen + when: inventory_hostname == groups.vault|first + +- name: bootstrap/gen_auth_ca | Copy auth CA cert to Vault nodes + copy: + content: "{{ hostvars[groups.vault|first]['vault_auth_ca_gen']['json']['data']['certificate'] }}" + dest: "{{ vault_cert_dir }}/auth-ca.pem" + +- name: bootstrap/gen_auth_ca | Copy auth CA key to Vault nodes + copy: + content: "{{ hostvars[groups.vault|first]['vault_auth_ca_gen']['json']['data']['private_key'] }}" + dest: "{{ vault_cert_dir }}/auth-ca-key.pem" diff --git a/roles/vault/tasks/bootstrap/gen_ca.yml b/roles/vault/tasks/bootstrap/gen_ca.yml new file mode 100644 index 000000000..ab1cb6345 --- /dev/null +++ b/roles/vault/tasks/bootstrap/gen_ca.yml @@ -0,0 +1,31 @@ +--- + +- name: bootstrap/gen_ca | Ensure vault_cert_dir exists + file: + mode: 0755 + path: "{{ vault_cert_dir }}" + state: directory + +- name: bootstrap/gen_ca | Generate Root CA in vault-temp + uri: + url: "{{ vault_leader_url }}/v1/pki/root/generate/exported" + headers: "{{ vault_headers }}" + method: POST + body_format: json + body: "{{ vault_ca_options }}" + register: vault_ca_gen + when: inventory_hostname == groups.vault|first and vault_ca_cert_needed + +- name: bootstrap/gen_ca | Copy root CA cert locally + copy: + content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['certificate'] }}" + dest: "{{ vault_cert_dir }}/ca.pem" + mode: 0644 + when: vault_ca_cert_needed + +- name: bootstrap/gen_ca | Copy root CA key locally + copy: + content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['private_key'] }}" + dest: "{{ vault_cert_dir }}/ca-key.pem" + mode: 0640 + when: vault_ca_cert_needed diff --git a/roles/vault/tasks/bootstrap/gen_vault_certs.yml b/roles/vault/tasks/bootstrap/gen_vault_certs.yml new file mode 100644 index 000000000..4a7f4ed31 --- /dev/null +++ b/roles/vault/tasks/bootstrap/gen_vault_certs.yml @@ -0,0 +1,28 @@ +--- + +- name: boostrap/gen_vault_certs | Add the vault role + uri: + url: "{{ vault_leader_url }}/v1/pki/roles/vault" + headers: "{{ vault_headers }}" + method: POST + body_format: json + body: "{{ vault_default_role_permissions }}" + status_code: 204 + when: inventory_hostname == groups.vault|first and vault_api_cert_needed + +- include: ../shared/issue_cert.yml + vars: + issue_cert_alt_names: "{{ groups.vault + ['localhost'] }}" + issue_cert_hosts: "{{ groups.vault }}" + issue_cert_ip_sans: >- + [ + {%- for host in groups.vault -%} + "{{ hostvars[host]['ansible_default_ipv4']['address'] }}", + {%- endfor -%} + "127.0.0.1","::1" + ] + issue_cert_path: "{{ vault_cert_dir }}/api.pem" + issue_cert_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + issue_cert_role: vault + issue_cert_url: "{{ vault_leader_url }}" + when: vault_api_cert_needed diff --git a/roles/vault/tasks/bootstrap/main.yml b/roles/vault/tasks/bootstrap/main.yml new file mode 100644 index 000000000..edd2912d3 --- /dev/null +++ b/roles/vault/tasks/bootstrap/main.yml @@ -0,0 +1,58 @@ +--- + +- include: ../shared/check_vault.yml + when: inventory_hostname in groups.vault +- include: sync_secrets.yml + when: inventory_hostname in groups.vault +- include: ../shared/find_leader.yml + when: inventory_hostname in groups.vault and vault_cluster_is_initialized|d() + +## Sync Certs + +- include: sync_vault_certs.yml + when: inventory_hostname in groups.vault + +## Generate Certs + +# Start a temporary instance of Vault +- include: start_vault_temp.yml + when: >- + inventory_hostname == groups.vault|first and + not vault_cluster_is_initialized + +# NOTE: The next 2 steps run against temp Vault and long-term Vault + +# Ensure PKI mount exists +- include: ../shared/pki_mount.yml + when: >- + inventory_hostname == groups.vault|first + +# If the Root CA already exists, ensure Vault's PKI is using it +- include: ../shared/config_ca.yml + vars: + ca_name: ca + mount_name: pki + when: >- + inventory_hostname == groups.vault|first and + not vault_ca_cert_needed + +# Generate root CA certs for Vault if none exist +- include: gen_ca.yml + when: >- + inventory_hostname in groups.vault and + not vault_cluster_is_initialized and + vault_ca_cert_needed + +# Generate Vault API certs +- include: gen_vault_certs.yml + when: inventory_hostname in groups.vault and vault_api_cert_needed + +# Update all host's CA bundle +- include: ca_trust.yml + +## Add Etcd Role to Vault (if needed) + +- include: role_auth_cert.yml + when: vault_role_auth_method == "cert" +- include: role_auth_userpass.yml + when: vault_role_auth_method == "userpass" diff --git a/roles/vault/tasks/bootstrap/role_auth_cert.yml b/roles/vault/tasks/bootstrap/role_auth_cert.yml new file mode 100644 index 000000000..7bbf58e86 --- /dev/null +++ b/roles/vault/tasks/bootstrap/role_auth_cert.yml @@ -0,0 +1,25 @@ +--- + +- include: ../shared/sync_auth_certs.yml + when: inventory_hostname in groups.vault + +- include: ../shared/cert_auth_mount.yml + when: inventory_hostname == groups.vault|first + +- include: ../shared/auth_backend.yml + vars: + auth_backend_description: A Cert-based Auth primarily for services needing to issue certificates + auth_backend_name: cert + auth_backend_type: cert + when: inventory_hostname == groups.vault|first + +- include: gen_auth_ca.yml + when: inventory_hostname in groups.vault and vault_auth_ca_cert_needed + +- include: ../shared/config_ca.yml + vars: + ca_name: auth-ca + mount_name: auth-pki + when: inventory_hostname == groups.vault|first and not vault_auth_ca_cert_needed +- include: create_etcd_role.yml + when: inventory_hostname in groups.etcd diff --git a/roles/vault/tasks/bootstrap/role_auth_userpass.yml b/roles/vault/tasks/bootstrap/role_auth_userpass.yml new file mode 100644 index 000000000..ad09ab05b --- /dev/null +++ b/roles/vault/tasks/bootstrap/role_auth_userpass.yml @@ -0,0 +1,10 @@ +--- + +- include: ../shared/auth_backend.yml + vars: + auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates + auth_backend_path: userpass + auth_backend_type: userpass + when: inventory_hostname == groups.vault|first +- include: create_etcd_role.yml + when: inventory_hostname in groups.etcd diff --git a/roles/vault/tasks/bootstrap/start_vault_temp.yml b/roles/vault/tasks/bootstrap/start_vault_temp.yml new file mode 100644 index 000000000..eeaaad535 --- /dev/null +++ b/roles/vault/tasks/bootstrap/start_vault_temp.yml @@ -0,0 +1,43 @@ +--- + +- name: bootstrap/start_vault_temp | Ensure vault-temp isn't already running + shell: if docker rm -f {{ vault_temp_container_name }} 2>&1 1>/dev/null;then echo true;else echo false;fi + register: vault_temp_stop_check + changed_when: "{{ 'true' in vault_temp_stop_check.stdout }}" + +- name: bootstrap/start_vault_temp | Start single node Vault with file backend + command: > + docker run -d --cap-add=IPC_LOCK --name {{ vault_temp_container_name }} + -p {{ vault_port }}:{{ vault_port }} + -e 'VAULT_LOCAL_CONFIG={{ vault_temp_config|to_json }}' + -v /etc/vault:/etc/vault + {{ vault_image_repo }}:{{ vault_version }} server + +- name: bootstrap/start_vault_temp | Initialize vault-temp + uri: + url: "http://localhost:{{ vault_port }}/v1/sys/init" + headers: "{{ vault_client_headers }}" + method: PUT + body_format: json + body: + secret_shares: 1 + secret_threshold: 1 + register: vault_temp_init + +# NOTE: vault_headers and vault_url are used by subsequent issue calls +- name: bootstrap/start_vault_temp | Set needed vault facts + set_fact: + vault_leader_url: "http://{{ inventory_hostname }}:{{ vault_port }}" + vault_temp_unseal_keys: "{{ vault_temp_init.json['keys'] }}" + vault_temp_root_token: "{{ vault_temp_init.json.root_token }}" + vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_temp_init.json.root_token}) }}" + +- name: bootstrap/start_vault_temp | Unseal vault-temp + uri: + url: "http://localhost:{{ vault_port }}/v1/sys/unseal" + headers: "{{ vault_headers }}" + method: POST + body_format: json + body: + key: "{{ item }}" + with_items: "{{ vault_temp_unseal_keys|default([]) }}" diff --git a/roles/vault/tasks/bootstrap/sync_secrets.yml b/roles/vault/tasks/bootstrap/sync_secrets.yml new file mode 100644 index 000000000..3b5af775a --- /dev/null +++ b/roles/vault/tasks/bootstrap/sync_secrets.yml @@ -0,0 +1,48 @@ +--- + +- include: ../shared/sync_file.yml + vars: + sync_file: "{{ item }}" + sync_file_dir: "{{ vault_secrets_dir }}" + sync_file_hosts: "{{ groups.vault }}" + with_items: + - root_token + - unseal_keys + +- name: bootstrap/sync_secrets | Set fact based on sync_file_results + set_fact: + vault_secrets_available: "{{ vault_secrets_available|default(true) and not item.no_srcs }}" + with_items: "{{ sync_file_results|d([]) }}" + +- name: bootstrap/sync_secrets | Reset sync_file_results to avoid variable bleed + set_fact: + sync_file_results: [] + +- name: bootstrap/sync_secrets | Print out warning message if secrets are not available and vault is initialized + pause: + prompt: > + Vault orchestration may not be able to proceed. The Vault cluster is initialzed, but + 'root_token' or 'unseal_keys' were not found in {{ vault_secrets_dir }}. These are + needed for many vault orchestration steps. + when: vault_cluster_is_initialized and not vault_secrets_available + +- name: bootstrap/sync_secrets | Cat root_token from a vault host + command: "cat {{ vault_secrets_dir }}/root_token" + register: vault_root_token_cat + when: vault_secrets_available and inventory_hostname == groups.vault|first + +- name: bootstrap/sync_secrets | Cat unseal_keys from a vault host + command: "cat {{ vault_secrets_dir }}/unseal_keys" + register: vault_unseal_keys_cat + when: vault_secrets_available and inventory_hostname == groups.vault|first + +- name: bootstrap/sync_secrets | Set needed facts for Vault API interaction when Vault is already running + set_fact: + vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token_cat']['stdout'] }}" + vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys_cat']['stdout_lines'] }}" + when: vault_secrets_available + +- name: bootstrap/sync_secrets | Update vault_headers if we have the root_token + set_fact: + vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token}) }}" + when: vault_secrets_available diff --git a/roles/vault/tasks/bootstrap/sync_vault_certs.yml b/roles/vault/tasks/bootstrap/sync_vault_certs.yml new file mode 100644 index 000000000..ab088753f --- /dev/null +++ b/roles/vault/tasks/bootstrap/sync_vault_certs.yml @@ -0,0 +1,32 @@ +--- + +- include: ../shared/sync_file.yml + vars: + sync_file: "ca.pem" + sync_file_dir: "{{ vault_cert_dir }}" + sync_file_hosts: "{{ groups.vault }}" + sync_file_is_cert: true + +- name: bootstrap/sync_vault_certs | Set facts for vault sync_file results + set_fact: + vault_ca_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}" + +- name: bootstrap/sync_vault_certs | Unset sync_file_results after ca.pem sync + set_fact: + sync_file_results: [] + +- include: ../shared/sync_file.yml + vars: + sync_file: "api.pem" + sync_file_dir: "{{ vault_cert_dir }}" + sync_file_hosts: "{{ groups.vault }}" + sync_file_is_cert: true + +- name: bootstrap/sync_vault_certs | Set fact if Vault's API cert is needed + set_fact: + vault_api_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}" + +- name: bootstrap/sync_vault_certs | Unset sync_file_results after api.pem sync + set_fact: + sync_file_results: [] + diff --git a/roles/vault/tasks/cluster/binary.yml b/roles/vault/tasks/cluster/binary.yml new file mode 100644 index 000000000..41024dd54 --- /dev/null +++ b/roles/vault/tasks/cluster/binary.yml @@ -0,0 +1,9 @@ +--- + +- name: cluster/binary | Copy vault binary from downloaddir + copy: + src: "{{ local_release_dir }}/vault/vault" + dest: "/usr/bin/vault" + remote_src: true + mode: "0755" + owner: vault diff --git a/roles/vault/tasks/cluster/configure.yml b/roles/vault/tasks/cluster/configure.yml new file mode 100644 index 000000000..7ac8f5f9e --- /dev/null +++ b/roles/vault/tasks/cluster/configure.yml @@ -0,0 +1,14 @@ +--- + +- name: cluster/configure | Ensure the vault/config directory exists + file: + dest: "{{ vault_config_dir }}" + mode: 0750 + state: directory + +- name: cluster/configure | Lay down the configuration file + copy: + content: "{{ vault_config | to_nice_json(indent=4) }}" + dest: "{{ vault_config_dir }}/config.json" + mode: 0640 + register: vault_config_change diff --git a/roles/vault/tasks/cluster/create_roles.yml b/roles/vault/tasks/cluster/create_roles.yml new file mode 100644 index 000000000..a135137da --- /dev/null +++ b/roles/vault/tasks/cluster/create_roles.yml @@ -0,0 +1,9 @@ +--- + +- include: ../shared/create_role.yml + vars: + create_role_name: "{{ item.name }}" + create_role_group: "{{ item.group }}" + create_role_policy_rules: "{{ item.policy_rules }}" + create_role_options: "{{ item.role_options }}" + with_items: "{{ vault_roles|d([]) }}" diff --git a/roles/vault/tasks/cluster/init.yml b/roles/vault/tasks/cluster/init.yml new file mode 100644 index 000000000..60aaf9591 --- /dev/null +++ b/roles/vault/tasks/cluster/init.yml @@ -0,0 +1,52 @@ +--- + +- name: cluster/init | Initialize Vault + uri: + url: "https://{{ groups.vault|first }}:{{ vault_port }}/v1/sys/init" + headers: "{{ vault_client_headers }}" + method: POST + body_format: json + body: + secret_shares: "{{ vault_secret_shares }}" + secret_threshold: "{{ vault_secret_threshold }}" + validate_certs: false + register: vault_init_result + when: not vault_cluster_is_initialized and inventory_hostname == groups.vault|first + +- name: cluster/init | Set facts on the results of the initialization + set_fact: + vault_unseal_keys: "{{ vault_init_result.json['keys'] }}" + vault_root_token: "{{ vault_init_result.json.root_token }}" + vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_init_result.json.root_token}) }}" + when: not vault_cluster_is_initialized and inventory_hostname == groups.vault|first + +- name: cluster/init | Ensure all hosts have these facts + set_fact: + vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys'] }}" + vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token'] }}" + when: not vault_cluster_is_initialized and inventory_hostname != groups.vault|first + +- name: cluster/init | Ensure the vault_secrets_dir exists + file: + mode: 0750 + path: "{{ vault_secrets_dir }}" + state: directory + +- name: cluster/init | Ensure all in groups.vault have the unseal_keys locally + copy: + content: "{{ vault_unseal_keys|join('\n') }}" + dest: "{{ vault_secrets_dir }}/unseal_keys" + mode: 0640 + when: not vault_cluster_is_initialized + +- name: cluster/init | Ensure all in groups.vault have the root_token locally + copy: + content: "{{ vault_root_token }}" + dest: "{{ vault_secrets_dir }}/root_token" + mode: 0640 + when: not vault_cluster_is_initialized + +- name: cluster/init | Ensure vault_headers and vault statuses are updated + set_fact: + vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token})}}" + vault_cluster_is_initialized: true diff --git a/roles/vault/tasks/cluster/main.yml b/roles/vault/tasks/cluster/main.yml new file mode 100644 index 000000000..5dab550aa --- /dev/null +++ b/roles/vault/tasks/cluster/main.yml @@ -0,0 +1,35 @@ +--- + +- include: ../shared/check_vault.yml + when: inventory_hostname in groups.vault +- include: ../shared/check_etcd.yml + when: inventory_hostname in groups.vault + +## Vault Cluster Setup + +- include: configure.yml + when: inventory_hostname in groups.vault +- include: binary.yml + when: inventory_hostname in groups.vault and vault_deployment_type == "host" +- include: systemd.yml + when: inventory_hostname in groups.vault +- include: init.yml + when: inventory_hostname in groups.vault +- include: unseal.yml + when: inventory_hostname in groups.vault +- include: ../shared/find_leader.yml + when: inventory_hostname in groups.vault +- include: ../shared/pki_mount.yml + when: inventory_hostname == groups.vault|first +- include: ../shared/config_ca.yml + vars: + ca_name: ca + mount_name: pki + when: inventory_hostname == groups.vault|first + +## Vault Policies, Roles, and Auth Backends + +- include: role_auth_cert.yml + when: vault_role_auth_method == "cert" +- include: role_auth_userpass.yml + when: vault_role_auth_method == "userpass" diff --git a/roles/vault/tasks/cluster/role_auth_cert.yml b/roles/vault/tasks/cluster/role_auth_cert.yml new file mode 100644 index 000000000..9f186e3ff --- /dev/null +++ b/roles/vault/tasks/cluster/role_auth_cert.yml @@ -0,0 +1,19 @@ +--- + +- include: ../shared/cert_auth_mount.yml + when: inventory_hostname == groups.vault|first + +- include: ../shared/auth_backend.yml + vars: + auth_backend_description: A Cert-based Auth primarily for services needing to issue certificates + auth_backend_name: cert + auth_backend_type: cert + when: inventory_hostname == groups.vault|first + +- include: ../shared/config_ca.yml + vars: + ca_name: auth-ca + mount_name: auth-pki + when: inventory_hostname == groups.vault|first + +- include: create_roles.yml diff --git a/roles/vault/tasks/cluster/role_auth_userpass.yml b/roles/vault/tasks/cluster/role_auth_userpass.yml new file mode 100644 index 000000000..ac3b2c6c1 --- /dev/null +++ b/roles/vault/tasks/cluster/role_auth_userpass.yml @@ -0,0 +1,10 @@ +--- + +- include: ../shared/auth_backend.yml + vars: + auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates + auth_backend_path: userpass + auth_backend_type: userpass + when: inventory_hostname == groups.vault|first + +- include: create_roles.yml diff --git a/roles/vault/tasks/cluster/systemd.yml b/roles/vault/tasks/cluster/systemd.yml new file mode 100644 index 000000000..4d2a1da58 --- /dev/null +++ b/roles/vault/tasks/cluster/systemd.yml @@ -0,0 +1,45 @@ +--- + +- name: cluster/systemd | Ensure mount points exist prior to vault.service startup + file: + mode: 0750 + path: "{{ item }}" + state: directory + with_items: + - "{{ vault_config_dir }}" + - "{{ vault_log_dir }}" + - "{{ vault_secrets_dir }}" + - /var/lib/vault/ + +- name: cluster/systemd | Ensure the vault user has access to needed directories + file: + owner: vault + path: "{{ item }}" + recurse: true + with_items: + - "{{ vault_base_dir }}" + - "{{ vault_log_dir }}" + - /var/lib/vault + +- name: cluster/systemd | Copy down vault.service systemd file + template: + src: "{{ vault_deployment_type }}.service.j2" + dest: /etc/systemd/system/vault.service + backup: yes + register: vault_systemd_placement + +- name: cluster/systemd | Enable vault.service + systemd: + daemon_reload: true + enabled: yes + name: vault + state: started + +- name: cluster/systemd | Query local vault until service is up + uri: + url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health" + headers: "{{ vault_client_headers }}" + status_code: 200,429,500,501 + register: vault_health_check + until: vault_health_check|succeeded + retries: 10 diff --git a/roles/vault/tasks/cluster/unseal.yml b/roles/vault/tasks/cluster/unseal.yml new file mode 100644 index 000000000..2fbdbce56 --- /dev/null +++ b/roles/vault/tasks/cluster/unseal.yml @@ -0,0 +1,22 @@ +--- + +- name: cluster/unseal | Unseal Vault + uri: + url: "https://localhost:{{ vault_port }}/v1/sys/unseal" + headers: "{{ vault_headers }}" + method: POST + body_format: json + body: + key: "{{ item }}" + with_items: "{{ vault_unseal_keys|default([]) }}" + when: vault_is_sealed + +- name: cluster/unseal | Wait until server is ready + uri: + url: "https://localhost:{{ vault_port }}/v1/sys/health" + headers: "{{ vault_headers }}" + method: HEAD + status_code: 200, 429 + register: vault_node_ready + until: vault_node_ready|succeeded + retries: 5 diff --git a/roles/vault/tasks/main.yml b/roles/vault/tasks/main.yml new file mode 100644 index 000000000..f7414b74f --- /dev/null +++ b/roles/vault/tasks/main.yml @@ -0,0 +1,19 @@ +--- +# The Vault role is typically a two step process: +# 1. Bootstrap +# This starts a temporary Vault to generate certs for Vault itself. This +# includes a Root CA for the cluster, assuming one doesn't exist already. +# The temporary instance will remain running after Bootstrap, to provide a +# running Vault for the Etcd role to generate certs against. +# 2. Cluster +# Once Etcd is started, then the Cluster tasks can start up a long-term +# Vault cluster using Etcd as the backend. The same Root CA is mounted as +# used during step 1, allowing all certs to have the same chain of trust. + +## Bootstrap +- include: bootstrap/main.yml + when: vault_bootstrap | d() + +## Cluster +- include: cluster/main.yml + when: not vault_bootstrap | d() diff --git a/roles/vault/tasks/shared/auth_backend.yml b/roles/vault/tasks/shared/auth_backend.yml new file mode 100644 index 000000000..ad5b191c9 --- /dev/null +++ b/roles/vault/tasks/shared/auth_backend.yml @@ -0,0 +1,21 @@ +--- + +- name: shared/auth_backend | Test if the auth backend exists + uri: + url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}/tune" + headers: "{{ vault_headers }}" + validate_certs: false + ignore_errors: true + register: vault_auth_backend_check + +- name: shared/auth_backend | Add the cert auth backend if needed + uri: + url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}" + headers: "{{ vault_headers }}" + method: POST + body_format: json + body: + description: "{{ auth_backend_description|d('') }}" + type: "{{ auth_backend_type }}" + status_code: 204 + when: vault_auth_backend_check|failed diff --git a/roles/vault/tasks/shared/cert_auth_mount.yml b/roles/vault/tasks/shared/cert_auth_mount.yml new file mode 100644 index 000000000..9710aa7ca --- /dev/null +++ b/roles/vault/tasks/shared/cert_auth_mount.yml @@ -0,0 +1,21 @@ +--- + +- include: ../shared/mount.yml + vars: + mount_name: auth-pki + mount_options: + description: PKI mount to generate certs for the Cert Auth Backend + config: + default_lease_ttl: "{{ vault_default_lease_ttl }}" + max_lease_ttl: "{{ vault_max_lease_ttl }}" + type: pki + +- name: shared/auth_mount | Create a dummy role for issuing certs from auth-pki + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth-pki/roles/dummy" + headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + method: POST + body_format: json + body: + {'allow_any_name': true} + status_code: 204 diff --git a/roles/vault/tasks/shared/check_etcd.yml b/roles/vault/tasks/shared/check_etcd.yml new file mode 100644 index 000000000..83c8b29e9 --- /dev/null +++ b/roles/vault/tasks/shared/check_etcd.yml @@ -0,0 +1,19 @@ +--- + +- name: check_etcd | Check if etcd is up an reachable + uri: + url: "{{ vault_etcd_url }}/health" + validate_certs: no + failed_when: false + register: vault_etcd_health_check + +- name: check_etcd | Set fact based off the etcd_health_check response + set_fact: + vault_etcd_available: "{{ vault_etcd_health_check.get('json', {}).get('health')|bool }}" + +- name: check_etcd | Fail if etcd is not available and needed + fail: + msg: > + Unable to start Vault cluster! Etcd is not available at + {{ vault_etcd_url }} however it is needed by Vault as a backend. + when: vault_etcd_needed|d() and not vault_etcd_available diff --git a/roles/vault/tasks/shared/check_vault.yml b/roles/vault/tasks/shared/check_vault.yml new file mode 100644 index 000000000..8a81ae5f2 --- /dev/null +++ b/roles/vault/tasks/shared/check_vault.yml @@ -0,0 +1,31 @@ +--- + +# Stop temporary Vault if it's running (can linger if playbook fails out) +- name: stop vault-temp container + shell: docker stop {{ vault_temp_container_name }} || rkt stop {{ vault_temp_container_name }} + failed_when: false + register: vault_temp_stop + changed_when: vault_temp_stop|succeeded + +# Check if vault is reachable on the localhost +- name: check_vault | Attempt to pull local https Vault health + uri: + url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health" + headers: "{{ vault_client_headers }}" + status_code: 200,429,500,501 + validate_certs: no + failed_when: false + register: vault_local_service_health + +- name: check_vault | Set facts about local Vault health + set_fact: + vault_is_running: "{{ vault_local_service_health|succeeded }}" + vault_is_initialized: "{{ vault_local_service_health.get('json', {}).get('initialized', false) }}" + vault_is_sealed: "{{ vault_local_service_health.get('json', {}).get('sealed', true) }}" + #vault_in_standby: "{{ vault_local_service_health.get('json', {}).get('standby', true) }}" + #vault_run_version: "{{ vault_local_service_health.get('json', {}).get('version', '') }}" + +- name: check_vault | Set fact about the Vault cluster's initialization state + set_fact: + vault_cluster_is_initialized: "{{ vault_is_initialized or hostvars[item]['vault_is_initialized'] }}" + with_items: "{{ groups.vault }}" diff --git a/roles/vault/tasks/shared/config_ca.yml b/roles/vault/tasks/shared/config_ca.yml new file mode 100644 index 000000000..79c972b4d --- /dev/null +++ b/roles/vault/tasks/shared/config_ca.yml @@ -0,0 +1,30 @@ +--- + +- name: config_ca | Read root CA cert for Vault + command: "cat /etc/vault/ssl/{{ ca_name }}.pem" + register: vault_ca_cert_cat + +- name: config_ca | Pull current CA cert from Vault + uri: + url: "{{ vault_leader_url }}/v1/{{ mount_name }}/ca/pem" + headers: "{{ vault_headers }}" + return_content: true + status_code: 200,204 + validate_certs: no + register: vault_pull_current_ca + +- name: config_ca | Read root CA key for Vault + command: "cat /etc/vault/ssl/{{ ca_name }}-key.pem" + register: vault_ca_key_cat + when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.content.strip() + +- name: config_ca | Configure pki mount to use the found root CA cert and key + uri: + url: "{{ vault_leader_url }}/v1/{{ mount_name }}/config/ca" + headers: "{{ vault_headers }}" + method: POST + body_format: json + body: + pem_bundle: "{{ vault_ca_cert_cat.stdout + '\n' + vault_ca_key_cat.stdout }}" + status_code: 204 + when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.get("content","").strip() diff --git a/roles/vault/tasks/shared/create_role.yml b/roles/vault/tasks/shared/create_role.yml new file mode 100644 index 000000000..c39fafe8c --- /dev/null +++ b/roles/vault/tasks/shared/create_role.yml @@ -0,0 +1,74 @@ +--- + +# The JSON inside JSON here is intentional (Vault API wants it) +- name: create_role | Create a policy for the new role allowing issuing + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/sys/policy/{{ create_role_name }}" + headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + method: PUT + body_format: json + body: + rules: >- + {%- if create_role_policy_rules|d("default") == "default" -%} + {{ + { 'path': { + 'pki/issue/' + create_role_name: {'policy': 'write'}, + 'pki/roles/' + create_role_name: {'policy': 'read'} + }} | to_json + '\n' + }} + {%- else -%} + {{ create_role_policy_rules | to_json + '\n' }} + {%- endif -%} + status_code: 204 + when: inventory_hostname == groups[create_role_group]|first + +- name: create_role | Create the new role in the pki mount + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/pki/roles/{{ create_role_name }}" + headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + method: POST + body_format: json + body: >- + {%- if create_role_options|d("default") == "default" -%} + {'allow_any_name': true} + {%- else -%} + {{ create_role_options }} + {%- endif -%} + status_code: 204 + when: inventory_hostname == groups[create_role_group]|first + +## Cert based auth method + +- include: gen_cert.yml + vars: + gen_cert_copy_ca: true + gen_cert_hosts: "{{ groups[create_role_group] }}" + gen_cert_mount: "auth-pki" + gen_cert_path: "{{ vault_roles_dir }}/{{ create_role_name }}/issuer.pem" + gen_cert_vault_headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + gen_cert_vault_role: "dummy" + gen_cert_vault_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + when: vault_role_auth_method == "cert" and inventory_hostname in groups[create_role_group] + +- name: create_role | Insert the auth-pki CA as the authenticating CA for that role + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/cert/certs/{{ create_role_name }}" + headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + method: POST + body_format: json + body: + certificate: "{{ hostvars[groups[create_role_group]|first]['gen_cert_result']['json']['data']['issuing_ca'] }}" + policies: "{{ create_role_name }}" + status_code: 204 + when: vault_role_auth_method == "cert" and inventory_hostname == groups[create_role_group]|first + +## Userpass based auth method + +- include: gen_userpass.yml + vars: + gen_userpass_group: "{{ create_role_group }}" + gen_userpass_password: "{{ create_role_password|d(''|to_uuid) }}" + gen_userpass_policies: "{{ create_role_name }}" + gen_userpass_role: "{{ create_role_name }}" + gen_userpass_username: "{{ create_role_name }}" + when: vault_role_auth_method == "userpass" and inventory_hostname in groups[create_role_group] diff --git a/roles/vault/tasks/shared/find_leader.yml b/roles/vault/tasks/shared/find_leader.yml new file mode 100644 index 000000000..0c1a697d3 --- /dev/null +++ b/roles/vault/tasks/shared/find_leader.yml @@ -0,0 +1,17 @@ +--- + +- name: find_leader | Find the current http Vault leader + uri: + url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health" + headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + method: HEAD + status_code: 200,429 + register: vault_leader_check + until: "vault_leader_check|succeeded" + retries: 10 + +- name: find_leader | Set fact for current http leader + set_fact: + vault_leader_url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://{{ item }}:{{ vault_port }}" + with_items: "{{ groups.vault }}" + when: "hostvars[item]['vault_leader_check'].get('status') == 200" diff --git a/roles/vault/tasks/shared/gen_userpass.yml b/roles/vault/tasks/shared/gen_userpass.yml new file mode 100644 index 000000000..ab3d171b8 --- /dev/null +++ b/roles/vault/tasks/shared/gen_userpass.yml @@ -0,0 +1,30 @@ +--- + +- name: shared/gen_userpass | Create the Username/Password combo for the role + uri: + url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/users/{{ gen_userpass_username }}" + headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" + method: POST + body_format: json + body: + username: "{{ gen_userpass_username }}" + password: "{{ gen_userpass_password }}" + policies: "{{ gen_userpass_role }}" + status_code: 204 + when: inventory_hostname == groups[gen_userpass_group]|first + +- name: shared/gen_userpass | Ensure destination directory exists + file: + path: "{{ vault_roles_dir }}/{{ gen_userpass_role }}" + state: directory + when: inventory_hostname in groups[gen_userpass_group] + +- name: shared/gen_userpass | Copy credentials to all hosts in the group + copy: + content: > + {{ + {'username': gen_userpass_username, + 'password': gen_userpass_password} | to_nice_json(indent=4) + }} + dest: "{{ vault_roles_dir }}/{{ gen_userpass_role }}/userpass" + when: inventory_hostname in groups[gen_userpass_group] diff --git a/roles/vault/tasks/shared/issue_cert.yml b/roles/vault/tasks/shared/issue_cert.yml new file mode 100644 index 000000000..0733e86a0 --- /dev/null +++ b/roles/vault/tasks/shared/issue_cert.yml @@ -0,0 +1,66 @@ +--- + +# This could be a role or custom module + +# Vars: +# issue_cert_alt_name: Requested Subject Alternative Names, in a list. +# issue_cert_common_name: Common Name included in the cert +# issue_cert_dir_mode: Mode of the placed cert directory +# issue_cert_file_group: Group of the placed cert file and directory +# issue_cert_file_mode: Mode of the placed cert file +# issue_cert_file_owner: Owner of the placed cert file and directory +# issue_cert_format: Format for returned data. Can be pem, der, or pem_bundle +# issue_cert_headers: Headers passed into the issue request +# issue_cert_hosts: List of hosts to distribute the cert to +# issue_cert_ip_sans: Requested IP Subject Alternative Names, in a list +# issue_cert_mount: Mount point in Vault to make the request to +# issue_cert_path: Full path to the cert, include its name +# issue_cert_role: The Vault role to issue the cert with +# issue_cert_url: Url to reach Vault, including protocol and port + +- name: issue_cert | Ensure target directory exists + file: + path: "{{ issue_cert_path | dirname }}" + state: directory + group: "{{ issue_cert_file_group | d('root' )}}" + mode: "{{ issue_cert_dir_mode | d('0755') }}" + owner: "{{ issue_cert_file_owner | d('root') }}" + +- name: issue_cert | Generate the cert + uri: + url: "{{ issue_cert_url }}/v1/{{ issue_cert_mount|d('pki') }}/issue/{{ issue_cert_role }}" + headers: "{{ issue_cert_headers }}" + method: POST + body_format: json + body: + alt_names: "{{ issue_cert_alt_names | d([]) | join(',') }}" + common_name: "{{ issue_cert_common_name | d(issue_cert_path.rsplit('/', 1)[1].rsplit('.', 1)[0]) }}" + format: "{{ issue_cert_format | d('pem') }}" + ip_sans: "{{ issue_cert_ip_sans | default([]) | join(',') }}" + register: issue_cert_result + when: inventory_hostname == issue_cert_hosts|first + +- name: issue_cert | Copy the cert to all hosts + copy: + content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['certificate'] }}" + dest: "{{ issue_cert_path }}" + group: "{{ issue_cert_file_group | d('root' )}}" + mode: "{{ issue_cert_file_mode | d('0644') }}" + owner: "{{ issue_cert_file_owner | d('root') }}" + +- name: issue_cert | Copy the key to all hosts + copy: + content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['private_key'] }}" + dest: "{{ issue_cert_path.rsplit('.', 1)|first }}-key.{{ issue_cert_path.rsplit('.', 1)|last }}" + group: "{{ issue_cert_file_group | d('root' )}}" + mode: "{{ issue_cert_file_mode | d('0640') }}" + owner: "{{ issue_cert_file_owner | d('root') }}" + +- name: issue_cert | Copy issuing CA cert + copy: + content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['issuing_ca'] }}" + dest: "{{ issue_cert_path | dirname }}/ca.pem" + group: "{{ issue_cert_file_group | d('root' )}}" + mode: "{{ issue_cert_file_mode | d('0644') }}" + owner: "{{ issue_cert_file_owner | d('root') }}" + when: issue_cert_copy_ca|default(false) diff --git a/roles/vault/tasks/shared/mount.yml b/roles/vault/tasks/shared/mount.yml new file mode 100644 index 000000000..b98b45c57 --- /dev/null +++ b/roles/vault/tasks/shared/mount.yml @@ -0,0 +1,18 @@ +--- + +- name: shared/mount | Test if PKI mount exists + uri: + url: "{{ vault_leader_url }}/v1/sys/mounts/{{ mount_name }}/tune" + headers: "{{ vault_headers }}" + ignore_errors: true + register: vault_pki_mount_check + +- name: shared/mount | Mount PKI mount if needed + uri: + url: "{{ vault_leader_url }}/v1/sys/mounts/{{ mount_name }}" + headers: "{{ vault_headers }}" + method: POST + body_format: json + body: "{{ mount_options|d() }}" + status_code: 204 + when: vault_pki_mount_check|failed diff --git a/roles/vault/tasks/shared/pki_mount.yml b/roles/vault/tasks/shared/pki_mount.yml new file mode 100644 index 000000000..31faef434 --- /dev/null +++ b/roles/vault/tasks/shared/pki_mount.yml @@ -0,0 +1,11 @@ +--- + +- include: mount.yml + vars: + mount_name: pki + mount_options: + config: + default_lease_ttl: "{{ vault_default_lease_ttl }}" + max_lease_ttl: "{{ vault_max_lease_ttl }}" + description: The default PKI mount for Kubernetes + type: pki diff --git a/roles/vault/tasks/shared/sync.yml b/roles/vault/tasks/shared/sync.yml new file mode 100644 index 000000000..02818b5f1 --- /dev/null +++ b/roles/vault/tasks/shared/sync.yml @@ -0,0 +1,47 @@ +--- + +- name: "sync_file | Cat the file" + command: "cat {{ sync_file_path }}" + register: sync_file_cat + when: inventory_hostname == sync_file_srcs|first + +- name: "sync_file | Cat the key file" + command: "cat {{ sync_file_key_path }}" + register: sync_file_key_cat + when: sync_file_is_cert|d() and inventory_hostname == sync_file_srcs|first + +- name: "sync_file | Set facts for file contents" + set_fact: + sync_file_contents: "{{ hostvars[sync_file_srcs|first]['sync_file_cat']['stdout'] }}" + +- name: "sync_file | Set fact for key contents" + set_fact: + sync_file_key_contents: "{{ hostvars[sync_file_srcs|first]['sync_file_key_cat']['stdout'] }}" + when: sync_file_is_cert|d() + +- name: "sync_file | Ensure the directory exists" + file: + group: "{{ sync_file_group|d('root') }}" + mode: "{{ sync_file_dir_mode|default('0750') }}" + owner: "{{ sync_file_owner|d('root') }}" + path: "{{ sync_file_dir }}" + state: directory + when: inventory_hostname not in sync_file_srcs + +- name: "sync_file | Copy the file to hosts that don't have it" + copy: + content: "{{ sync_file_contents }}" + dest: "{{ sync_file_path }}" + group: "{{ sync_file_group|d('root') }}" + mode: "{{ sync_file_mode|default('0640') }}" + owner: "{{ sync_file_owner|d('root') }}" + when: inventory_hostname not in sync_file_srcs + +- name: "sync_file | Copy the key file to hosts that don't have it" + copy: + content: "{{ sync_file_key_contents }}" + dest: "{{ sync_file_key_path }}" + group: "{{ sync_file_group|d('root') }}" + mode: "{{ sync_file_mode|default('0640') }}" + owner: "{{ sync_file_owner|d('root') }}" + when: sync_file_is_cert|d() and inventory_hostname not in sync_file_srcs diff --git a/roles/vault/tasks/shared/sync_auth_certs.yml b/roles/vault/tasks/shared/sync_auth_certs.yml new file mode 100644 index 000000000..7bd9c4b49 --- /dev/null +++ b/roles/vault/tasks/shared/sync_auth_certs.yml @@ -0,0 +1,17 @@ +--- + +- include: sync_file.yml + vars: + sync_file: "auth-ca.pem" + sync_file_dir: "{{ vault_cert_dir }}" + sync_file_hosts: "{{ groups.vault }}" + sync_file_is_cert: true + +- name: shared/sync_auth_certs | Set facts for vault sync_file results + set_fact: + vault_auth_ca_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}" + + +- name: shared/sync_auth_certs | Unset sync_file_results after auth-ca.pem sync + set_fact: + sync_file_results: [] diff --git a/roles/vault/tasks/shared/sync_file.yml b/roles/vault/tasks/shared/sync_file.yml new file mode 100644 index 000000000..484d4aced --- /dev/null +++ b/roles/vault/tasks/shared/sync_file.yml @@ -0,0 +1,97 @@ +--- + +# NOTE: This should be a role (or custom module), but currently include_role is too buggy to use + +- name: "sync_file | Set facts for directory and file when sync_file_path is defined" + set_fact: + sync_file_dir: "{{ sync_file_path | dirname }}" + sync_file: "{{ sync_file_path | basename }}" + when: sync_file_path is defined and sync_file_path != '' + +- name: "sync_file | Set fact for sync_file_path when undefined" + set_fact: + sync_file_path: "{{ (sync_file_dir, sync_file)|join('/') }}" + when: sync_file_path is not defined or sync_file_path == '' + +- name: "sync_file | Set fact for key path name" + set_fact: + sync_file_key_path: "{{ sync_file_path.rsplit('.', 1)|first + '-key.' + sync_file_path.rsplit('.', 1)|last }}" + when: >- + sync_file_is_cert|d() and (sync_file_key_path is not defined or sync_file_key_path == '') + +- name: "sync_file | Check if file exists" + stat: + path: "{{ sync_file_path }}" + register: sync_file_stat + +- name: "sync_file | Check if key file exists" + stat: + path: "{{ sync_file_key_path }}" + register: sync_file_key_stat + when: sync_file_is_cert|d() + +- name: "sync_file | Combine all possible file sync sources" + set_fact: + sync_file_srcs: "{{ sync_file_srcs|default([]) + [host_item] }}" + with_items: "{{ sync_file_hosts | unique }}" + loop_control: + loop_var: host_item + when: hostvars[host_item]["sync_file_stat"]["stat"]["exists"]|bool + +- name: "sync_file | Combine all possible key file sync sources" + set_fact: + sync_file_key_srcs: "{{ sync_file_key_srcs|default([]) + [host_item] }}" + with_items: "{{ sync_file_hosts | unique }}" + loop_control: + loop_var: host_item + when: sync_file_is_cert|d() and hostvars[host_item]["sync_file_key_stat"]["stat"]["exists"]|bool + +- name: "sync_file | Remove sync sources with files that do not match sync_file_srcs|first" + set_fact: + _: "{% if inventory_hostname in sync_file_srcs %}{{ sync_file_srcs.remove(inventory_hostname) }}{% endif %}" + when: >- + sync_file_srcs|d([])|length > 1 and + inventory_hostname != sync_file_srcs|first and + sync_file_stat.stat.get("checksum") != hostvars[sync_file_srcs|first]["sync_file_stat"]["stat"]["checksum"] + +- name: "sync_file | Remove sync sources with keys that do not match sync_file_srcs|first" + set_fact: + _: "{% if inventory_hostname in sync_file_srcs %}{{ sync_file_srcs.remove(inventory_hostname) }}{% endif %}" + when: >- + sync_file_is_cert|d() and + sync_file_key_srcs|d([])|length > 1 and + inventory_hostname != sync_file_key_srcs|first and + sync_file_key_stat.stat.checksum != hostvars[sync_file_srcs|first]["sync_file_key_stat"]["stat"]["checksum"] + +- name: "sync_file | Consolidate file and key sources" + set_fact: + sync_file_srcs: "{{ sync_file_srcs|d([]) | intersect(sync_file_key_srcs) }}" + when: sync_file_is_cert|d() + +- name: "sync_file | Set facts for situations where sync is not needed" + set_fact: + sync_file_no_srcs: "{{ true if sync_file_srcs|d([])|length == 0 else false }}" + sync_file_unneeded: "{{ true if sync_file_srcs|d([])|length == sync_file_hosts|length else false }}" + +- name: "sync_file | Set sync_file_result fact" + set_fact: + sync_file_result: + no_srcs: "{{ sync_file_no_srcs }}" + path: "{{ sync_file_path }}" + sync_unneeded: "{{ sync_file_unneeded }}" + +- name: "sync_file | Update sync_file_results fact" + set_fact: + sync_file_results: "{{ sync_file_results|default([]) + [sync_file_result] }}" + +- include: sync.yml + when: not (sync_file_no_srcs or sync_file_unneeded) + +- name: "Unset local vars to avoid variable bleed into next iteration" + set_fact: + sync_file: '' + sync_file_dir: '' + sync_file_key_path: '' + sync_file_key_srcs: [] + sync_file_path: '' + sync_file_srcs: [] diff --git a/roles/vault/templates/docker.service.j2 b/roles/vault/templates/docker.service.j2 new file mode 100644 index 000000000..c355b7f01 --- /dev/null +++ b/roles/vault/templates/docker.service.j2 @@ -0,0 +1,32 @@ +[Unit] +Description=hashicorp vault on docker +Documentation=https://github.com/hashicorp/vault +Wants=docker.socket +After=docker.service + +[Service] +User=root +Restart=always +RestartSec=15s +TimeoutStartSec=5 +LimitNOFILE=10000 +ExecReload={{ docker_bin_dir }}/docker restart {{ vault_container_name }} +ExecStop={{ docker_bin_dir }}/docker stop {{ vault_container_name }} +ExecStartPre=-{{ docker_bin_dir }}/docker rm -f {{ vault_container_name }} +# Container has the following internal mount points: +# /vault/file/ # File backend storage location +# /vault/logs/ # Log files +ExecStart={{ docker_bin_dir }}/docker run \ +--name {{ vault_container_name }} --net=host \ +--cap-add=IPC_LOCK \ +-v {{ vault_cert_dir }}:{{ vault_cert_dir }} \ +-v {{ vault_config_dir }}:{{ vault_config_dir }} \ +-v {{ vault_log_dir }}:/vault/logs \ +-v {{ vault_roles_dir }}:{{ vault_roles_dir }} \ +-v {{ vault_secrets_dir }}:{{ vault_secrets_dir }} \ +--entrypoint=vault \ +{{ vault_image_repo }}:{{ vault_image_tag }} \ +server --config={{ vault_config_dir }}/config.json + +[Install] +WantedBy=multi-user.target diff --git a/roles/vault/templates/host.service.j2 b/roles/vault/templates/host.service.j2 new file mode 100644 index 000000000..11bce2f29 --- /dev/null +++ b/roles/vault/templates/host.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=vault +After=network.target + +[Service] +AmbientCapabilities=CAP_IPC_LOCK +ExecStart=/usr/bin/vault server --config={{ vault_config_dir }}/config.json +LimitNOFILE=40000 +NotifyAccess=all +Restart=always +RestartSec=10s +User={{ vault_adduser_vars.name }} + +[Install] +WantedBy=multi-user.target diff --git a/roles/vault/templates/rkt.service.j2 b/roles/vault/templates/rkt.service.j2 new file mode 100644 index 000000000..42b9458ac --- /dev/null +++ b/roles/vault/templates/rkt.service.j2 @@ -0,0 +1,33 @@ +[Unit] +Description=hashicorp vault on rkt +Documentation=https://github.com/hashicorp/vault +Wants=network.target + +[Service] +User=root +Restart=on-failure +RestartSec=10s +TimeoutStartSec=5 +LimitNOFILE=40000 +# Container has the following internal mount points: +# /vault/file/ # File backend storage location +# /vault/logs/ # Log files +ExecStart=/usr/bin/rkt run \ +--insecure-options=image \ +--volume=volume-vault-file,kind=host,source=/var/lib/vault \ +--volume=volume-vault-logs,kind=host,source={{ vault_log_dir }} \ +--volume=vault-cert-dir,kind=host,source={{ vault_cert_dir }} \ +--mount=volume=vault-cert-dir,target={{ vault_cert_dir }} \ +--volume=vault-conf-dir,kind=host,source={{ vault_config_dir }} \ +--mount=volume=vault-conf-dir,target={{ vault_config_dir }} \ +--volume=vault-secrets-dir,kind=host,source={{ vault_secrets_dir }} \ +--mount=volume=vault-secrets-dir,target={{ vault_secrets_dir }} \ +--volume=vault-roles-dir,kind=host,source={{ vault_roles_dir }} \ +--mount=volume=vault-roles-dir,target={{ vault_roles_dir }} \ +docker://{{ vault_image_repo }}:{{ vault_image_tag }} \ +--name={{ vault_container_name }} --net=host \ +--caps-retain=CAP_IPC_LOCK \ +--exec vault -- server --config={{ vault_config_dir }}/config.json + +[Install] +WantedBy=multi-user.target diff --git a/tests/templates/inventory-aws.j2 b/tests/templates/inventory-aws.j2 index e0771a94e..ee89bb5a4 100644 --- a/tests/templates/inventory-aws.j2 +++ b/tests/templates/inventory-aws.j2 @@ -15,6 +15,10 @@ node3 node1 node2 +[vault] +node1 +node2 + [k8s-cluster:children] kube-node kube-master diff --git a/tests/templates/inventory-gce.j2 b/tests/templates/inventory-gce.j2 index 3ede8f529..015bdb6a4 100644 --- a/tests/templates/inventory-gce.j2 +++ b/tests/templates/inventory-gce.j2 @@ -13,6 +13,9 @@ node2 [etcd] node3 + +[vault] +node3 {% elif mode is defined and mode == "ha" %} [kube-master] node1 @@ -24,6 +27,10 @@ node3 [etcd] node2 node3 + +[vault] +node2 +node3 {% else %} [kube-master] node1 @@ -33,6 +40,9 @@ node2 [etcd] node1 + +[vault] +node1 {% endif %} [k8s-cluster:children]