diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e03e64017..3e31ce1bd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -304,6 +304,10 @@ before_script: # stage: deploy-part1 MOVED_TO_GROUP_VARS: "true" +.coreos_vault_upgrade_variables: &coreos_vault_upgrade_variables +# stage: deploy-part1 + UPGRADE_TEST: "basic" + .ubuntu_flannel_variables: &ubuntu_flannel_variables # stage: deploy-special MOVED_TO_GROUP_VARS: "true" @@ -560,7 +564,7 @@ gce_rhel7-canal-sep: <<: *rhel7_canal_sep_variables when: manual except: ['triggers'] - only: ['master', /^pr-.*$/,] + only: ['master', /^pr-.*$/] gce_rhel7-canal-sep-triggers: stage: deploy-part2 @@ -638,6 +642,17 @@ gce_ubuntu-vault-sep: except: ['triggers'] only: ['master', /^pr-.*$/] +gce_coreos-vault-upgrade: + stage: deploy-part2 + <<: *job + <<: *gce + variables: + <<: *gce_variables + <<: *coreos_vault_upgrade_variables + when: manual + except: ['triggers'] + only: ['master', /^pr-.*$/] + gce_ubuntu-flannel-sep: stage: deploy-special <<: *job diff --git a/requirements.txt b/requirements.txt index 80ffd74ed..01ff9f23c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ -pbr>=1.6 ansible>=2.4.0 -netaddr jinja2>=2.9.6 +netaddr +pbr>=1.6 +ansible-modules-hashivault>=3.9.4 +hvac diff --git a/roles/bootstrap-os/tasks/bootstrap-ubuntu.yml b/roles/bootstrap-os/tasks/bootstrap-ubuntu.yml index 37c327f6c..baf3a433d 100644 --- a/roles/bootstrap-os/tasks/bootstrap-ubuntu.yml +++ b/roles/bootstrap-os/tasks/bootstrap-ubuntu.yml @@ -8,6 +8,7 @@ changed_when: false with_items: - python + - python-apt - pip - dbus-daemon tags: diff --git a/roles/dnsmasq/tasks/main.yml b/roles/dnsmasq/tasks/main.yml index 831330175..8fbd547b4 100644 --- a/roles/dnsmasq/tasks/main.yml +++ b/roles/dnsmasq/tasks/main.yml @@ -3,15 +3,11 @@ file: path: /etc/dnsmasq.d state: directory - tags: - - bootstrap-os - name: ensure dnsmasq.d-available directory exists file: path: /etc/dnsmasq.d-available state: directory - tags: - - bootstrap-os - name: check system nameservers shell: awk '/^nameserver/ {print $NF}' /etc/resolv.conf diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 10b39967b..41b5a098a 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -37,7 +37,7 @@ calico_rr_version: "v0.4.2" flannel_version: "v0.10.0" flannel_cni_version: "v0.3.0" istio_version: "0.2.6" -vault_version: 0.8.1 +vault_version: 0.10.1 weave_version: 2.3.0 pod_infra_version: 3.0 contiv_version: 1.1.7 diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index 7b7a1fc5a..4d3938341 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -50,7 +50,7 @@ etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr', etcd_compaction_retention: "8" -etcd_vault_mount_path: etcd +etcd_vault_mount_path: "/etcd" # Force clients like etcdctl to use TLS certs (different than peer security) etcd_secure_client: true diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 52b04be50..a050be1da 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -100,3 +100,6 @@ kube_encrypt_secret_data: false kube_encrypt_token: "{{ lookup('password', inventory_dir + '/credentials/kube_encrypt_token.creds length=32 chars=ascii_letters,digits') }}" # Must be either: aescbc, secretbox or aesgcm kube_encryption_algorithm: "aescbc" + +# You may want to use ca.pem depending on your situation +kube_front_proxy_ca: "front-proxy-ca.pem" diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index e3efa73df..b638ff457 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -117,8 +117,13 @@ spec: - --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kube_version | version_compare('v1.9', '>=') %} - - --requestheader-client-ca-file={{ kube_cert_dir }}/front-proxy-ca.pem + - --requestheader-client-ca-file={{ kube_cert_dir }}/{{ kube_front_proxy_ca }} +{# FIXME(mattymo): Vault certs do not work with front-proxy-client #} +{% if cert_management == "vault" %} + - --requestheader-allowed-names= +{% else %} - --requestheader-allowed-names=front-proxy-client +{% endif %} - --requestheader-extra-headers-prefix=X-Remote-Extra- - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index f01f661ba..c85de5ad0 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -128,13 +128,13 @@ vsphere_public_network: "{{ lookup('env', 'VSPHERE_PUBLIC_NETWORK')|default('') ## When azure is used, you need to also set the following variables. ## see docs/azure.md for details on how to get these values -#azure_tenant_id: -#azure_subscription_id: -#azure_aad_client_id: -#azure_aad_client_secret: -#azure_resource_group: -#azure_location: -#azure_subnet_name: -#azure_security_group_name: -#azure_vnet_name: -#azure_route_table_name: +# azure_tenant_id: +# azure_subscription_id: +# azure_aad_client_id: +# azure_aad_client_secret: +# azure_resource_group: +# azure_location: +# azure_subnet_name: +# azure_security_group_name: +# azure_vnet_name: +# azure_route_table_name: diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index a945e715e..a514aa079 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -13,6 +13,5 @@ - import_tasks: set_resolv_facts.yml tags: - - bootstrap-os - resolvconf - facts diff --git a/roles/kubernetes/secrets/defaults/main.yml b/roles/kubernetes/secrets/defaults/main.yml index cda85eeb2..34c42bc20 100644 --- a/roles/kubernetes/secrets/defaults/main.yml +++ b/roles/kubernetes/secrets/defaults/main.yml @@ -1,4 +1,3 @@ --- kube_cert_group: kube-cert -kube_vault_mount_path: kube -front_proxy_vault_mount_path: front-proxy +kube_vault_mount_path: "/kube" diff --git a/roles/kubernetes/secrets/tasks/gen_certs_script.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml index 1e9211b34..b77275b49 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs_script.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml @@ -12,7 +12,6 @@ - k8s-secrets - kube-controller-manager - kube-apiserver - - bootstrap-os - apps - network - master @@ -28,7 +27,6 @@ when: gen_certs|default(false) tags: - k8s-secrets - - bootstrap-os - name: Gen_certs | write openssl config template: diff --git a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml index 8c9d12384..88db2f5a4 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml @@ -127,7 +127,7 @@ issue_cert_path: "{{ item }}" issue_cert_role: front-proxy-client issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" - issue_cert_mount_path: "{{ front_proxy_vault_mount_path }}" + issue_cert_mount_path: "{{ kube_vault_mount_path }}" with_items: "{{ kube_front_proxy_clients_certs_needed|d([]) }}" when: inventory_hostname in groups['kube-master'] notify: set secret_changed diff --git a/roles/kubernetes/secrets/tasks/main.yml b/roles/kubernetes/secrets/tasks/main.yml index 5d20a755a..52fedae5b 100644 --- a/roles/kubernetes/secrets/tasks/main.yml +++ b/roles/kubernetes/secrets/tasks/main.yml @@ -41,7 +41,6 @@ - k8s-secrets - kube-controller-manager - kube-apiserver - - bootstrap-os - apps - network - master @@ -57,7 +56,6 @@ when: gen_certs|default(false) or gen_tokens|default(false) tags: - k8s-secrets - - bootstrap-os - name: "Get_tokens | Make sure the tokens directory exits (on {{groups['kube-master'][0]}})" file: diff --git a/roles/vault/defaults/main.yml b/roles/vault/defaults/main.yml index f19c73438..b194ff18c 100644 --- a/roles/vault/defaults/main.yml +++ b/roles/vault/defaults/main.yml @@ -16,10 +16,11 @@ vault_cert_dir: "{{ vault_base_dir }}/ssl" vault_config_dir: "{{ vault_base_dir }}/config" vault_roles_dir: "{{ vault_base_dir }}/roles" vault_secrets_dir: "{{ vault_base_dir }}/secrets" +vault_lib_dir: "/var/lib/vault" vault_log_dir: "/var/log/vault" -vault_version: 0.8.1 -vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188 +vault_version: 0.10.1 +vault_binary_checksum: 66f0f1b0b221d664dd5913f8697409d7401df4bb2a19c7277e8fbad152063fae vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_amd64.zip" vault_download_vars: container: "{{ vault_deployment_type != 'host' }}" @@ -64,10 +65,10 @@ vault_config: etcd: address: "{{ vault_etcd_url }}" ha_enabled: "true" - redirect_addr: "https://{{ ansible_default_ipv4.address }}:{{ vault_port }}" - tls_ca_file: "{{ vault_etcd_cert_dir }}/ca.pem" - tls_cert_file: "{{ vault_etcd_cert_dir}}/node-{{ inventory_hostname }}.pem" - tls_key_file: "{{ vault_etcd_cert_dir}}/node-{{ inventory_hostname }}-key.pem" + redirect_addr: "https://{{ inventory_hostname }}:{{ vault_port }}" + tls_ca_file: "{{ etcd_cert_dir }}/ca.pem" + tls_cert_file: "{{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem" + tls_key_file: "{{ etcd_cert_dir}}/node-{{ inventory_hostname }}-key.pem" cluster_name: "kubernetes-vault" default_lease_ttl: "{{ vault_default_lease_ttl }}" max_lease_ttl: "{{ vault_max_lease_ttl }}" @@ -80,6 +81,8 @@ vault_config: vault_secret_shares: 1 vault_secret_threshold: 1 +vault_successful_http_codes: ["200", "429", "500", "501", "503"] + vault_ca_options: vault: common_name: vault @@ -97,20 +100,29 @@ vault_ca_options: format: pem ttl: "{{ vault_max_lease_ttl }}" exclude_cn_from_sans: true - front_proxy: - common_name: front-proxy - format: pem - ttl: "{{ vault_max_lease_ttl }}" - exclude_cn_from_sans: true vault_client_headers: Accept: "application/json" Content-Type: "application/json" -vault_etcd_cert_dir: /etc/ssl/etcd/ssl -vault_kube_cert_dir: /etc/kubernetes/ssl +etcd_cert_dir: /etc/ssl/etcd/ssl +kube_cert_dir: /etc/kubernetes/ssl vault_pki_mounts: + userpass: + name: userpass + default_lease_ttl: "{{ vault_default_lease_ttl }}" + max_lease_ttl: "{{ vault_max_lease_ttl }}" + description: "Userpass" + cert_dir: "{{ vault_cert_dir }}" + roles: + - name: userpass + group: userpass + password: "{{ lookup('password', inventory_dir + '/credentials/vault/userpass.creds length=15') }}" + policy_rules: default + role_options: + allow_any_name: true + vault: name: vault default_lease_ttl: "{{ vault_default_lease_ttl }}" @@ -122,13 +134,14 @@ vault_pki_mounts: group: vault password: "{{ lookup('password', inventory_dir + '/credentials/vault/vault.creds length=15') }}" policy_rules: default - role_options: default + role_options: + allow_any_name: true etcd: name: etcd default_lease_ttl: "{{ vault_default_lease_ttl }}" max_lease_ttl: "{{ vault_max_lease_ttl }}" description: "Etcd Root CA" - cert_dir: "{{ vault_etcd_cert_dir }}" + cert_dir: "{{ etcd_cert_dir }}" roles: - name: etcd group: etcd @@ -143,7 +156,7 @@ vault_pki_mounts: default_lease_ttl: "{{ vault_default_lease_ttl }}" max_lease_ttl: "{{ vault_max_lease_ttl }}" description: "Kubernetes Root CA" - cert_dir: "{{ vault_kube_cert_dir }}" + cert_dir: "{{ kube_cert_dir }}" roles: - name: kube-master group: kube-master @@ -153,6 +166,14 @@ vault_pki_mounts: allow_any_name: true enforce_hostnames: false organization: "system:masters" + - name: front-proxy-client + group: kube-master + password: "{{ lookup('password', inventory_dir + '/credentials/vault/kube-proxy.creds length=15') }}" + policy_rules: default + role_options: + allow_any_name: true + enforce_hostnames: false + organization: "system:front-proxy-client" - name: kube-node group: k8s-cluster password: "{{ lookup('password', inventory_dir + '/credentials/vault/kube-node.creds length=15') }}" @@ -169,18 +190,3 @@ vault_pki_mounts: allow_any_name: true enforce_hostnames: false organization: "system:node-proxier" - front_proxy: - name: front-proxy - default_lease_ttl: "{{ vault_default_lease_ttl }}" - max_lease_ttl: "{{ vault_max_lease_ttl }}" - description: "Kubernetes Front Proxy CA" - cert_dir: "{{ vault_kube_cert_dir }}" - roles: - - name: front-proxy-client - group: k8s-cluster - password: "{{ lookup('password', inventory_dir + '/credentials/vault/front-proxy-client.creds length=15') }}" - policy_rules: default - role_options: - allow_any_name: true - enforce_hostnames: false - organization: "system:front-proxy" \ No newline at end of file diff --git a/roles/vault/handlers/main.yml b/roles/vault/handlers/main.yml new file mode 100644 index 000000000..55d6d592f --- /dev/null +++ b/roles/vault/handlers/main.yml @@ -0,0 +1,50 @@ +--- +- name: restart vault + command: /bin/true + notify: + - restart vault service + - set facts about local Vault health + - unseal vault + +- name: wait for vault up + uri: + url: "{{ vault_leader_url | default('https://localhost:8200') }}/v1/sys/health" + headers: "{{ vault_client_headers }}" + status_code: "{{ vault_successful_http_codes | join(',') }}" + register: vault_health_check + until: vault_health_check|succeeded + retries: 10 + delay: "{{ retry_stagger | random + 3 }}" + run_once: yes + notify: set facts about local Vault health + +- name: wait for vault up nowait + uri: + url: "{{ vault_leader_url | default('https://localhost:8200') }}/v1/sys/health" + headers: "{{ vault_client_headers }}" + status_code: "{{ vault_successful_http_codes | join(',') }}" + register: vault_health_check + run_once: yes + failed_when: false + notify: set facts about local Vault health + +- name: set facts about local Vault health + set_fact: + vault_is_running: "{{ vault_health_check.get('status', '-1') in vault_successful_http_codes }}" + vault_cluster_is_initialized: "{{ vault_health_check.get('json', {}).get('initialized', false) }}" + vault_is_sealed: "{{ vault_health_check.get('json', {}).get('sealed', true) }}" + +- name: restart vault service + systemd: + daemon_reload: true + enabled: yes + name: vault + state: restarted + +- name: unseal vault + hashivault_unseal: + url: "{{ vault_leader_url | default('https://localhost:8200') }}" + token: "{{ vault_root_token }}" + ca_cert: "{{ vault_cert_dir }}/ca.pem" + keys: "{{ item }}" + with_items: "{{ vault_unseal_keys|default([]) }}" diff --git a/roles/vault/tasks/bootstrap/create_mounts.yml b/roles/vault/tasks/bootstrap/create_mounts.yml index 0c82990e6..ad0fab915 100644 --- a/roles/vault/tasks/bootstrap/create_mounts.yml +++ b/roles/vault/tasks/bootstrap/create_mounts.yml @@ -1,12 +1,13 @@ --- - include_tasks: ../shared/create_mount.yml vars: - create_mount_path: "{{ item.name }}" + create_mount_path: "/{{ item.name }}" create_mount_default_lease_ttl: "{{ item.default_lease_ttl }}" create_mount_max_lease_ttl: "{{ item.max_lease_ttl }}" create_mount_description: "{{ item.description }}" create_mount_cert_dir: "{{ item.cert_dir }}" create_mount_config_ca_needed: "{{ item.config_ca }}" with_items: + - "{{ vault_pki_mounts.userpass|combine({'config_ca': not vault_ca_cert_needed}) }}" - "{{ vault_pki_mounts.vault|combine({'config_ca': not vault_ca_cert_needed}) }}" - "{{ vault_pki_mounts.etcd|combine({'config_ca': not vault_etcd_ca_cert_needed}) }}" diff --git a/roles/vault/tasks/bootstrap/create_roles.yml b/roles/vault/tasks/bootstrap/create_roles.yml index 8b252aaf5..c9cdad393 100644 --- a/roles/vault/tasks/bootstrap/create_roles.yml +++ b/roles/vault/tasks/bootstrap/create_roles.yml @@ -6,5 +6,5 @@ create_role_policy_rules: "{{ item.policy_rules }}" create_role_password: "{{ item.password }}" create_role_options: "{{ item.role_options }}" - create_role_mount_path: "{{ mount.name }}" + create_role_mount_path: "/{{ mount.name }}" with_items: "{{ mount.roles }}" diff --git a/roles/vault/tasks/bootstrap/gen_vault_certs.yml b/roles/vault/tasks/bootstrap/gen_vault_certs.yml index f982986cb..2857163dc 100644 --- a/roles/vault/tasks/bootstrap/gen_vault_certs.yml +++ b/roles/vault/tasks/bootstrap/gen_vault_certs.yml @@ -14,7 +14,7 @@ {%- endfor -%} "127.0.0.1","::1" ] - issue_cert_mount_path: "{{ vault_pki_mounts.vault.name }}" + issue_cert_mount_path: "/{{ vault_pki_mounts.vault.name }}" issue_cert_path: "{{ vault_cert_dir }}/api.pem" issue_cert_role: "{{ vault_pki_mounts.vault.roles[0].name }}" issue_cert_url: "{{ vault_leader_url }}" diff --git a/roles/vault/tasks/bootstrap/main.yml b/roles/vault/tasks/bootstrap/main.yml index 7ca82a9c4..18373ad9a 100644 --- a/roles/vault/tasks/bootstrap/main.yml +++ b/roles/vault/tasks/bootstrap/main.yml @@ -1,4 +1,9 @@ --- +- import_tasks: ../shared/check_etcd.yml + vars: + vault_etcd_needed: no + when: inventory_hostname in groups.vault + - import_tasks: ../shared/check_vault.yml when: inventory_hostname in groups.vault @@ -23,14 +28,14 @@ when: not vault_cluster_is_initialized - import_tasks: create_mounts.yml - when: inventory_hostname == groups.vault|first + when: inventory_hostname == groups.vault|first and not vault_cluster_is_initialized - include_tasks: ../shared/auth_backend.yml vars: auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates auth_backend_path: userpass auth_backend_type: userpass - when: inventory_hostname == groups.vault|first + when: inventory_hostname == groups.vault|first and not vault_cluster_is_initialized - include_tasks: create_roles.yml with_items: @@ -38,14 +43,15 @@ - "{{ vault_pki_mounts.etcd }}" loop_control: loop_var: mount - when: inventory_hostname in groups.vault + when: inventory_hostname in groups.vault and not vault_cluster_is_initialized - include_tasks: ../shared/gen_ca.yml vars: gen_ca_cert_dir: "{{ vault_pki_mounts.vault.cert_dir }}" - gen_ca_mount_path: "{{ vault_pki_mounts.vault.name }}" + gen_ca_mount_path: "/{{ vault_pki_mounts.vault.name }}" gen_ca_vault_headers: "{{ vault_headers }}" gen_ca_vault_options: "{{ vault_ca_options.vault }}" + gen_ca_copy_group: "kube-master" when: >- inventory_hostname in groups.vault and not vault_cluster_is_initialized @@ -54,13 +60,13 @@ - include_tasks: ../shared/gen_ca.yml vars: gen_ca_cert_dir: "{{ vault_pki_mounts.etcd.cert_dir }}" - gen_ca_mount_path: "{{ vault_pki_mounts.etcd.name }}" + gen_ca_mount_path: "/{{ vault_pki_mounts.etcd.name }}" gen_ca_vault_headers: "{{ vault_headers }}" gen_ca_vault_options: "{{ vault_ca_options.etcd }}" gen_ca_copy_group: "etcd" - when: inventory_hostname in groups.etcd and vault_etcd_ca_cert_needed + when: inventory_hostname in groups.etcd and not vault_cluster_is_initialized and vault_etcd_ca_cert_needed - import_tasks: gen_vault_certs.yml - when: inventory_hostname in groups.vault and vault_api_cert_needed + when: inventory_hostname in groups.vault and not vault_cluster_is_initialized and vault_api_cert_needed - import_tasks: ca_trust.yml diff --git a/roles/vault/tasks/bootstrap/start_vault_temp.yml b/roles/vault/tasks/bootstrap/start_vault_temp.yml index 1048b8c3e..3720d9beb 100644 --- a/roles/vault/tasks/bootstrap/start_vault_temp.yml +++ b/roles/vault/tasks/bootstrap/start_vault_temp.yml @@ -12,19 +12,14 @@ -v /etc/vault:/etc/vault {{ vault_image_repo }}:{{ vault_version }} server -# FIXME(mattymo): Crashes on first start with aufs docker storage. See hashicorp/docker-vault#19 - name: bootstrap/start_vault_temp | Start again single node Vault with file backend command: docker start {{ vault_temp_container_name }} - name: bootstrap/start_vault_temp | Initialize vault-temp - uri: - url: "http://localhost:{{ vault_port }}/v1/sys/init" - headers: "{{ vault_client_headers }}" - method: PUT - body_format: json - body: - secret_shares: 1 - secret_threshold: 1 + hashivault_init: + url: "http://localhost:{{ vault_port }}/" + secret_shares: 1 + secret_threshold: 1 until: "vault_temp_init|succeeded" retries: 4 delay: "{{ retry_stagger | random + 3 }}" @@ -34,16 +29,14 @@ - name: bootstrap/start_vault_temp | Set needed vault facts set_fact: vault_leader_url: "http://{{ inventory_hostname }}:{{ vault_port }}" - vault_temp_unseal_keys: "{{ vault_temp_init.json['keys'] }}" - vault_temp_root_token: "{{ vault_temp_init.json.root_token }}" - vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_temp_init.json.root_token}) }}" + vault_temp_unseal_keys: "{{ vault_temp_init.keys_base64 }}" + vault_root_token: "{{ vault_temp_init.root_token }}" + vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_temp_init.root_token}) }}" - name: bootstrap/start_vault_temp | Unseal vault-temp - uri: - url: "http://localhost:{{ vault_port }}/v1/sys/unseal" - headers: "{{ vault_headers }}" - method: POST - body_format: json - body: - key: "{{ item }}" + hashivault_unseal: + url: "http://localhost:{{ vault_port }}/" + token: "{{ vault_root_token }}" + keys: "{{ item }}" with_items: "{{ vault_temp_unseal_keys|default([]) }}" + no_log: true diff --git a/roles/vault/tasks/bootstrap/sync_etcd_certs.yml b/roles/vault/tasks/bootstrap/sync_etcd_certs.yml index f29b65022..1d6eabb5c 100644 --- a/roles/vault/tasks/bootstrap/sync_etcd_certs.yml +++ b/roles/vault/tasks/bootstrap/sync_etcd_certs.yml @@ -3,7 +3,7 @@ - include_tasks: ../shared/sync_file.yml vars: sync_file: "ca.pem" - sync_file_dir: "{{ vault_etcd_cert_dir }}" + sync_file_dir: "{{ etcd_cert_dir }}" sync_file_hosts: "{{ groups.etcd }}" sync_file_is_cert: true diff --git a/roles/vault/tasks/bootstrap/sync_secrets.yml b/roles/vault/tasks/bootstrap/sync_secrets.yml index cbbb581a7..8c1ae3929 100644 --- a/roles/vault/tasks/bootstrap/sync_secrets.yml +++ b/roles/vault/tasks/bootstrap/sync_secrets.yml @@ -29,19 +29,23 @@ - name: bootstrap/sync_secrets | Cat root_token from a vault host command: "cat {{ vault_secrets_dir }}/root_token" register: vault_root_token_cat - when: vault_secrets_available and inventory_hostname == groups.vault|first + run_once: yes + when: vault_secrets_available - name: bootstrap/sync_secrets | Cat unseal_keys from a vault host command: "cat {{ vault_secrets_dir }}/unseal_keys" register: vault_unseal_keys_cat - when: vault_secrets_available and inventory_hostname == groups.vault|first + run_once: yes + when: vault_secrets_available - name: bootstrap/sync_secrets | Set needed facts for Vault API interaction when Vault is already running set_fact: - vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token_cat']['stdout'] }}" - vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys_cat']['stdout_lines'] }}" + vault_root_token: "{{ vault_root_token_cat.stdout }}" + vault_unseal_keys: "{{ vault_unseal_keys_cat.stdout_lines }}" + run_once: yes when: vault_secrets_available +# FIXME: Remove all uri calls - name: bootstrap/sync_secrets | Update vault_headers if we have the root_token set_fact: vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token}) }}" diff --git a/roles/vault/tasks/bootstrap/sync_vault_certs.yml b/roles/vault/tasks/bootstrap/sync_vault_certs.yml index cb35ff66c..d6b2c6e91 100644 --- a/roles/vault/tasks/bootstrap/sync_vault_certs.yml +++ b/roles/vault/tasks/bootstrap/sync_vault_certs.yml @@ -14,6 +14,23 @@ set_fact: sync_file_results: [] +# FIXME: Distribute ca.pem alone in a better way +- include_tasks: ../shared/sync_file.yml + vars: + sync_file: "ca.pem" + sync_file_dir: "{{ vault_cert_dir }}" + sync_file_hosts: "{{ groups['kube-master'] }}" + sync_file_is_cert: false + +- name: bootstrap/sync_vault_certs | Set facts for vault sync_file results + set_fact: + vault_ca_cert_needed: "{{ sync_file_results[0]['no_srcs'] }}" + +- name: bootstrap/sync_vault_certs | Unset sync_file_results after ca.pem sync + set_fact: + sync_file_results: [] + + - include_tasks: ../shared/sync_file.yml vars: sync_file: "api.pem" diff --git a/roles/vault/tasks/cluster/binary.yml b/roles/vault/tasks/cluster/binary.yml index 41024dd54..5a055cbae 100644 --- a/roles/vault/tasks/cluster/binary.yml +++ b/roles/vault/tasks/cluster/binary.yml @@ -3,7 +3,7 @@ - name: cluster/binary | Copy vault binary from downloaddir copy: src: "{{ local_release_dir }}/vault/vault" - dest: "/usr/bin/vault" + dest: "{{ bin_dir }}/vault" remote_src: true mode: "0755" owner: vault diff --git a/roles/vault/tasks/cluster/configure.yml b/roles/vault/tasks/cluster/configure.yml index 7ac8f5f9e..fd20b9646 100644 --- a/roles/vault/tasks/cluster/configure.yml +++ b/roles/vault/tasks/cluster/configure.yml @@ -1,10 +1,19 @@ --- - -- name: cluster/configure | Ensure the vault/config directory exists +- name: cluster/configure | Ensure the vault directories exist file: - dest: "{{ vault_config_dir }}" + dest: "{{ item }}" + owner: vault mode: 0750 state: directory + recurse: true + with_items: + - "{{ vault_base_dir }}" + - "{{ vault_cert_dir }}" + - "{{ vault_config_dir }}" + - "{{ vault_roles_dir }}" + - "{{ vault_secrets_dir }}" + - "{{ vault_log_dir }}" + - "{{ vault_lib_dir }}" - name: cluster/configure | Lay down the configuration file copy: diff --git a/roles/vault/tasks/cluster/create_mounts.yml b/roles/vault/tasks/cluster/create_mounts.yml index 087430942..f5dd9609b 100644 --- a/roles/vault/tasks/cluster/create_mounts.yml +++ b/roles/vault/tasks/cluster/create_mounts.yml @@ -1,14 +1,13 @@ --- - include_tasks: ../shared/create_mount.yml vars: - create_mount_path: "{{ item.name }}" + create_mount_path: "/{{ item.name }}" create_mount_default_lease_ttl: "{{ item.default_lease_ttl }}" create_mount_max_lease_ttl: "{{ item.max_lease_ttl }}" create_mount_description: "{{ item.description }}" create_mount_cert_dir: "{{ item.cert_dir }}" - create_mount_config_ca_needed: item.name != vault_pki_mounts.kube.name and item.name != vault_pki_mounts.front_proxy.name + create_mount_config_ca_needed: item.name != vault_pki_mounts.kube.name with_items: - "{{ vault_pki_mounts.vault }}" - "{{ vault_pki_mounts.etcd }}" - "{{ vault_pki_mounts.kube }}" - - "{{ vault_pki_mounts.front_proxy }}" diff --git a/roles/vault/tasks/cluster/create_roles.yml b/roles/vault/tasks/cluster/create_roles.yml index 7b9d0b4f7..7a0e84982 100644 --- a/roles/vault/tasks/cluster/create_roles.yml +++ b/roles/vault/tasks/cluster/create_roles.yml @@ -6,5 +6,5 @@ create_role_password: "{{ item.password }}" create_role_policy_rules: "{{ item.policy_rules }}" create_role_options: "{{ item.role_options }}" - create_role_mount_path: "{{ mount.name }}" + create_role_mount_path: "/{{ mount.name }}" with_items: "{{ mount.roles }}" diff --git a/roles/vault/tasks/cluster/init.yml b/roles/vault/tasks/cluster/init.yml index 60aaf9591..30f64f3b1 100644 --- a/roles/vault/tasks/cluster/init.yml +++ b/roles/vault/tasks/cluster/init.yml @@ -1,36 +1,28 @@ --- +- name: cluster/init | wait for vault + command: /bin/true + notify: wait for vault up + +- meta: flush_handlers + - name: cluster/init | Initialize Vault - uri: - url: "https://{{ groups.vault|first }}:{{ vault_port }}/v1/sys/init" - headers: "{{ vault_client_headers }}" - method: POST - body_format: json - body: - secret_shares: "{{ vault_secret_shares }}" - secret_threshold: "{{ vault_secret_threshold }}" - validate_certs: false + hashivault_init: + url: "https://localhost:{{ vault_port }}/" + ca_cert: "{{ vault_cert_dir }}/ca.pem" + secret_shares: "{{ vault_secret_shares }}" + secret_threshold: "{{ vault_secret_threshold }}" + run_once: true register: vault_init_result - when: not vault_cluster_is_initialized and inventory_hostname == groups.vault|first + when: not vault_cluster_is_initialized - name: cluster/init | Set facts on the results of the initialization set_fact: - vault_unseal_keys: "{{ vault_init_result.json['keys'] }}" - vault_root_token: "{{ vault_init_result.json.root_token }}" - vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_init_result.json.root_token}) }}" - when: not vault_cluster_is_initialized and inventory_hostname == groups.vault|first - -- name: cluster/init | Ensure all hosts have these facts - set_fact: - vault_unseal_keys: "{{ hostvars[groups.vault|first]['vault_unseal_keys'] }}" - vault_root_token: "{{ hostvars[groups.vault|first]['vault_root_token'] }}" - when: not vault_cluster_is_initialized and inventory_hostname != groups.vault|first - -- name: cluster/init | Ensure the vault_secrets_dir exists - file: - mode: 0750 - path: "{{ vault_secrets_dir }}" - state: directory + vault_unseal_keys: "{{ vault_init_result.keys_base64 }}" + vault_root_token: "{{ vault_init_result.root_token }}" + vault_headers: "{{ vault_client_headers|combine({'X-Vault-Token': vault_init_result.root_token}) }}" + run_once: true + when: not vault_cluster_is_initialized - name: cluster/init | Ensure all in groups.vault have the unseal_keys locally copy: @@ -48,5 +40,5 @@ - name: cluster/init | Ensure vault_headers and vault statuses are updated set_fact: - vault_headers: "{{ vault_client_headers | combine({'X-Vault-Token': vault_root_token})}}" vault_cluster_is_initialized: true + run_once: true diff --git a/roles/vault/tasks/cluster/main.yml b/roles/vault/tasks/cluster/main.yml index 7f535d068..3ed23b2cc 100644 --- a/roles/vault/tasks/cluster/main.yml +++ b/roles/vault/tasks/cluster/main.yml @@ -1,8 +1,8 @@ --- -- import_tasks: ../shared/check_vault.yml +- import_tasks: ../shared/check_etcd.yml when: inventory_hostname in groups.vault -- import_tasks: ../shared/check_etcd.yml +- import_tasks: ../shared/check_vault.yml when: inventory_hostname in groups.vault - import_tasks: configure.yml @@ -14,6 +14,9 @@ - import_tasks: systemd.yml when: inventory_hostname in groups.vault +- import_tasks: ../shared/find_leader.yml + when: inventory_hostname in groups.vault + - import_tasks: init.yml when: inventory_hostname in groups.vault @@ -29,20 +32,12 @@ - include_tasks: ../shared/gen_ca.yml vars: gen_ca_cert_dir: "{{ vault_pki_mounts.kube.cert_dir }}" - gen_ca_mount_path: "{{ vault_pki_mounts.kube.name }}" + gen_ca_mount_path: "/{{ vault_pki_mounts.kube.name }}" gen_ca_vault_headers: "{{ vault_headers }}" gen_ca_vault_options: "{{ vault_ca_options.kube }}" gen_ca_copy_group: "kube-master" when: inventory_hostname in groups.vault -- include_tasks: ../shared/gen_ca.yml - vars: - gen_ca_cert_dir: "{{ vault_pki_mounts.front_proxy.cert_dir }}" - gen_ca_mount_path: "{{ vault_pki_mounts.front_proxy.name }}" - gen_ca_vault_headers: "{{ vault_headers }}" - gen_ca_vault_options: "{{ vault_ca_options.front_proxy }}" - when: inventory_hostname in groups.vault - - include_tasks: ../shared/auth_backend.yml vars: auth_backend_description: A Username/Password Auth Backend primarily used for services needing to issue certificates @@ -55,7 +50,6 @@ - "{{ vault_pki_mounts.vault }}" - "{{ vault_pki_mounts.etcd }}" - "{{ vault_pki_mounts.kube }}" - - "{{ vault_pki_mounts.front_proxy }}" loop_control: loop_var: mount when: inventory_hostname in groups.vault diff --git a/roles/vault/tasks/cluster/systemd.yml b/roles/vault/tasks/cluster/systemd.yml index f7139d336..f09cb8ed7 100644 --- a/roles/vault/tasks/cluster/systemd.yml +++ b/roles/vault/tasks/cluster/systemd.yml @@ -1,32 +1,11 @@ --- - -- name: cluster/systemd | Ensure mount points exist prior to vault.service startup - file: - mode: 0750 - path: "{{ item }}" - state: directory - with_items: - - "{{ vault_config_dir }}" - - "{{ vault_log_dir }}" - - "{{ vault_secrets_dir }}" - - /var/lib/vault/ - -- name: cluster/systemd | Ensure the vault user has access to needed directories - file: - owner: vault - path: "{{ item }}" - recurse: true - with_items: - - "{{ vault_base_dir }}" - - "{{ vault_log_dir }}" - - /var/lib/vault - - name: cluster/systemd | Copy down vault.service systemd file template: src: "{{ vault_deployment_type }}.service.j2" dest: /etc/systemd/system/vault.service backup: yes register: vault_systemd_placement + notify: restart vault - name: Create vault service systemd directory file: @@ -39,6 +18,7 @@ dest: /etc/systemd/system/vault.service.d/http-proxy.conf backup: yes when: http_proxy is defined or https_proxy is defined + notify: restart vault - name: cluster/systemd | Enable vault.service systemd: @@ -46,13 +26,4 @@ enabled: yes name: vault state: started - -- name: cluster/systemd | Query local vault until service is up - uri: - url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health" - headers: "{{ vault_client_headers }}" - status_code: 200,429,500,501 - register: vault_health_check - until: vault_health_check|succeeded - retries: 10 - delay: "{{ retry_stagger | random + 3 }}" + notify: wait for vault up diff --git a/roles/vault/tasks/cluster/unseal.yml b/roles/vault/tasks/cluster/unseal.yml index b9257bf49..6d0414d0d 100644 --- a/roles/vault/tasks/cluster/unseal.yml +++ b/roles/vault/tasks/cluster/unseal.yml @@ -1,25 +1,16 @@ --- - name: cluster/unseal | Current sealed state - debug: " Sealed? {{vault_is_sealed}}" + debug: + msg: "Sealed? {{ vault_is_sealed }}" - name: cluster/unseal | Unseal Vault - uri: - url: "https://localhost:{{ vault_port }}/v1/sys/unseal" - headers: "{{ vault_headers }}" - method: POST - body_format: json - body: - key: "{{ item }}" + hashivault_unseal: + url: "https://localhost:{{ vault_port }}/" + token: "{{ vault_root_token }}" + ca_cert: "{{ vault_cert_dir }}/ca.pem" + keys: "{{ item }}" + no_log: true with_items: "{{ vault_unseal_keys|default([]) }}" + notify: wait for vault up when: vault_is_sealed - -- name: cluster/unseal | Wait until server is ready - uri: - url: "https://localhost:{{ vault_port }}/v1/sys/health" - headers: "{{ vault_headers }}" - method: HEAD - status_code: 200, 429 - register: vault_node_ready - until: vault_node_ready|succeeded - retries: 5 diff --git a/roles/vault/tasks/main.yml b/roles/vault/tasks/main.yml index 7a10def8a..ae59132a3 100644 --- a/roles/vault/tasks/main.yml +++ b/roles/vault/tasks/main.yml @@ -10,6 +10,11 @@ # Vault cluster using Etcd as the backend. The same Root CA is mounted as # used during step 1, allowing all certs to have the same chain of trust. +- name: install hvac + pip: + name: "hvac" + state: "present" + ## Bootstrap - include_tasks: bootstrap/main.yml when: cert_management == 'vault' and vault_bootstrap | d() diff --git a/roles/vault/tasks/shared/auth_backend.yml b/roles/vault/tasks/shared/auth_backend.yml index 82a4c94fb..fa5d86865 100644 --- a/roles/vault/tasks/shared/auth_backend.yml +++ b/roles/vault/tasks/shared/auth_backend.yml @@ -1,20 +1,10 @@ --- -- name: shared/auth_backend | Test if the auth backend exists - uri: - url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}/tune" - headers: "{{ vault_headers }}" - validate_certs: false - ignore_errors: true - register: vault_auth_backend_check - -- name: shared/auth_backend | Add the cert auth backend if needed - uri: - url: "{{ vault_leader_url }}/v1/sys/auth/{{ auth_backend_path }}" - headers: "{{ vault_headers }}" - method: POST - body_format: json - body: - description: "{{ auth_backend_description|d('') }}" - type: "{{ auth_backend_type }}" - status_code: 204 - when: vault_auth_backend_check|failed +- name: shared/auth_backend | Enable auth backend {{ auth_backend_path }} + hashivault_auth_enable: + url: "{{ vault_leader_url }}" + token: "{{ vault_root_token }}" + ca_cert: "{{ vault_cert_dir }}/ca.pem" + name: "{{ auth_backend_type }}" + mount_point: "{{ auth_backend_path }}" + description: "{{ auth_backend_description|d('') }}" + register: result diff --git a/roles/vault/tasks/shared/cert_auth_mount.yml b/roles/vault/tasks/shared/cert_auth_mount.yml index d9af9c310..b94f53130 100644 --- a/roles/vault/tasks/shared/cert_auth_mount.yml +++ b/roles/vault/tasks/shared/cert_auth_mount.yml @@ -10,11 +10,10 @@ max_lease_ttl: "{{ vault_max_lease_ttl }}" - name: shared/auth_mount | Create a dummy role for issuing certs from auth-pki - uri: - url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth-pki/roles/dummy" - headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" - method: POST - body_format: json - body: - {'allow_any_name': true} - status_code: 204 + hashivault_approle_role_create: + url: "{{ vault_leader_url }}" + token: "{{ vault_root_token }}" + ca_cert: "{{ vault_cert_dir }}/ca.pem" + name: "auth-pki/roles/dummy" + policies: + allow_any_name: true diff --git a/roles/vault/tasks/shared/check_etcd.yml b/roles/vault/tasks/shared/check_etcd.yml index 6158a3126..eaa951114 100644 --- a/roles/vault/tasks/shared/check_etcd.yml +++ b/roles/vault/tasks/shared/check_etcd.yml @@ -4,9 +4,12 @@ uri: url: "{{ vault_etcd_url }}/health" validate_certs: no + client_cert: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" + client_key: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + return_content: yes until: vault_etcd_health_check.status == 200 or vault_etcd_health_check.status == 401 - retries: 10 + retries: 3 delay: 2 delegate_to: "{{groups['etcd'][0]}}" run_once: true diff --git a/roles/vault/tasks/shared/check_vault.yml b/roles/vault/tasks/shared/check_vault.yml index 83328768a..1ffd515fd 100644 --- a/roles/vault/tasks/shared/check_vault.yml +++ b/roles/vault/tasks/shared/check_vault.yml @@ -8,24 +8,42 @@ # Check if vault is reachable on the localhost - name: check_vault | Attempt to pull local https Vault health - uri: - url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health" - headers: "{{ vault_client_headers }}" - status_code: 200,429,500,501,503 - validate_certs: no - ignore_errors: true - register: vault_local_service_health + command: /bin/true + notify: wait for vault up nowait + +- meta: flush_handlers - name: check_vault | Set facts about local Vault health set_fact: - vault_is_running: "{{ vault_local_service_health|succeeded }}" - vault_is_initialized: "{{ vault_local_service_health.get('json', {}).get('initialized', false) }}" - vault_is_sealed: "{{ vault_local_service_health.get('json', {}).get('sealed', true) }}" - # vault_in_standby: "{{ vault_local_service_health.get('json', {}).get('standby', true) }}" + vault_is_running: "{{ vault_health_check.get('status', '-1') in vault_successful_http_codes }}" + +- name: check_vault | Set facts about local Vault health + set_fact: + vault_is_initialized: "{{ vault_health_check.get('json', {}).get('initialized', false) }}" + vault_is_sealed: "{{ vault_health_check.get('json', {}).get('sealed', true) }}" + # vault_in_standby: "{{ vault_health_check.get('json', {}).get('standby', true) }}" # vault_run_version: "{{ vault_local_service_health.get('json', {}).get('version', '') }}" +- name: check_vault | Check is vault is initialized in etcd if vault is not running + command: |- + curl \ + --cacert {{ etcd_cert_dir }}/ca.pem \ + --cert {{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem \ + --key {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem \ + -X POST -d '{"key": "{{ "/vault/core/seal-config" | b64encode }}"}' \ + {{ etcd_access_addresses.split(',') | first }}/v3alpha/kv/range + register: vault_etcd_exists + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + run_once: true + when: not vault_is_running and vault_etcd_available + changed_when: false + - name: check_vault | Set fact about the Vault cluster's initialization state set_fact: - vault_cluster_is_initialized: "{{ vault_is_initialized or hostvars[item]['vault_is_initialized'] }}" + vault_cluster_is_initialized: >- + {{ vault_is_initialized or + hostvars[item]['vault_is_initialized'] or + 'Key not found' not in vault_etcd_exists.stdout|default('Key not found') }} with_items: "{{ groups.vault }}" run_once: true diff --git a/roles/vault/tasks/shared/config_ca.yml b/roles/vault/tasks/shared/config_ca.yml index 0ef34e7b8..f3f00d0be 100644 --- a/roles/vault/tasks/shared/config_ca.yml +++ b/roles/vault/tasks/shared/config_ca.yml @@ -4,26 +4,26 @@ register: vault_ca_cert_cat - name: config_ca | Pull current CA cert from Vault - uri: - url: "{{ vault_leader_url }}/v1/{{ config_ca_mount_path }}/ca/pem" - headers: "{{ vault_headers }}" - return_content: true - status_code: 200,204 - validate_certs: no + hashivault_read: + url: "{{ vault_leader_url }}" + token: "{{ vault_root_token }}" + ca_cert: "{{ vault_cert_dir }}/ca.pem" + secret: "{{ config_ca_mount_path }}/ca" + key: "pem" register: vault_pull_current_ca + failed_when: false - name: config_ca | Read root CA key for Vault command: "cat {{ config_ca_ca_key }}" register: vault_ca_key_cat - when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.content.strip() + when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.get("data","").strip() - name: config_ca | Configure pki mount to use the found root CA cert and key - uri: - url: "{{ vault_leader_url }}/v1/{{ config_ca_mount_path }}/config/ca" - headers: "{{ vault_headers }}" - method: POST - body_format: json - body: + hashivault_write: + url: "{{ vault_leader_url }}" + token: "{{ vault_root_token }}" + ca_cert: "{{ vault_cert_dir }}/ca.pem" + secret: "{{ config_ca_mount_path }}/config/ca" + data: pem_bundle: "{{ vault_ca_cert_cat.stdout + '\n' + vault_ca_key_cat.stdout }}" - status_code: 204 - when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.get("content","").strip() + when: vault_ca_cert_cat.stdout.strip() != vault_pull_current_ca.get("data","").strip() diff --git a/roles/vault/tasks/shared/create_role.yml b/roles/vault/tasks/shared/create_role.yml index 4d1915a54..d3aa3e441 100644 --- a/roles/vault/tasks/shared/create_role.yml +++ b/roles/vault/tasks/shared/create_role.yml @@ -1,42 +1,36 @@ --- -# The JSON inside JSON here is intentional (Vault API wants it) -- name: create_role | Create a policy for the new role allowing issuing - uri: - url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/sys/policy/{{ create_role_name }}" - headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" - method: PUT - body_format: json - body: - rules: >- - {%- if create_role_policy_rules|d("default") == "default" -%} - {{ - { 'path': { - create_role_mount_path + '/issue/' + create_role_name: {'policy': 'write'}, - create_role_mount_path + '/roles/' + create_role_name: {'policy': 'read'} - }} | to_json + '\n' - }} - {%- else -%} - {{ create_role_policy_rules | to_json + '\n' }} - {%- endif -%} - status_code: 204 - delegate_to: "{{ groups.vault|first }}" - run_once: true +- name: create_role | Create a policy for the new role + hashivault_policy_set: + url: "{{ vault_leader_url }}" + token: "{{ vault_root_token }}" + ca_cert: "{{ vault_cert_dir }}/ca.pem" + name: "{{ create_role_name }}" + rules: >- + {%- if create_role_policy_rules|d("default") == "default" -%} + {{ + { 'path': { + create_role_mount_path + '/issue/' + create_role_name: {'policy': 'write'}, + create_role_mount_path + '/roles/' + create_role_name: {'policy': 'read'} + }} | to_json + '\n' + }} + {%- else -%} + {{ create_role_policy_rules | to_json + '\n' }} + {%- endif -%} - name: create_role | Create {{ create_role_name }} role in the {{ create_role_mount_path }} pki mount - uri: - url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/{{ create_role_mount_path }}/roles/{{ create_role_name }}" - headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" - method: POST - body_format: json - body: >- - {%- if create_role_options|d("default") == "default" -%} - {'allow_any_name': true} - {%- else -%} - {{ create_role_options }} - {%- endif -%} - status_code: 204 - delegate_to: "{{ groups.vault|first }}" - run_once: true + hashivault_write: + url: "{{ vault_leader_url }}" + token: "{{ vault_root_token }}" + ca_cert: "{{ vault_cert_dir }}/ca.pem" + secret: "{{ create_role_mount_path }}/roles/{{ create_role_name }}" + data: | + {%- if create_role_options|d("default") == "default" -%} + { + allow_any_name: true + } + {%- else -%} + {{ create_role_options | to_json }} + {%- endif -%} ## Userpass based auth method diff --git a/roles/vault/tasks/shared/find_leader.yml b/roles/vault/tasks/shared/find_leader.yml index 3afee482d..1c1dcdf30 100644 --- a/roles/vault/tasks/shared/find_leader.yml +++ b/roles/vault/tasks/shared/find_leader.yml @@ -5,16 +5,16 @@ url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health" headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" method: HEAD - status_code: 200,429,503 + status_code: 200,429,501,503 register: vault_leader_check until: "vault_leader_check|succeeded" retries: 10 - name: find_leader | Set fact for current http leader set_fact: - vault_leader_url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://{{ item }}:{{ vault_port }}" + vault_leader_url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://{{ inventory_hostname }}:{{ vault_port }}" with_items: "{{ groups.vault }}" - when: "hostvars[item]['vault_leader_check'].get('status') in [200,503]" + when: "hostvars[item]['vault_leader_check'].get('status') in [200,501,503]" # run_once: true - name: find_leader| show vault_leader_url diff --git a/roles/vault/tasks/shared/gen_ca.yml b/roles/vault/tasks/shared/gen_ca.yml index 77f2f82b9..eaf7a61ff 100644 --- a/roles/vault/tasks/shared/gen_ca.yml +++ b/roles/vault/tasks/shared/gen_ca.yml @@ -1,35 +1,38 @@ --- -- name: "bootstrap/gen_ca | Ensure cert_dir {{ gen_ca_cert_dir }} exists" +- name: "bootstrap/gen_ca | Ensure cert_dir {{ gen_ca_cert_dir }} exists on necessary hosts" file: mode: 0755 path: "{{ gen_ca_cert_dir }}" state: directory + delegate_to: "{{ item }}" + with_items: "{{ (groups[gen_ca_copy_group|default('vault')]) | union(groups['vault']) }}" - name: "bootstrap/gen_ca | Generate {{ gen_ca_mount_path }} root CA" - uri: - url: "{{ vault_leader_url }}/v1/{{ gen_ca_mount_path }}/root/generate/exported" - headers: "{{ gen_ca_vault_headers }}" - method: POST - body_format: json - body: "{{ gen_ca_vault_options }}" - status_code: 200,204 - register: vault_ca_gen - delegate_to: "{{ groups.vault|first }}" + hashivault_write: + url: "{{ vault_leader_url }}" + token: "{{ vault_root_token }}" + ca_cert: "{{ vault_cert_dir }}/ca.pem" + secret: "{{ gen_ca_mount_path }}/root/generate/exported" + data: "{{ gen_ca_vault_options }}" run_once: true + no_log: true + register: vault_ca_gen - name: "bootstrap/gen_ca | Copy {{ gen_ca_mount_path }} root CA cert locally" copy: - content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['certificate'] }}" + content: "{{ vault_ca_gen['data']['data']['certificate'] }}" dest: "{{ gen_ca_cert_dir }}/ca.pem" mode: 0644 - when: vault_ca_gen.status == 200 + when: '"data" in vault_ca_gen.keys()' + delegate_to: "{{ item }}" + with_items: "{{ (groups[gen_ca_copy_group|default('vault')]) | union(groups['vault']) }}" - name: "bootstrap/gen_ca | Copy {{ gen_ca_mount_path }} root CA key to necessary hosts" copy: - content: "{{ hostvars[groups.vault|first]['vault_ca_gen']['json']['data']['private_key'] }}" + content: "{{ vault_ca_gen['data']['data']['private_key']}}" dest: "{{ gen_ca_cert_dir }}/ca-key.pem" mode: 0640 - when: vault_ca_gen.status == 200 + when: '"data" in vault_ca_gen.keys()' delegate_to: "{{ item }}" with_items: "{{ (groups[gen_ca_copy_group|default('vault')]) | union(groups['vault']) }}" diff --git a/roles/vault/tasks/shared/gen_userpass.yml b/roles/vault/tasks/shared/gen_userpass.yml index 5def39d0e..a49b443e3 100644 --- a/roles/vault/tasks/shared/gen_userpass.yml +++ b/roles/vault/tasks/shared/gen_userpass.yml @@ -1,16 +1,13 @@ --- - name: shared/gen_userpass | Create the Username/Password combo for the role - uri: - url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/users/{{ gen_userpass_username }}" - headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}" - method: POST - body_format: json - body: - username: "{{ gen_userpass_username }}" - password: "{{ gen_userpass_password }}" - policies: "{{ gen_userpass_role }}" - status_code: 204 - delegate_to: "{{ groups.vault|first }}" + hashivault_userpass_create: + url: "{{ vault_leader_url }}" + token: "{{ vault_root_token }}" + ca_cert: "{{ vault_cert_dir }}/ca.pem" + name: "{{ gen_userpass_username }}" + pass: "{{ gen_userpass_password }}" + policies: + - "{{ gen_userpass_role }}" run_once: true - name: shared/gen_userpass | Ensure destination directory exists diff --git a/roles/vault/tasks/shared/issue_cert.yml b/roles/vault/tasks/shared/issue_cert.yml index 36a42efaa..89921b345 100644 --- a/roles/vault/tasks/shared/issue_cert.yml +++ b/roles/vault/tasks/shared/issue_cert.yml @@ -39,52 +39,58 @@ delegate_to: "{{ groups.vault|first }}" run_once: true -- name: gen_certs_vault | Log into Vault and obtain an token - uri: - url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ user_vault_creds.username }}" - headers: - Accept: application/json - Content-Type: application/json - method: POST - body_format: json - body: - password: "{{ user_vault_creds.password }}" - register: vault_login_result - delegate_to: "{{ groups.vault|first }}" +- name: gen_certs_vault | Ensure vault cert dir exists + file: + path: "{{ vault_cert_dir }}" + state: directory + recurse: yes + owner: "vault" + group: "vault" + mode: 0755 + +- name: gen_certs_vault | install hvac + pip: + name: "hvac" + state: "present" + +- name: gen_certs_vault | Pull vault CA + get_url: + url: "{{ issue_cert_url }}/v1/vault/ca/pem" + dest: "{{ vault_cert_dir }}/ca.pem" + validate_certs: no + when: '"https" in issue_cert_url' + +- name: gen_certs_vault | Log into Vault and obtain a scoped token + hashivault_token_create: + url: "{{ issue_cert_url }}" + token: "{{ vault_root_token | default(hostvars[groups.vault|first]['vault_root_token']) }}" + ca_cert: "{{ vault_cert_dir }}/ca.pem" + policies: "{{ user_vault_creds.username }}" + display_name: "{{ user_vault_creds.username }}" + register: vault_client_token_request run_once: true -- name: gen_certs_vault | Set fact for vault_client_token +- name: gen_certs_vault | Pull token from request set_fact: - vault_client_token: "{{ vault_login_result.get('json', {}).get('auth', {}).get('client_token') }}" + vault_client_token: "{{ vault_client_token_request['token']['auth']['client_token'] }}" run_once: true -- name: gen_certs_vault | Set fact for Vault API token - set_fact: - issue_cert_headers: - Accept: application/json - Content-Type: application/json - X-Vault-Token: "{{ vault_client_token }}" - run_once: true - when: vault_client_token != "" - - name: "issue_cert | Generate {{ issue_cert_path }} for {{ issue_cert_role }} role" - uri: - url: "{{ issue_cert_url }}/v1/{{ issue_cert_mount_path|d('pki') }}/issue/{{ issue_cert_role }}" - headers: "{{ issue_cert_headers }}" - method: POST - body_format: json - body: + hashivault_write: + url: "{{ issue_cert_url }}" + token: "{{ vault_client_token }}" + ca_cert: "{% if 'https' in issue_cert_url %}{{ vault_cert_dir }}/ca.pem{% endif %}" + secret: "{{ issue_cert_mount_path|d('/pki') }}/issue/{{ issue_cert_role }}" + data: alt_names: "{{ issue_cert_alt_names | d([]) | join(',') }}" common_name: "{{ issue_cert_common_name | d(issue_cert_path.rsplit('/', 1)[1].rsplit('.', 1)[0]) }}" format: "{{ issue_cert_format | d('pem') }}" ip_sans: "{{ issue_cert_ip_sans | default([]) | join(',') }}" register: issue_cert_result - delegate_to: "{{ issue_cert_hosts|first }}" - run_once: true - name: "issue_cert | Copy {{ issue_cert_path }} cert to all hosts" copy: - content: "{{ issue_cert_result['json']['data']['certificate'] }}\n" + content: "{{ issue_cert_result['data']['data']['certificate'] }}\n" dest: "{{ issue_cert_path }}" group: "{{ issue_cert_file_group | d('root' )}}" mode: "{{ issue_cert_file_mode | d('0644') }}" @@ -92,7 +98,7 @@ - name: "issue_cert | Copy key for {{ issue_cert_path }} to all hosts" copy: - content: "{{ issue_cert_result['json']['data']['private_key'] }}" + content: "{{ issue_cert_result['data']['data']['private_key'] }}" dest: "{{ issue_cert_path.rsplit('.', 1)|first }}-key.{{ issue_cert_path.rsplit('.', 1)|last }}" group: "{{ issue_cert_file_group | d('root' )}}" mode: "{{ issue_cert_file_mode | d('0640') }}" @@ -100,7 +106,7 @@ - name: issue_cert | Copy issuing CA cert copy: - content: "{{ issue_cert_result['json']['data']['issuing_ca'] }}\n" + content: "{{ issue_cert_result['data']['data']['issuing_ca'] }}\n" dest: "{{ issue_cert_path | dirname }}/{{ issue_cert_ca_filename | default('ca.pem') }}" group: "{{ issue_cert_file_group | d('root' )}}" mode: "{{ issue_cert_file_mode | d('0644') }}" @@ -109,7 +115,7 @@ - name: issue_cert | Copy certificate serial to all hosts copy: - content: "{{ issue_cert_result['json']['data']['serial_number'] }}" + content: "{{ issue_cert_result['data']['data']['serial_number'] }}" dest: "{{ issue_cert_path.rsplit('.', 1)|first }}.serial" group: "{{ issue_cert_file_group | d('root' )}}" mode: "{{ issue_cert_file_mode | d('0640') }}" diff --git a/roles/vault/tasks/shared/pki_mount.yml b/roles/vault/tasks/shared/pki_mount.yml index 3df56e0f8..19fa34ab4 100644 --- a/roles/vault/tasks/shared/pki_mount.yml +++ b/roles/vault/tasks/shared/pki_mount.yml @@ -1,27 +1,12 @@ --- -- name: "shared/mount | Test if {{ pki_mount_path }} PKI mount exists" - uri: - url: "{{ vault_leader_url }}/v1/sys/mounts/{{ pki_mount_path }}/tune" - headers: "{{ vault_headers }}" - ignore_errors: true - register: vault_pki_mount_check -- name: shared/mount | Set pki mount type - set_fact: - mount_options: "{{ pki_mount_options | combine({'type': 'pki'}) }}" - when: vault_pki_mount_check|failed - -- name: shared/mount | Mount {{ pki_mount_path }} PKI mount if needed - uri: - url: "{{ vault_leader_url }}/v1/sys/mounts/{{ pki_mount_path }}" - headers: "{{ vault_headers }}" - method: POST - body_format: json - body: "{{ mount_options|d() }}" - status_code: 204 - when: vault_pki_mount_check|failed - -- name: shared/mount | Unset mount options - set_fact: - mount_options: {} - when: vault_pki_mount_check|failed +- name: shared/mount | Enable {{ pki_mount_path }} PKI mount + hashivault_secret_enable: + url: "{{ vault_leader_url }}" + token: "{{ vault_root_token }}" + ca_cert: "{{ vault_cert_dir }}/ca.pem" + name: "{{ pki_mount_path }}" + backend: "pki" + config: "{{ pki_mount_options }}" + register: secret_enable_result + failed_when: 'secret_enable_result.rc !=0 and "existing mount" not in secret_enable_result.msg' diff --git a/roles/vault/tasks/shared/sync.yml b/roles/vault/tasks/shared/sync.yml index 102532f0c..5bbe1bd59 100644 --- a/roles/vault/tasks/shared/sync.yml +++ b/roles/vault/tasks/shared/sync.yml @@ -1,5 +1,4 @@ --- - - name: "sync_file | Cat the file" command: "cat {{ sync_file_path }}" register: sync_file_cat diff --git a/roles/vault/tasks/shared/sync_file.yml b/roles/vault/tasks/shared/sync_file.yml index 2abef1c5a..00750df2f 100644 --- a/roles/vault/tasks/shared/sync_file.yml +++ b/roles/vault/tasks/shared/sync_file.yml @@ -1,7 +1,6 @@ --- # NOTE: This should be a role (or custom module), but currently include_role is too buggy to use - - name: "sync_file | Set facts for directory and file when sync_file_path is defined" set_fact: sync_file_dir: "{{ sync_file_path | dirname }}" diff --git a/roles/vault/templates/docker.service.j2 b/roles/vault/templates/docker.service.j2 index f99035c77..0ee00732b 100644 --- a/roles/vault/templates/docker.service.j2 +++ b/roles/vault/templates/docker.service.j2 @@ -21,13 +21,15 @@ ExecStart={{ docker_bin_dir }}/docker run \ --cap-add=IPC_LOCK \ -v {{ vault_cert_dir }}:{{ vault_cert_dir }} \ -v {{ vault_config_dir }}:{{ vault_config_dir }} \ --v {{ vault_etcd_cert_dir }}:{{ vault_etcd_cert_dir }} \ +-v /etc/ssl:/etc/ssl \ +-v {{ etcd_cert_dir }}:{{ etcd_cert_dir }} \ -v {{ vault_log_dir }}:/vault/logs \ -v {{ vault_roles_dir }}:{{ vault_roles_dir }} \ -v {{ vault_secrets_dir }}:{{ vault_secrets_dir }} \ --entrypoint=vault \ {{ vault_image_repo }}:{{ vault_image_tag }} \ -server --config={{ vault_config_dir }}/config.json +server --config={{ vault_config_dir }}/config.json \ +--log-level=trace [Install] WantedBy=multi-user.target diff --git a/roles/vault/templates/rkt.service.j2 b/roles/vault/templates/rkt.service.j2 index 43dd8fc6d..6a4c3d77a 100644 --- a/roles/vault/templates/rkt.service.j2 +++ b/roles/vault/templates/rkt.service.j2 @@ -26,8 +26,8 @@ ExecStart=/usr/bin/rkt run \ --mount=volume=vault-secrets-dir,target={{ vault_secrets_dir }} \ --volume=vault-roles-dir,kind=host,source={{ vault_roles_dir }} \ --mount=volume=vault-roles-dir,target={{ vault_roles_dir }} \ ---volume=vault-etcd-cert-dir,kind=host,source={{ vault_etcd_cert_dir }} \ ---mount=volume=vault-etcd-cert-dir,target={{ vault_etcd_cert_dir }} \ +--volume=etcd-cert-dir,kind=host,source={{ etcd_cert_dir }} \ +--mount=volume=etcd-cert-dir,target={{ etcd_cert_dir }} \ docker://{{ vault_image_repo }}:{{ vault_image_tag }} \ --name={{ vault_container_name }} --net=host \ --caps-retain=CAP_IPC_LOCK \ diff --git a/tests/files/gce_coreos-vault-upgrade.yml b/tests/files/gce_coreos-vault-upgrade.yml new file mode 100644 index 000000000..9e6d89276 --- /dev/null +++ b/tests/files/gce_coreos-vault-upgrade.yml @@ -0,0 +1,13 @@ +# Instance settings +cloud_machine_type: "n1-standard-4" +cloud_image_family: coreos-stable +cloud_region: us-central1-b +mode: aio + +# Instance settings +bootstrap_os: coreos +cert_management: vault +kube_network_plugin: flannel +deploy_netchecker: true +kubedns_min_replicas: 1 +cloud_provider: gce