wip pr for improved cert sync

library/vault_cert_issue.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+DOCUMENTATION = '''
+---
+module: hashivault_pki_issue
+version_added: "0.1"
+short_description: Hashicorp Vault PKI issue module
+description:
+    - Module to issue PKI certs from Hashicorp Vault.
+options:
+    url:
+        description:
+            - url for vault
+        default: to environment variable VAULT_ADDR
+    ca_cert:
+        description:
+            - "path to a PEM-encoded CA cert file to use to verify the Vault server TLS certificate"
+        default: to environment variable VAULT_CACERT
+    ca_path:
+        description:
+            - "path to a directory of PEM-encoded CA cert files to verify the Vault server TLS certificate : if ca_cert is specified, its value will take precedence"
+        default: to environment variable VAULT_CAPATH
+    client_cert:
+        description:
+            - "path to a PEM-encoded client certificate for TLS authentication to the Vault server"
+        default: to environment variable VAULT_CLIENT_CERT
+    client_key:
+        description:
+            - "path to an unencrypted PEM-encoded private key matching the client certificate"
+        default: to environment variable VAULT_CLIENT_KEY
+    verify:
+        description:
+            - "if set, do not verify presented TLS certificate before communicating with Vault server : setting this variable is not recommended except during testing"
+        default: to environment variable VAULT_SKIP_VERIFY
+    authtype:
+        description:
+            - "authentication type to use: token, userpass, github, ldap, approle"
+        default: token
+    token:
+        description:
+            - token for vault
+        default: to environment variable VAULT_TOKEN
+    username:
+        description:
+            - username to login to vault.
+        default: to environment variable VAULT_USER
+    password:
+        description:
+            - password to login to vault.
+        default: to environment variable VAULT_PASSWORD
+    secret:
+        description:
+            - secret to read.
+    data:
+        description:
+            - Keys and values to write.
+    update:
+        description:
+            - Update rather than overwrite.
+        default: False
+    min_ttl:
+        description:
+            - Issue new cert if existing cert has lower TTL expressed in hours or a percentage. Examples: 70800h, 50%
+    force:
+        description:
+            - Force issue of new cert
+
+'''
+EXAMPLES = '''
+---
+- hosts: localhost
+  tasks:
+    - hashivault_write:
+        secret: giant
+        data:
+            foo: foe
+            fie: fum
+'''
+
+
+def main():
+    argspec = hashivault_argspec()
+    argspec['secret'] = dict(required=True, type='str')
+    argspec['update'] = dict(required=False, default=False, type='bool')
+    argspec['data'] = dict(required=False, default={}, type='dict')
+    module = hashivault_init(argspec, supports_check_mode=True)
+    result = hashivault_write(module)
+    if result.get('failed'):
+        module.fail_json(**result)
+    else:
+        module.exit_json(**result)
+
+
+def _convert_to_seconds(original_value):
+    try:
+        value = str(original_value)
+        seconds = 0
+        if 'h' in value:
+            ray = value.split('h')
+            seconds = int(ray.pop(0)) * 3600
+            value = ''.join(ray)
+        if 'm' in value:
+            ray = value.split('m')
+            seconds += int(ray.pop(0)) * 60
+            value = ''.join(ray)
+        if value:
+            ray = value.split('s')
+            seconds += int(ray.pop(0))
+        return seconds
+    except Exception:
+        pass
+    return original_value
+
+def hashivault_needs_refresh(old_data, min_ttl):
+     print("Checking refresh")
+     print_r(old_data)
+     return False
+#    if sorted(old_data.keys()) != sorted(new_data.keys()):
+#        return True
+#    for key in old_data:
+#        old_value = old_data[key]
+#        new_value = new_data[key]
+#        if old_value == new_value:
+#            continue
+#        if key != 'ttl' and key != 'max_ttl':
+#            return True
+#        old_value = _convert_to_seconds(old_value)
+#        new_value = _convert_to_seconds(new_value)
+#        if old_value != new_value:
+#            return True
+#    return False
+#
+def hashivault_changed(old_data, new_data):
+    if sorted(old_data.keys()) != sorted(new_data.keys()):
+        return True
+    for key in old_data:
+        old_value = old_data[key]
+        new_value = new_data[key]
+        if old_value == new_value:
+            continue
+        if key != 'ttl' and key != 'max_ttl':
+            return True
+        old_value = _convert_to_seconds(old_value)
+        new_value = _convert_to_seconds(new_value)
+        if old_value != new_value:
+            return True
+    return False
+
+
+from ansible.module_utils.hashivault import *
+
+
+@hashiwrapper
+def hashivault_write(module):
+    result = {"changed": False, "rc": 0}
+    params = module.params
+    client = hashivault_auth_client(params)
+    secret = params.get('secret')
+    force = params.get('force', False)
+    min_ttl = params.get('min_ttl', "100%")
+    returned_data = None
+    
+    if secret.startswith('/'):
+        secret = secret.lstrip('/')
+    #else:
+    #    secret = ('secret/%s' % secret)
+    data = params.get('data')
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        changed = True
+        write_data = data
+
+        if params.get('update') or module.check_mode:
+            # Do not move this read outside of the update
+            read_data = client.read(secret) or {}
+            read_data = read_data.get('data', {})
+
+            write_data = dict(read_data)
+            write_data.update(data)
+
+            result['write_data'] = write_data
+            result['read_data'] = read_data
+            changed = hashivault_changed(read_data, write_data)
+            if not changed:
+                changed = hashivault_needs_refresh(read_data, min_ttl)
+                
+        if changed:
+            if not module.check_mode:
+                returned_data = client.write((secret), **write_data)
+
+            if returned_data:
+                result['data'] = returned_data
+            result['msg'] = "Secret %s written" % secret
+        result['changed'] = changed
+    return result
+
+
+if __name__ == '__main__':
+    main()
+

roles/etcd/tasks/gen_certs_vault.yml

@@ -62,3 +62,5 @@
   with_items: "{{ etcd_node_certs_needed|d([]) }}"
   when: inventory_hostname in etcd_node_cert_hosts
   notify: set etcd_secret_changed
+
+- fail:

roles/etcd/tasks/sync_etcd_master_certs.yml

@@ -8,13 +8,13 @@
         "member-" + inventory_hostname + ".pem"
         ] }}
 
-- include_tasks: ../../vault/tasks/shared/sync_file.yml
+#- include_tasks: ../../vault/tasks/shared/sync_file.yml
-  vars:
+#  vars:
-    sync_file: "{{ item }}"
+#    sync_file: "{{ item }}"
-    sync_file_dir: "{{ etcd_cert_dir }}"
+#    sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: [ "{{ inventory_hostname }}" ]
+#    sync_file_hosts: [ "{{ inventory_hostname }}" ]
-    sync_file_is_cert: true
+#    sync_file_is_cert: true
-  with_items: "{{ etcd_master_cert_list|d([]) }}"
+#  with_items: "{{ etcd_master_cert_list|d([]) }}"
 
 - name: sync_etcd_certs | Set facts for etcd sync_file results
   set_fact:
@@ -22,16 +22,16 @@
   with_items: "{{ sync_file_results|d([]) }}"
   when: item.no_srcs|bool
 
-- name: sync_etcd_certs | Unset sync_file_results after etcd certs sync
+#- name: sync_etcd_certs | Unset sync_file_results after etcd certs sync
-  set_fact:
+#  set_fact:
-    sync_file_results: []
+#    sync_file_results: []
-
+#
-- include_tasks: ../../vault/tasks/shared/sync_file.yml
+#- include_tasks: ../../vault/tasks/shared/sync_file.yml
-  vars:
+#  vars:
-    sync_file: ca.pem
+#    sync_file: ca.pem
-    sync_file_dir: "{{ etcd_cert_dir }}"
+#    sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: [ "{{ inventory_hostname }}" ]
+#    sync_file_hosts: [ "{{ inventory_hostname }}" ]
-
+#
-- name: sync_etcd_certs | Unset sync_file_results after ca.pem sync
+#- name: sync_etcd_certs | Unset sync_file_results after ca.pem sync
-  set_fact:
+#  set_fact:
-    sync_file_results: []
+#    sync_file_results: []

roles/etcd/tasks/sync_etcd_node_certs.yml

@@ -4,30 +4,30 @@
   set_fact:
     etcd_node_cert_list: "{{ etcd_node_cert_list|default([]) +  ['node-' + inventory_hostname + '.pem'] }}"
 
-- include_tasks: ../../vault/tasks/shared/sync_file.yml
+#- include_tasks: ../../vault/tasks/shared/sync_file.yml
-  vars:
+#  vars:
-    sync_file: "{{ item }}"
+#    sync_file: "{{ item }}"
-    sync_file_dir: "{{ etcd_cert_dir }}"
+#    sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: [ "{{ inventory_hostname }}" ]
+#    sync_file_hosts: [ "{{ inventory_hostname }}" ]
-    sync_file_is_cert: true
+#    sync_file_is_cert: true
-  with_items: "{{ etcd_node_cert_list|d([]) }}"
+#  with_items: "{{ etcd_node_cert_list|d([]) }}"
-
+#
 - name: sync_etcd_node_certs | Set facts for etcd sync_file results
   set_fact:
     etcd_node_certs_needed: "{{ etcd_node_certs_needed|default([]) + [item.path] }}"
   with_items: "{{ sync_file_results|d([]) }}"
   when: item.no_srcs|bool
 
-- name: sync_etcd_node_certs | Unset sync_file_results after etcd node certs
+#- name: sync_etcd_node_certs | Unset sync_file_results after etcd node certs
-  set_fact:
+#  set_fact:
-    sync_file_results: []
+#    sync_file_results: []
-
+#
-- include_tasks: ../../vault/tasks/shared/sync_file.yml
+#- include_tasks: ../../vault/tasks/shared/sync_file.yml
-  vars:
+#  vars:
-    sync_file: ca.pem
+#    sync_file: ca.pem
-    sync_file_dir: "{{ etcd_cert_dir }}"
+#    sync_file_dir: "{{ etcd_cert_dir }}"
-    sync_file_hosts: "{{ groups['etcd'] }}"
+#    sync_file_hosts: "{{ groups['etcd'] }}"
-
+#
-- name: sync_etcd_node_certs | Unset sync_file_results after ca.pem
+#- name: sync_etcd_node_certs | Unset sync_file_results after ca.pem
-  set_fact:
+#  set_fact:
-    sync_file_results: []
+#    sync_file_results: []

roles/kubernetes/secrets/tasks/gen_certs_vault.yml

@@ -1,23 +1,22 @@
 ---
-- import_tasks: sync_kube_master_certs.yml
+#- import_tasks: sync_kube_master_certs.yml
-  when: inventory_hostname in groups['kube-master']
+#  when: inventory_hostname in groups['kube-master']
-
+#
-- import_tasks: sync_kube_node_certs.yml
+#- import_tasks: sync_kube_node_certs.yml
-  when: inventory_hostname in groups['k8s-cluster']
+#  when: inventory_hostname in groups['k8s-cluster']
 
 # Issue admin certs to kube-master hosts
 - include_tasks: ../../../vault/tasks/shared/issue_cert.yml
   vars:
     issue_cert_common_name: "admin"
-    issue_cert_copy_ca: "{{ item == kube_admin_certs_needed|first }}"
+    issue_cert_copy_ca: true
     issue_cert_file_group: "{{ kube_cert_group }}"
     issue_cert_file_owner: kube
     issue_cert_hosts: "{{ groups['kube-master'] }}"
-    issue_cert_path: "{{ item }}"
+    issue_cert_path: "{{ inventory_hostname }}"
     issue_cert_role: kube-master
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
     issue_cert_mount_path: "{{ kube_vault_mount_path }}"
-  with_items: "{{ kube_admin_certs_needed|d([]) }}"
   when: inventory_hostname in groups['kube-master']
 
 - name: gen_certs_vault | Set fact about certificate alt names
@@ -59,11 +58,10 @@
         {%- endif -%}
         "127.0.0.1","::1","{{ kube_apiserver_ip }}"
         ]
-    issue_cert_path: "{{ item }}"
+    issue_cert_path: "{{ inventory_hostname }}"
     issue_cert_role: kube-master
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
     issue_cert_mount_path: "{{ kube_vault_mount_path }}"
-  with_items: "{{ kube_master_components_certs_needed|d([]) }}"
   when: inventory_hostname in groups['kube-master']
   notify: set secret_changed
 
@@ -73,37 +71,33 @@
     # Need to strip out the 'node-' prefix from the cert name so it can be used
     # with the node authorization plugin ( CN matches kubelet node name )
     issue_cert_common_name: "system:node:{{ item.rsplit('/', 1)[1].rsplit('.', 1)[0] | regex_replace('^node-', '') }}"
-    issue_cert_copy_ca: "{{ item == kube_node_certs_needed|first }}"
+    issue_cert_copy_ca: yes
     issue_cert_file_group: "{{ kube_cert_group }}"
     issue_cert_file_owner: kube
     issue_cert_hosts: "{{ groups['k8s-cluster'] }}"
-    issue_cert_path: "{{ item }}"
+    issue_cert_path: "{{ inventory_hostname }}"
     issue_cert_role: kube-node
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
     issue_cert_mount_path: "{{ kube_vault_mount_path }}"
-  with_items: "{{ kube_node_certs_needed|d([]) }}"
-  when: inventory_hostname in groups['k8s-cluster']
 
 # Issue proxy certs to k8s-cluster nodes
 - include_tasks: ../../../vault/tasks/shared/issue_cert.yml
   vars:
     issue_cert_common_name: "system:kube-proxy"
-    issue_cert_copy_ca: "{{ item == kube_proxy_certs_needed|first }}"
+    issue_cert_copy_ca: true
     issue_cert_file_group: "{{ kube_cert_group }}"
     issue_cert_file_owner: kube
     issue_cert_hosts: "{{ groups['k8s-cluster'] }}"
-    issue_cert_path: "{{ item }}"
+    issue_cert_path: "{{ inventory_hostname }}"
     issue_cert_role: kube-proxy
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
     issue_cert_mount_path: "{{ kube_vault_mount_path }}"
-  with_items: "{{ kube_proxy_certs_needed|d([]) }}"
-  when: inventory_hostname in groups['k8s-cluster']
 
 # Issue front proxy cert to kube-master hosts
 - include_tasks: ../../../vault/tasks/shared/issue_cert.yml
   vars:
     issue_cert_common_name: "front-proxy-client"
-    issue_cert_copy_ca: "{{ item == kube_front_proxy_clients_certs_needed|first }}"
+    issue_cert_copy_ca: true
     issue_cert_ca_filename: front-proxy-ca.pem
     issue_cert_alt_names: "{{ kube_cert_alt_names }}"
     issue_cert_file_group: "{{ kube_cert_group }}"
@@ -124,10 +118,9 @@
         {%- endif -%}
         "127.0.0.1","::1","{{ kube_apiserver_ip }}"
         ]
-    issue_cert_path: "{{ item }}"
+    issue_cert_path: "{{ inventory_hostname }}"
     issue_cert_role: front-proxy-client
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
     issue_cert_mount_path: "{{ kube_vault_mount_path }}"
-  with_items: "{{ kube_front_proxy_clients_certs_needed|d([]) }}"
   when: inventory_hostname in groups['kube-master']
   notify: set secret_changed

roles/vault/tasks/shared/issue_cert.yml

@@ -76,7 +76,8 @@
   run_once: true
 
 - name: "issue_cert | Generate {{ issue_cert_path }} for {{ issue_cert_role }} role"
-  hashivault_write:
+  #hashivault_write:
+  vault_cert_issue:
     url: "{{ issue_cert_url }}"
     token: "{{ vault_client_token }}"
     ca_cert: "{% if 'https' in issue_cert_url %}{{ vault_cert_dir }}/ca.pem{% endif %}"