diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a53edd83..a7e5130c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -367,15 +367,14 @@ before_script: KUBELET_DEPLOYMENT: rkt STARTUP_SCRIPT: "" -#Note(mattymo): Vault deployment is broken and needs work -#.ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables -## stage: deploy-gce-part1 -# KUBE_NETWORK_PLUGIN: canal -# CERT_MGMT: vault -# CLOUD_IMAGE: ubuntu-1604-xenial -# CLOUD_REGION: us-central1-b -# CLUSTER_MODE: separate -# STARTUP_SCRIPT: "" +.ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables +# stage: deploy-gce-part1 + KUBE_NETWORK_PLUGIN: canal + CERT_MGMT: vault + CLOUD_IMAGE: ubuntu-1604-xenial + CLOUD_REGION: us-central1-b + CLUSTER_MODE: separate + STARTUP_SCRIPT: "" .ubuntu_flannel_rbac_variables: &ubuntu_flannel_rbac_variables # stage: deploy-gce-special @@ -600,17 +599,16 @@ ubuntu-rkt-sep: except: ['triggers'] only: ['master', /^pr-.*$/] -#Note(mattymo): Vault deployment is broken (https://github.com/kubernetes-incubator/kubespray/issues/1545) -#ubuntu-vault-sep: -# stage: deploy-gce-part1 -# <<: *job -# <<: *gce -# variables: -# <<: *gce_variables -# <<: *ubuntu_vault_sep_variables -# when: manual -# except: ['triggers'] -# only: ['master', /^pr-.*$/] +ubuntu-vault-sep: + stage: deploy-gce-part1 + <<: *job + <<: *gce + variables: + <<: *gce_variables + <<: *ubuntu_vault_sep_variables + when: manual + except: ['triggers'] + only: ['master', /^pr-.*$/] ubuntu-flannel-rbac-sep: stage: deploy-gce-special diff --git a/roles/etcd/tasks/gen_certs_vault.yml b/roles/etcd/tasks/gen_certs_vault.yml index 75be6c37c..a0bf6cfdc 100644 --- a/roles/etcd/tasks/gen_certs_vault.yml +++ b/roles/etcd/tasks/gen_certs_vault.yml @@ -58,6 +58,9 @@ [ {%- for host in groups.etcd -%} "{{ hostvars[host]['ansible_default_ipv4']['address'] }}", + {%- if hostvars[host]['ip'] is defined -%} + "{{ hostvars[host]['ip'] }}", + {%- endif -%} {%- endfor -%} "127.0.0.1","::1" ] @@ -81,6 +84,9 @@ [ {%- for host in etcd_node_cert_hosts -%} "{{ hostvars[host]['ansible_default_ipv4']['address'] }}", + {%- if hostvars[host]['ip'] is defined -%} + "{{ hostvars[host]['ip'] }}", + {%- endif -%} {%- endfor -%} "127.0.0.1","::1" ] diff --git a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml index ac8e128b4..e516db0f2 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml @@ -10,16 +10,16 @@ - name: gen_certs_vault | Read in the local credentials command: cat /etc/vault/roles/kube/userpass register: kube_vault_creds_cat - when: inventory_hostname == groups['k8s-cluster']|first + delegate_to: "{{ groups['k8s-cluster'][0] }}" - name: gen_certs_vault | Set facts for read Vault Creds set_fact: - kube_vault_creds: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_creds_cat']['stdout'] | from_json }}" - when: inventory_hostname == groups['k8s-cluster']|first + kube_vault_creds: "{{ kube_vault_creds_cat.stdout|from_json }}" + delegate_to: "{{ groups['k8s-cluster'][0] }}" - name: gen_certs_vault | Log into Vault and obtain an token uri: - url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ kube_vault_creds.username }}" + url: "{{ hostvars[groups['vault'][0]]['vault_leader_url'] }}/v1/auth/userpass/login/{{ kube_vault_creds.username }}" headers: Accept: application/json Content-Type: application/json @@ -28,14 +28,15 @@ body: password: "{{ kube_vault_creds.password }}" register: kube_vault_login_result - when: inventory_hostname == groups['k8s-cluster']|first + delegate_to: "{{ groups['k8s-cluster'][0] }}" - name: gen_certs_vault | Set fact for Vault API token set_fact: kube_vault_headers: Accept: application/json Content-Type: application/json - X-Vault-Token: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_login_result']['json']['auth']['client_token'] }}" + X-Vault-Token: "{{ kube_vault_login_result.get('json',{}).get('auth', {}).get('client_token') }}" + run_once: true # Issue certs to kube-master nodes - include: ../../../vault/tasks/shared/issue_cert.yml @@ -67,13 +68,16 @@ [ {%- for host in groups['kube-master'] -%} "{{ hostvars[host]['ansible_default_ipv4']['address'] }}", + {%- if hostvars[host]['ip'] is defined -%} + "{{ hostvars[host]['ip'] }}", + {%- endif -%} {%- endfor -%} "127.0.0.1","::1","{{ kube_apiserver_ip }}" ] issue_cert_path: "{{ item }}" issue_cert_role: kube issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" - with_items: "{{ kube_api_certs_needed|d([]) }}" + with_items: "{{ kube_master_components_certs_needed|d([]) }}" when: inventory_hostname in groups['kube-master'] # Issue node certs to k8s-cluster nodes @@ -89,3 +93,16 @@ issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" with_items: "{{ kube_node_certs_needed|d([]) }}" when: inventory_hostname in groups['k8s-cluster'] + +- include: ../../../vault/tasks/shared/issue_cert.yml + vars: + issue_cert_copy_ca: "{{ item == kube_proxy_certs_needed|first }}" + issue_cert_file_group: "{{ kube_cert_group }}" + issue_cert_file_owner: kube + issue_cert_headers: "{{ kube_vault_headers }}" + issue_cert_hosts: "{{ groups['k8s-cluster'] }}" + issue_cert_path: "{{ item }}" + issue_cert_role: kube + issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}" + with_items: "{{ kube_proxy_certs_needed|d([]) }}" + when: inventory_hostname in groups['k8s-cluster'] diff --git a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml index b02120ccb..6fa861a36 100644 --- a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml +++ b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml @@ -27,24 +27,24 @@ - include: ../../../vault/tasks/shared/sync_file.yml vars: - sync_file: "apiserver.pem" + sync_file: "{{ item }}" sync_file_dir: "{{ kube_cert_dir }}" sync_file_group: "{{ kube_cert_group }}" sync_file_hosts: "{{ groups['kube-master'] }}" sync_file_is_cert: true sync_file_owner: kube + with_items: ["apiserver.pem", "kube-scheduler.pem", "kube-controller-manager.pem"] -- name: sync_kube_master_certs | Set facts for apiserver sync_file results +- name: sync_kube_master_certs | Set facts for kube master components sync_file results set_fact: - kube_api_certs_needed: "{{ item.path }}" + kube_master_components_certs_needed: "{{ kube_master_components_certs_needed|d([]) + [item.path] }}" with_items: "{{ sync_file_results|d([]) }}" - when: item.no_srcs + when: item.no_srcs|bool -- name: sync_kube_master_certs | Unset sync_file_results after apiserver cert +- name: sync_kube_master_certs | Unset sync_file_results after kube master components cert set_fact: sync_file_results: [] - - include: ../../../vault/tasks/shared/sync_file.yml vars: sync_file: ca.pem diff --git a/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml b/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml index 884f6c436..b97b85e17 100644 --- a/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml +++ b/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml @@ -36,3 +36,27 @@ - name: sync_kube_node_certs | Unset sync_file_results after ca.pem set_fact: sync_file_results: [] + +- name: sync_kube_node_certs | Create list of needed kube-proxy certs + set_fact: + kube_proxy_cert_list: "{{ kube_proxy_cert_list|default([]) + ['kube-proxy-' + item + '.pem'] }}" + with_items: "{{ groups['k8s-cluster'] }}" + +- include: ../../../vault/tasks/shared/sync_file.yml + vars: + sync_file: "{{ item }}" + sync_file_dir: "{{ kube_cert_dir }}" + sync_file_group: "{{ kube_cert_group }}" + sync_file_hosts: "{{ groups['k8s-cluster'] }}" + sync_file_owner: kube + with_items: "{{ kube_proxy_cert_list|default([]) }}" + +- name: sync_kube_node_certs | Set facts for kube-proxy sync_file results + set_fact: + kube_proxy_certs_needed: "{{ kube_proxy_certs_needed|default([]) + [item.path] }}" + with_items: "{{ sync_file_results|d([]) }}" + when: item.no_srcs|bool + +- name: sync_kube_node_certs | Unset sync_file_results after kube proxy certs + set_fact: + sync_file_results: [] diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index 77140ba6a..e13065404 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -90,6 +90,7 @@ - /usr/local/share/ca-certificates/etcd-ca.crt - /etc/ssl/certs/kube-ca.pem - /etc/ssl/certs/etcd-ca.pem + - /etc/vault - /var/log/pods/ - "{{ bin_dir }}/kubelet" - "{{ bin_dir }}/etcd-scripts" @@ -102,7 +103,6 @@ - "{{ bin_dir }}/weave" tags: ['files'] - - name: reset | remove dns settings from dhclient.conf blockinfile: dest: "{{ item }}" diff --git a/roles/vault/defaults/main.yml b/roles/vault/defaults/main.yml index 4a0a60982..9694599cb 100644 --- a/roles/vault/defaults/main.yml +++ b/roles/vault/defaults/main.yml @@ -54,7 +54,7 @@ vault_download_vars: unarchive: true url: "{{ vault_download_url }}" version: "{{ vault_version }}" -vault_etcd_url: "https://{{ hostvars[groups.etcd[0]]['ansible_default_ipv4']['address'] }}:2379" +vault_etcd_url: "https://{{ hostvars[groups.etcd[0]]['ip']|d(hostvars[groups.etcd[0]]['ansible_default_ipv4']['address']) }}:2379" vault_image_repo: "vault" vault_image_tag: "{{ vault_version }}" vault_log_dir: "/var/log/vault" diff --git a/roles/vault/tasks/shared/issue_cert.yml b/roles/vault/tasks/shared/issue_cert.yml index 11f7abb41..4854e8b9e 100644 --- a/roles/vault/tasks/shared/issue_cert.yml +++ b/roles/vault/tasks/shared/issue_cert.yml @@ -5,6 +5,7 @@ # Vars: # issue_cert_alt_name: Requested Subject Alternative Names, in a list. # issue_cert_common_name: Common Name included in the cert +# issue_cert_copy_ca: Copy issuing CA cert needed # issue_cert_dir_mode: Mode of the placed cert directory # issue_cert_file_group: Group of the placed cert file and directory # issue_cert_file_mode: Mode of the placed cert file