From ed253e5c5a8dd35cbae540b0a8ef58beb2406c1c Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Tue, 27 Dec 2016 14:02:45 +0300 Subject: [PATCH] Generate individual certificates for k8s hosts --- .../manifests/kube-apiserver.manifest.j2 | 6 +-- .../kube-controller-manager.manifest.j2 | 2 +- .../node/templates/node-kubeconfig.yaml.j2 | 4 +- roles/kubernetes/secrets/files/make-ssl.sh | 47 ++++++++++++------ .../kubernetes/secrets/tasks/check-certs.yml | 33 +++++++++---- roles/kubernetes/secrets/tasks/gen_certs.yml | 48 +++++++++++++++---- 6 files changed, 101 insertions(+), 39 deletions(-) diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index c05030697..fe9a49fa8 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -35,10 +35,10 @@ spec: - --service-node-port-range={{ kube_apiserver_node_port_range }} - --client-ca-file={{ kube_cert_dir }}/ca.pem - --basic-auth-file={{ kube_users_dir }}/known_users.csv - - --tls-cert-file={{ kube_cert_dir }}/apiserver.pem - - --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem + - --tls-cert-file={{ kube_cert_dir }}/apiserver-{{ inventory_hostname }}.pem + - --tls-private-key-file={{ kube_cert_dir }}/apiserver-{{ inventory_hostname }}-key.pem - --token-auth-file={{ kube_token_dir }}/known_tokens.csv - - --service-account-key-file={{ kube_cert_dir }}/apiserver-key.pem + - --service-account-key-file={{ kube_cert_dir }}/apiserver-{{ inventory_hostname }}-key.pem - --secure-port={{ kube_apiserver_port }} - --insecure-port={{ kube_apiserver_insecure_port }} {% if kube_api_runtime_config is defined %} diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 49dd05ba8..c604587af 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -23,7 +23,7 @@ spec: - controller-manager - --master={{ kube_apiserver_endpoint }} - --leader-elect=true - - --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem + - --service-account-private-key-file={{ kube_cert_dir }}/apiserver-{{ inventory_hostname }}-key.pem - --root-ca-file={{ kube_cert_dir }}/ca.pem - --cluster-signing-cert-file={{ kube_cert_dir }}/ca.pem - --cluster-signing-key-file={{ kube_cert_dir }}/ca-key.pem diff --git a/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 index e1593303d..0951f2342 100644 --- a/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 +++ b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 @@ -8,8 +8,8 @@ clusters: users: - name: kubelet user: - client-certificate: {{ kube_cert_dir }}/node.pem - client-key: {{ kube_cert_dir }}/node-key.pem + client-certificate: {{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem + client-key: {{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem contexts: - context: cluster: local diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh index f90fb7e8b..422bde6cf 100755 --- a/roles/kubernetes/secrets/files/make-ssl.sh +++ b/roles/kubernetes/secrets/files/make-ssl.sh @@ -27,8 +27,11 @@ Usage : $(basename $0) -f [-d ] -f | --config : Openssl configuration file -d | --ssldir : Directory where the certificates will be installed - ex : - $(basename $0) -f openssl.conf -d /srv/ssl + Environmental variables MASTERS and HOSTS should be set to generate keys + for each host. + + ex : + MASTERS=node1 HOSTS="node1 node2" $(basename $0) -f openssl.conf -d /srv/ssl EOF } @@ -61,21 +64,37 @@ cd "${tmpdir}" mkdir -p "${SSLDIR}" # Root CA -openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1 -openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" > /dev/null 2>&1 +if [ -e "$SSLDIR/ca-key.pem" ]; then + # Reuse existing CA + cp $SSLDIR/{ca.pem,ca-key.pem} . +else + openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1 + openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" > /dev/null 2>&1 +fi -# Apiserver -openssl genrsa -out apiserver-key.pem 2048 > /dev/null 2>&1 -openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config ${CONFIG} > /dev/null 2>&1 -openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1 -cat ca.pem >> apiserver.pem +if [ -n "$MASTERS" ]; then + for host in $MASTERS; do + # kube-apiserver key + openssl genrsa -out apiserver-${host}-key.pem 2048 > /dev/null 2>&1 + openssl req -new -key apiserver-${host}-key.pem -out apiserver-${host}.csr -subj "/CN=kube-apiserver-${host}" -config ${CONFIG} > /dev/null 2>&1 + openssl x509 -req -in apiserver-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver-${host}.pem -days 365 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1 + cat ca.pem >> apiserver-${host}.pem + + # admin key + openssl genrsa -out admin-${host}-key.pem 2048 > /dev/null 2>&1 + openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=kube-admin-${host}" > /dev/null 2>&1 + openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days 365 > /dev/null 2>&1 + done +fi # Nodes and Admin -for i in node admin; do - openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1 - openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1 - openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 > /dev/null 2>&1 -done +if [ -n "$HOSTS" ]; then + for host in $HOSTS; do + openssl genrsa -out node-${host}-key.pem 2048 > /dev/null 2>&1 + openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=kube-node-${host}" > /dev/null 2>&1 + openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days 365 > /dev/null 2>&1 + done +fi # Install certs mv *.pem ${SSLDIR}/ diff --git a/roles/kubernetes/secrets/tasks/check-certs.yml b/roles/kubernetes/secrets/tasks/check-certs.yml index 97c6f7226..aa3e47754 100644 --- a/roles/kubernetes/secrets/tasks/check-certs.yml +++ b/roles/kubernetes/secrets/tasks/check-certs.yml @@ -1,36 +1,49 @@ --- - name: "Check_certs | check if the certs have already been generated on first master" stat: - path: "{{ kube_cert_dir }}/ca.pem" + path: "{{ kube_cert_dir }}/{{ item }}" delegate_to: "{{groups['kube-master'][0]}}" register: kubecert_master run_once: true + with_items: >- + ['ca.pem', + {% for host in groups['k8s-cluster'] %} + 'node-{{ host }}-key.pem' + {% if not loop.last %}{{','}}{% endif %} + {% endfor %}] -- name: "Check_certs | Set default value for 'sync_certs' and 'gen_certs' to false" +- name: "Check_certs | Set default value for 'sync_certs', 'gen_certs', and 'secret_changed' to false" set_fact: sync_certs: false gen_certs: false + secret_changed: false -- name: "Check_certs | Set 'sync_certs' and 'gen_certs' to true" +- name: "Check_certs | Set 'gen_certs' to true" set_fact: gen_certs: true - when: not kubecert_master.stat.exists + when: "not {{ item.stat.exists }}" run_once: true + with_items: "{{ kubecert_master.results }}" - name: "Check certs | check if a cert already exists" stat: - path: "{{ kube_cert_dir }}/ca.pem" + path: "{{ kube_cert_dir }}/{{ item }}" register: kubecert + with_items: + - ca.pem + - node-{{ inventory_hostname }}-key.pem - name: "Check_certs | Set 'sync_certs' to true" set_fact: sync_certs: true when: >- {%- set certs = {'sync': False} -%} - {%- for server in play_hosts - if (not hostvars[server].kubecert.stat.exists|default(False)) or - (hostvars[server].kubecert.stat.checksum|default('') != kubecert_master.stat.checksum|default('')) -%} - {%- set _ = certs.update({'sync': True}) -%} + {%- for host in groups['k8s-cluster'] %} + {% if host == inventory_hostname %} + {% if (not kubecert.results[0].stat.exists|default(False)) or + (kubecert.results[1].stat.checksum|default('') != kubecert_master.results[loop.index].stat.checksum|default('')) -%} + {%- set _ = certs.update({'sync': True}) -%} + {% endif %} + {% endif %} {%- endfor -%} {{ certs.sync }} - run_once: true diff --git a/roles/kubernetes/secrets/tasks/gen_certs.yml b/roles/kubernetes/secrets/tasks/gen_certs.yml index 22116fd3d..bd652e340 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs.yml @@ -18,35 +18,65 @@ - name: Gen_certs | run cert generation script command: "{{ kube_script_dir }}/make-ssl.sh -f {{ kube_config_dir }}/openssl.conf -d {{ kube_cert_dir }}" + environment: + - MASTERS: "{% for m in groups['kube-master'] %} + {% if hostvars[m].sync_certs|default(true) %} + {{ m }} + {% endif %} + {% endfor %}" + - HOSTS: "{% for h in groups['k8s-cluster'] %} + {% if hostvars[h].sync_certs|default(true) %} + {{ h }} + {% endif %} + {% endfor %}" run_once: yes delegate_to: "{{groups['kube-master'][0]}}" when: gen_certs|default(false) notify: set secret_changed - set_fact: - master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'apiserver-key.pem', 'apiserver.pem'] - node_certs: ['ca.pem', 'node.pem', 'node-key.pem'] + all_master_certs: "['ca-key.pem', + {% for node in groups['kube-master'] %} + 'admin-{{ node }}.pem', + 'admin-{{ node }}-key.pem', + 'apiserver-{{ node }}.pem', + 'apiserver-{{ node }}-key.pem', + {% endfor %}]" + my_master_certs: ['ca-key.pem', + 'admin-{{ inventory_hostname }}.pem', + 'admin-{{ inventory_hostname }}-key.pem', + 'apiserver-{{ inventory_hostname }}.pem', + 'apiserver-{{ inventory_hostname }}-key.pem' + ] + all_node_certs: "['ca.pem', + {% for node in groups['k8s-cluster'] %} + 'node-{{ node }}.pem', + 'node-{{ node }}-key.pem', + {% endfor %}]" + my_node_certs: ['ca.pem', 'node-{{ inventory_hostname }}.pem', 'node-{{ inventory_hostname }}-key.pem'] tags: facts - name: Gen_certs | Gather master certs - shell: "tar cfz - -C {{ kube_cert_dir }} {{ master_certs|join(' ') }} {{ node_certs|join(' ') }} | base64 --wrap=0" + shell: "tar cfz - -C {{ kube_cert_dir }} {{ my_master_certs|join(' ') }} {{ all_node_certs|join(' ') }} | base64 --wrap=0" register: master_cert_data delegate_to: "{{groups['kube-master'][0]}}" - run_once: true - when: sync_certs|default(false) + when: inventory_hostname in groups['kube-master'] and sync_certs|default(false) and + inventory_hostname != groups['kube-master'][0] - name: Gen_certs | Gather node certs - shell: "tar cfz - -C {{ kube_cert_dir }} {{ node_certs|join(' ') }} | base64 --wrap=0" + shell: "tar cfz - -C {{ kube_cert_dir }} {{ my_node_certs|join(' ') }} | base64 --wrap=0" register: node_cert_data delegate_to: "{{groups['kube-master'][0]}}" - run_once: true - when: sync_certs|default(false) + when: inventory_hostname in groups['kube-node'] and + sync_certs|default(false) and + inventory_hostname != groups['kube-master'][0] - name: Gen_certs | Copy certs on masters shell: "echo '{{master_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ kube_cert_dir }}" changed_when: false when: inventory_hostname in groups['kube-master'] and sync_certs|default(false) and inventory_hostname != groups['kube-master'][0] + notify: set secret_changed - name: Gen_certs | Copy certs on nodes shell: "echo '{{node_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ kube_cert_dir }}" @@ -54,6 +84,7 @@ when: inventory_hostname in groups['kube-node'] and sync_certs|default(false) and inventory_hostname != groups['kube-master'][0] + notify: set secret_changed - name: Gen_certs | check certificate permissions file: @@ -93,4 +124,3 @@ - name: Gen_certs | update ca-certificates (RedHat) command: update-ca-trust extract when: kube_ca_cert.changed and ansible_os_family == "RedHat" -