From 5457799aa3d45ad45c4e861b75fcbba0f0209438 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Tue, 13 Dec 2016 09:03:35 +0000 Subject: [PATCH] Individual etcd ssl certs Includes hooks for triggering calico, kubelet, and kube-apiserver restarts if etcd certs changed. --- roles/etcd/files/make-ssl-etcd.sh | 46 ++++++++++++++----- roles/etcd/tasks/check_certs.yml | 37 ++++++++++----- roles/etcd/tasks/gen_certs.yml | 39 +++++++++++++--- roles/etcd/tasks/main.yml | 3 +- roles/etcd/tasks/pre_upgrade.yml | 5 -- roles/etcd/templates/etcd.j2 | 8 ++-- .../templates/calico-policy-controller.yml.j2 | 4 +- roles/kubernetes/master/tasks/pre-upgrade.yml | 20 ++++++++ .../manifests/kube-apiserver.manifest.j2 | 5 +- roles/network_plugin/calico/tasks/main.yml | 17 ++++--- .../calico/templates/cni-calico.conf.j2 | 4 +- roles/network_plugin/canal/tasks/main.yml | 4 +- .../flannel/templates/flannel-pod.yml | 2 +- 13 files changed, 140 insertions(+), 54 deletions(-) diff --git a/roles/etcd/files/make-ssl-etcd.sh b/roles/etcd/files/make-ssl-etcd.sh index 4c7db9430..99e7dc307 100755 --- a/roles/etcd/files/make-ssl-etcd.sh +++ b/roles/etcd/files/make-ssl-etcd.sh @@ -16,7 +16,6 @@ set -o errexit set -o pipefail - usage() { cat << EOF @@ -61,20 +60,43 @@ cd "${tmpdir}" mkdir -p "${SSLDIR}" # Root CA -openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1 -openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1 +if [ -e "$SSLDIR/ca-key.pem" ]; then + # Reuse existing CA + cp $SSLDIR/{ca.pem,ca-key.pem} . +else + openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1 + openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1 +fi # ETCD member -openssl genrsa -out member-key.pem 2048 > /dev/null 2>&1 -openssl req -new -key member-key.pem -out member.csr -subj "/CN=etcd-member" -config ${CONFIG} > /dev/null 2>&1 -openssl x509 -req -in member.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 +if [ -n "$MASTERS" ]; then + for host in $MASTERS; do + openssl genrsa -out member-${host}-key.pem 2048 > /dev/null 2>&1 + openssl req -new -key member-${host}-key.pem -out member-${host}.csr -subj "/CN=etcd-member-${host}" -config ${CONFIG} > /dev/null 2>&1 + openssl x509 -req -in member-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-${host}.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 + done +else + openssl genrsa -out member-key.pem 2048 > /dev/null 2>&1 + openssl req -new -key member-key.pem -out member.csr -subj "/CN=etcd-member" -config ${CONFIG} > /dev/null 2>&1 + openssl x509 -req -in member.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 +fi -# Nodes and Admin -for i in node admin; do - openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1 - openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1 - openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 -done +# Node and admin keys +if [ -n "$HOSTS" ]; then + for host in $HOSTS; do + for i in node admin; do + openssl genrsa -out ${i}-${host}-key.pem 2048 > /dev/null 2>&1 + openssl req -new -key ${i}-${host}-key.pem -out ${i}-${host}.csr -subj "/CN=kube-${i}-${host}" > /dev/null 2>&1 + openssl x509 -req -in ${i}-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}-${host}.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 + done + done +else + for i in node admin; do + openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1 + openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1 + openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1 + done +fi # Install certs mv *.pem ${SSLDIR}/ diff --git a/roles/etcd/tasks/check_certs.yml b/roles/etcd/tasks/check_certs.yml index 03a875517..eeea8353e 100644 --- a/roles/etcd/tasks/check_certs.yml +++ b/roles/etcd/tasks/check_certs.yml @@ -1,36 +1,51 @@ --- -- name: "Check_certs | check if the certs have already been generated on first master" +- name: "Check_certs | check if all certs have already been generated on first master" stat: - path: "{{ etcd_cert_dir }}/ca.pem" + path: "{{ etcd_cert_dir }}/{{ item }}" delegate_to: "{{groups['etcd'][0]}}" register: etcdcert_master run_once: true + with_items: >- + ['ca.pem', + {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|unique %} + {% for host in all_etcd_hosts %} + 'node-{{ host }}-key.pem' + {% if not loop.last %}{{','}}{% endif %} + {% endfor %}] -- name: "Check_certs | Set default value for 'sync_certs' and 'gen_certs' to false" +- name: "Check_certs | Set default value for 'sync_certs', 'gen_certs' and 'etcd_secret_changed' to false" set_fact: sync_certs: false gen_certs: false + etcd_secret_changed: false -- name: "Check_certs | Set 'sync_certs' and 'gen_certs' to true" +- name: "Check_certs | Set 'gen_certs' to true" set_fact: gen_certs: true - when: not etcdcert_master.stat.exists + when: "not {{item.stat.exists}}" run_once: true + with_items: "{{etcdcert_master.results}}" - name: "Check certs | check if a cert already exists" stat: - path: "{{ etcd_cert_dir }}/ca.pem" + path: "{{ etcd_cert_dir }}/node-{{inventory_hostname}}-key.pem" register: etcdcert + with_items: + - ca.pem + - node-{{ inventory_hostname }}-key.pem - name: "Check_certs | Set 'sync_certs' to true" set_fact: sync_certs: true when: >- {%- set certs = {'sync': False} -%} - {%- for server in play_hosts - if (not hostvars[server].etcdcert.stat.exists|default(False)) or - (hostvars[server].etcdcert.stat.checksum|default('') != etcdcert_master.stat.checksum|default('')) -%} - {%- set _ = certs.update({'sync': True}) -%} + {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|unique %} + {% for host in all_etcd_hosts %} + {% if host == inventory_hostname %} + {% if (not etcdcert.results[0].stat.exists|default(False)) or + (etcdcert.results[1].stat.checksum|default('') != etcdcert_master.results[loop.index].stat.checksum|default('')) -%} + {%- set _ = certs.update({'sync': True}) -%} + {% endif %} + {% endif %} {%- endfor -%} {{ certs.sync }} - run_once: true diff --git a/roles/etcd/tasks/gen_certs.yml b/roles/etcd/tasks/gen_certs.yml index a12c63ac8..29e6ee318 100644 --- a/roles/etcd/tasks/gen_certs.yml +++ b/roles/etcd/tasks/gen_certs.yml @@ -34,29 +34,56 @@ - name: Gen_certs | run cert generation script command: "{{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}" + environment: + - MASTERS: "{% for m in groups['etcd'] %} + {% if hostvars[m].sync_certs|default(false) %} + {{ m }} + {% endif %} + {% endfor %}" + - HOSTS: "{% for h in groups['k8s-cluster'] %} + {% if hostvars[h].sync_certs|default(false) %} + {{ h }} + {% endif %} + {% endfor %}" run_once: yes delegate_to: "{{groups['etcd'][0]}}" when: gen_certs|default(false) notify: set etcd_secret_changed - set_fact: - master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'member.pem', 'member-key.pem'] - node_certs: ['ca.pem', 'node.pem', 'node-key.pem'] + all_master_certs: "['ca-key.pem', + {% for node in groups['etcd'] %} + 'admin-{{ node }}.pem', + 'admin-{{ node }}-key.pem', + 'member-{{ node }}.pem', + 'member-{{ node }}-key.pem', + {% endfor %}]" + my_master_certs: ['ca-key.pem', + 'admin-{{ inventory_hostname }}.pem', + 'admin-{{ inventory_hostname }}-key.pem', + 'member-{{ inventory_hostname }}.pem', + 'member-{{ inventory_hostname }}-key.pem' + ] + all_node_certs: "['ca.pem', + {% for node in groups['k8s-cluster'] %} + 'node-{{ node }}.pem', + 'node-{{ node }}-key.pem', + {% endfor %}]" + my_node_certs: ['ca.pem', 'node-{{ inventory_hostname }}.pem', 'node-{{ inventory_hostname }}-key.pem'] tags: facts - name: Gen_certs | Gather etcd master certs - shell: "tar cfz - -C {{ etcd_cert_dir }} {{ master_certs|join(' ') }} {{ node_certs|join(' ') }}| base64 --wrap=0" + shell: "tar cfz - -C {{ etcd_cert_dir }} {{ my_master_certs|join(' ') }} {{ all_node_certs|join(' ') }}| base64 --wrap=0" register: etcd_master_cert_data delegate_to: "{{groups['etcd'][0]}}" - run_once: true + #run_once: true when: sync_certs|default(false) notify: set etcd_secret_changed - name: Gen_certs | Gather etcd node certs - shell: "tar cfz - -C {{ etcd_cert_dir }} {{ node_certs|join(' ') }} | base64 --wrap=0" + shell: "tar cfz - -C {{ etcd_cert_dir }} {{ my_node_certs|join(' ') }} | base64 --wrap=0" register: etcd_node_cert_data delegate_to: "{{groups['etcd'][0]}}" - run_once: true when: sync_certs|default(false) notify: set etcd_secret_changed diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index 5b25a8392..d6320619c 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -22,10 +22,11 @@ enabled: yes when: is_etcd_master -- name: Restart etcd if binary changed +- name: Restart etcd if binary or certs changed command: /bin/true notify: restart etcd when: etcd_deployment_type == "host" and etcd_copy.stdout_lines and is_etcd_master + or etcd_secret_changed|default(false) # Reload systemd before starting service - meta: flush_handlers diff --git a/roles/etcd/tasks/pre_upgrade.yml b/roles/etcd/tasks/pre_upgrade.yml index b7574d9f2..448366295 100644 --- a/roles/etcd/tasks/pre_upgrade.yml +++ b/roles/etcd/tasks/pre_upgrade.yml @@ -34,10 +34,6 @@ command: "docker rm -f {{item}}" with_items: "{{etcd_proxy_container.stdout_lines}}" -- name: "Pre-upgrade | remove etcd-proxy if it exists" - command: "docker rm -f {{item}}" - with_items: "{{etcd_proxy_container.stdout_lines}}" - - name: "Pre-upgrade | check if member list is non-SSL" command: "{{ bin_dir }}/etcdctl --no-sync --peers={{ etcd_access_addresses | regex_replace('https','http') }} member list" register: etcd_member_list @@ -53,4 +49,3 @@ awk -F"[: =]" '{print "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses | regex_replace('https','http') }} member update "$1" https:"$7":"$8}' | bash run_once: true when: 'etcd_member_list is defined and "http://" in etcd_member_list.stdout' - diff --git a/roles/etcd/templates/etcd.j2 b/roles/etcd/templates/etcd.j2 index 0b7e1eb9f..cce0aa2e9 100644 --- a/roles/etcd/templates/etcd.j2 +++ b/roles/etcd/templates/etcd.j2 @@ -13,9 +13,9 @@ ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }} # TLS settings ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem -ETCD_CERT_FILE={{ etcd_cert_dir }}/node.pem -ETCD_KEY_FILE={{ etcd_cert_dir }}/node-key.pem +ETCD_CERT_FILE={{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem +ETCD_KEY_FILE={{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem -ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member.pem -ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-key.pem +ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem +ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem ETCD_PEER_CLIENT_CERT_AUTH=true diff --git a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 index 1bc553316..eb98267f3 100644 --- a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 @@ -31,9 +31,9 @@ spec: - name: ETCD_CA_CERT_FILE value: "{{ etcd_cert_dir }}/ca.pem" - name: ETCD_CERT_FILE - value: "{{ etcd_cert_dir }}/node.pem" + value: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - name: ETCD_KEY_FILE - value: "{{ etcd_cert_dir }}/node-key.pem" + value: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" # Location of the Kubernetes API - this shouldn't need to be # changed so long as it is used in conjunction with # CONFIGURE_ETC_HOSTS="true". diff --git a/roles/kubernetes/master/tasks/pre-upgrade.yml b/roles/kubernetes/master/tasks/pre-upgrade.yml index cf7d2bea9..8c6bf7bb1 100644 --- a/roles/kubernetes/master/tasks/pre-upgrade.yml +++ b/roles/kubernetes/master/tasks/pre-upgrade.yml @@ -27,3 +27,23 @@ - /etc/systemd/system/kube-apiserver.service - /etc/init.d/kube-apiserver tags: kube-apiserver + +- name: "Pre-upgrade | See if kube-apiserver manifest exists" + stat: + path: /etc/kubernetes/manifests/kube-apiserver.manifest + register: kube_apiserver_manifest + when: secret_changed|default(false) or etcd_secret_changed|default(false) + +- name: "Pre-upgrade | Write invalid image to kube-apiserver manifest if secrets were changed" + replace: + dest: /etc/kubernetes/manifests/kube-apiserver.manifest + regexp: '(\s+)image:\s+.*?$' + replace: '\1image: kill.apiserver.using.fake.image.in:manifest' + register: kube_apiserver_manifest_replaced + when: (secret_changed|default(false) or etcd_secret_changed|default(false)) and kube_apiserver_manifest.stat.exists + +- name: "Pre-upgrade | Pause while waiting for kubelet to delete kube-apiserver pod" + pause: seconds=20 + when: (secret_changed|default(false) or etcd_secret_changed|default(false)) and kube_apiserver_manifest.stat.exists + tags: kube-apiserver + diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 4f4a92bf3..65505526d 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -5,6 +5,7 @@ metadata: namespace: {{system_namespace}} labels: k8s-app: kube-apiserver + kargo: v2 spec: hostNetwork: true containers: @@ -18,8 +19,8 @@ spec: - --etcd-servers={{ etcd_access_endpoint }} - --etcd-quorum-read=true - --etcd-cafile={{ etcd_cert_dir }}/ca.pem - - --etcd-certfile={{ etcd_cert_dir }}/node.pem - - --etcd-keyfile={{ etcd_cert_dir }}/node-key.pem + - --etcd-certfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem + - --etcd-keyfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem - --insecure-bind-address={{ kube_apiserver_insecure_bind_address }} - --apiserver-count={{ kube_apiserver_count }} - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index 19d74759c..81979a910 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -27,8 +27,8 @@ force: yes with_items: - {s: "ca.pem", d: "ca_cert.crt"} - - {s: "node.pem", d: "cert.crt"} - - {s: "node-key.pem", d: "key.pem"} + - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"} + - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"} - name: Calico | Install calicoctl container script template: @@ -72,8 +72,8 @@ command: |- curl \ --cacert {{ etcd_cert_dir }}/ca.pem \ - --cert {{ etcd_cert_dir}}/admin.pem \ - --key {{ etcd_cert_dir }}/admin-key.pem \ + --cert {{ etcd_cert_dir}}/admin-{{ groups['etcd'][0] }}.pem \ + --key {{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}-key.pem \ https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool register: calico_conf delegate_to: "{{groups['etcd'][0]}}" @@ -131,8 +131,8 @@ command: |- curl \ --cacert {{ etcd_cert_dir }}/ca.pem \ - --cert {{ etcd_cert_dir}}/admin.pem \ - --key {{ etcd_cert_dir }}/admin-key.pem \ + --cert {{ etcd_cert_dir}}/admin-{{ groups['etcd'][0] }}.pem \ + --key {{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}-key.pem \ https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool register: calico_pools_raw delegate_to: "{{groups['etcd'][0]}}" @@ -191,6 +191,11 @@ when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "RedHat" notify: restart calico-node +- name: Calico | Restart calico-node if secrets changed + command: /bin/true + when: secret_changed|default(false) or etcd_secret_changed|default(false) + notify: restart calico-node + - meta: flush_handlers - name: Calico | Enable calico-node diff --git a/roles/network_plugin/calico/templates/cni-calico.conf.j2 b/roles/network_plugin/calico/templates/cni-calico.conf.j2 index 8a3016324..2834e4cba 100644 --- a/roles/network_plugin/calico/templates/cni-calico.conf.j2 +++ b/roles/network_plugin/calico/templates/cni-calico.conf.j2 @@ -5,8 +5,8 @@ {% endif %} "type": "calico", "etcd_endpoints": "{{ etcd_access_endpoint }}", - "etcd_cert_file": "{{ etcd_cert_dir }}/node.pem", - "etcd_key_file": "{{ etcd_cert_dir }}/node-key.pem", + "etcd_cert_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem", + "etcd_key_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem", "etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem", "log_level": "info", "ipam": { diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml index d968e9e46..fec09cb48 100644 --- a/roles/network_plugin/canal/tasks/main.yml +++ b/roles/network_plugin/canal/tasks/main.yml @@ -21,8 +21,8 @@ force: yes with_items: - {s: "ca.pem", d: "ca_cert.crt"} - - {s: "node.pem", d: "cert.crt"} - - {s: "node-key.pem", d: "key.pem"} + - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"} + - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"} - name: Canal | Set Flannel etcd configuration command: |- diff --git a/roles/network_plugin/flannel/templates/flannel-pod.yml b/roles/network_plugin/flannel/templates/flannel-pod.yml index 1af2152ea..2edd9ada1 100644 --- a/roles/network_plugin/flannel/templates/flannel-pod.yml +++ b/roles/network_plugin/flannel/templates/flannel-pod.yml @@ -22,7 +22,7 @@ command: - "/bin/sh" - "-c" - - "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ etcd_cert_dir }}/ca.pem -etcd-certfile {{ etcd_cert_dir }}/node.pem -etcd-keyfile {{ etcd_cert_dir }}/node-key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}" + - "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ etcd_cert_dir }}/ca.pem -etcd-certfile {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem -etcd-keyfile {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}" ports: - hostPort: 10253 containerPort: 10253