Merge pull request #1948 from sgmitchell/secured-etcd

Enable etcd secure client to prevent etcdctl access without cert and key

roles/etcd/defaults/main.yml

@@ -37,3 +37,6 @@ etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr',
 etcd_compaction_retention: "8"
 
 etcd_vault_mount_path: etcd
+
+# Force clients like etcdctl to use TLS certs (different than peer security)
+etcd_secure_client: true

roles/etcd/handlers/backup.yml

@@ -48,5 +48,7 @@
       snapshot save {{ etcd_backup_directory }}/snapshot.db
   environment:
     ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
   retries: 3
   delay: "{{ retry_stagger | random + 3 }}"

roles/etcd/handlers/main.yml

@@ -22,6 +22,8 @@
   uri:
     url: "https://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health"
     validate_certs: no
+    client_cert: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem"
+    client_key: "{{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem"
   register: result
   until: result.status is defined and result.status == 200
   retries: 10

roles/etcd/tasks/configure.yml

@@ -8,6 +8,9 @@
   when: is_etcd_master
   tags:
     - facts
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
 
 - name: Install etcd launch script
   template:

roles/etcd/tasks/join_member.yml

@@ -6,6 +6,9 @@
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   when: target_node == inventory_hostname
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
 
 - include: refresh_config.yml
   vars:
@@ -39,3 +42,6 @@
   tags:
     - facts
   when: target_node == inventory_hostname
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"

roles/etcd/tasks/set_cluster_health.yml

@@ -8,3 +8,6 @@
   when: is_etcd_master
   tags:
     - facts
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"

roles/etcd/templates/etcd.env.j2

@@ -18,6 +18,8 @@ ETCD_AUTO_COMPACTION_RETENTION={{ etcd_compaction_retention }}
 ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
 ETCD_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
 ETCD_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
+ETCD_CLIENT_CERT_AUTH={{ etcd_secure_client | lower}}
+
 ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
 ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
 ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem

roles/kubernetes/master/tasks/pre-upgrade.yml

@@ -3,6 +3,8 @@
   command: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} ls /registry/minions"
   environment:
     ETCDCTL_API: 2
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
   register: old_data_exists
   delegate_to: "{{groups['etcd'][0]}}"
   changed_when: false

roles/network_plugin/calico/rr/tasks/main.yml

@@ -57,6 +57,9 @@
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   delegate_to: "{{groups['etcd'][0]}}"
+  environment:
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
 
 - meta: flush_handlers
 

roles/network_plugin/calico/tasks/main.yml

@@ -83,6 +83,8 @@
   uri:
     url: https://localhost:2379/health
     validate_certs: no
+    client_cert: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    client_key: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
   register: result
   until: result.status == 200 or result.status == 401
   retries: 10

roles/network_plugin/canal/tasks/main.yml

@@ -34,6 +34,9 @@
   delegate_to: "{{groups['etcd'][0]}}"
   changed_when: false
   run_once: true
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
 
 - name: Canal | Create canal node manifests
   template:

roles/network_plugin/canal/templates/cni-canal.conflist.j2

@@ -7,6 +7,9 @@
       "delegate": {
         "type": "calico",
         "etcd_endpoints": "{{ etcd_access_addresses }}",
+        "etcd_key_file": "{{ canal_cert_dir }}/key.pem",
+        "etcd_cert_file": "{{ canal_cert_dir }}/cert.crt",
+        "etcd_ca_cert_file": "{{ canal_cert_dir }}/ca_cert.crt",
         "log_level": "info",
         "policy": {
           "type": "k8s"

scripts/collect-info.yaml

@@ -8,6 +8,7 @@
     bin_dir: /usr/local/bin
     system_namespace: kube-system
     ansible_ssh_pipelining: true
+    etcd_cert_dir: /etc/ssl/etcd/ssl
     commands:
       - name: timedate_info
         cmd: timedatectl status
@@ -85,6 +86,10 @@
       - /var/log/calico/felix/current
       - /var/log/calico/confd/current
 
+  environment:
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+
   tasks:
     - set_fact:
         etcd_access_addresses: |-