Merge remote-tracking branch 'upstream/master'

inventory/sample/group_vars/k8s-cluster.yml

@@ -111,7 +111,10 @@ kube_apiserver_insecure_port: 8080 # (http)
 
 # Kube-proxy proxyMode configuration.
 # Can be ipvs, iptables
-kube_proxy_mode: iptables 
+kube_proxy_mode: iptables
+
+## Encrypting Secret Data at Rest (experimental)
+kube_encrypt_secret_data: false
 
 # DNS configuration.
 # Kubernetes cluster name, also will be used as DNS domain

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2

@@ -7,4 +7,4 @@ metadata:
   labels:
     k8s-app: ingress-nginx
 data:
-  {{ ingress_nginx_configmap | to_nice_yaml }}
+  {{ ingress_nginx_configmap | to_nice_yaml | indent(2) }}

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2

@@ -7,4 +7,4 @@ metadata:
   labels:
     k8s-app: ingress-nginx
 data:
-  {{ ingress_nginx_configmap_tcp_services | to_nice_yaml }}
+  {{ ingress_nginx_configmap_tcp_services | to_nice_yaml | indent(2) }}

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2

@@ -7,4 +7,4 @@ metadata:
   labels:
     k8s-app: ingress-nginx
 data:
-  {{ ingress_nginx_configmap_udp_services | to_nice_yaml }}
+  {{ ingress_nginx_configmap_udp_services | to_nice_yaml | indent(2) }}

roles/kubernetes/master/defaults/main.yml

@@ -92,3 +92,8 @@ kube_kubeadm_scheduler_extra_args: {}
 
 ## Variable for influencing kube-scheduler behaviour
 volume_cross_zone_attachment: false
+
+## Encrypting Secret Data at Rest
+kube_encrypt_secret_data: false
+kube_encrypt_token: "{{ lookup('password', inventory_dir + '/credentials/kube_encrypt_token length=32 chars=ascii_letters,digits') }}"
+kube_encryption_algorithm: "aescbc" # Must be either: aescbc, secretbox or aesgcm

roles/kubernetes/master/tasks/encrypt-at-rest.yml

@@ -0,0 +1,10 @@
+---
+- name: Write secrets for encrypting secret data at rest
+  template:
+    src: secrets_encryption.yaml.j2
+    dest: "{{ kube_config_dir }}/ssl/secrets_encryption.yaml"
+    owner: root
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  tags:
+    - kube-apiserver

roles/kubernetes/master/tasks/main.yml

@@ -12,6 +12,9 @@
 - import_tasks: users-file.yml
   when: kube_basic_auth|default(true)
 
+- import_tasks: encrypt-at-rest.yml
+  when: kube_encrypt_secret_data
+
 - name: Compare host kubectl with hyperkube container
   command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/cmp /hyperkube /systembindir/kubectl"
   register: kubectl_task_compare_result

roles/kubernetes/master/templates/kubeadm-config.yaml.j2

@@ -37,6 +37,7 @@ apiServerExtraArgs:
   admission-control: {{ kube_apiserver_admission_control | join(',') }}
   apiserver-count: "{{ kube_apiserver_count }}"
   service-node-port-range: {{ kube_apiserver_node_port_range }}
+  kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
 {% if kube_basic_auth|default(true) %}
   basic-auth-file: {{ kube_users_dir }}/known_users.csv
 {% endif %}
@@ -52,6 +53,9 @@ apiServerExtraArgs:
 {%   if kube_oidc_groups_claim is defined %}
   oidc-groups-claim: {{ kube_oidc_groups_claim }}
 {%   endif %}
+{% endif %}
+{% if kube_encrypt_secret_data %}
+  experimental-encryption-provider-config: {{ kube_config_dir }}/ssl/secrets_encryption.yaml
 {% endif %}
   storage-backend: {{ kube_apiserver_storage_backend }}
 {% if kube_api_runtime_config is defined %}
@@ -59,7 +63,7 @@ apiServerExtraArgs:
 {% endif %}
   allow-privileged: "true"
 {% for key in kube_kubeadm_apiserver_extra_args %}
-  {{ key }}: {{ kube_kubeadm_apiserver_extra_args[key] }}
+  {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
 {% endfor %}
 controllerManagerExtraArgs:
   node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
@@ -69,12 +73,12 @@ controllerManagerExtraArgs:
   feature-gates: {{ kube_feature_gates|join(',') }}
 {% endif %}
 {% for key in kube_kubeadm_controller_extra_args %}
-  {{ key }}: {{ kube_kubeadm_controller_extra_args[key] }}
+  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
 {% endfor %}
 {% if kube_kubeadm_scheduler_extra_args|length > 0 %}
 schedulerExtraArgs:
 {% for key in kube_kubeadm_scheduler_extra_args %}
-  {{ key }}: {{ kube_kubeadm_scheduler_extra_args[key] }}
+  {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
 {% endfor %}
 {% endif %}
 apiServerCertSANs:

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -103,6 +103,9 @@ spec:
 {% if authorization_modes %}
     - --authorization-mode={{ authorization_modes|join(',') }}
 {% endif %}
+{% if kube_encrypt_secret_data %}
+    - --experimental-encryption-provider-config={{ kube_config_dir }}/ssl/secrets_encryption.yaml
+{% endif %}
 {% if kube_feature_gates %}
     - --feature-gates={{ kube_feature_gates|join(',') }}
 {% endif %}

roles/kubernetes/master/templates/secrets_encryption.yaml.j2

@@ -0,0 +1,11 @@
+kind: EncryptionConfig
+apiVersion: v1
+resources:
+  - resources:
+    - secrets
+    providers:
+    - {{ kube_encryption_algorithm }}:
+        keys:
+        - name: key
+          secret: {{ kube_encrypt_token | b64encode }}
+    - identity: {}

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -1,4 +1,4 @@
-### Upstream source https://github.com/kubernetes/release/blob/master/debian/xenial/kubeadm/channel/stable/etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+### Upstream source https://github.com/kubernetes/release/blob/master/debian/xenial/kubeadm/channel/stable/etc/systemd/system/kubelet.service.d/
 ### All upstream values should be present in this file
 
 # logging to stderr means we get it in the systemd journal
@@ -23,13 +23,14 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% if kubelet_authentication_token_webhook %}
 --authentication-token-webhook \
 {% endif %}
+{% if kubelet_authorization_mode_webhook %}
 --authorization-mode=Webhook \
+{% endif %}
 --client-ca-file={{ kube_cert_dir }}/ca.crt \
 --pod-manifest-path={{ kube_manifest_dir }} \
 --cadvisor-port={{ kube_cadvisor_port }} \
 {# end kubeadm specific settings #}
 --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
---kube-reserved cpu={{ kube_cpu_reserved }},memory={{ kube_memory_reserved|regex_replace('Mi', 'M') }} \
 --node-status-update-frequency={{ kubelet_status_update_frequency }} \
 --cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \
 --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \

tests/files/gce_centos7-flannel-addons.yml

@@ -15,3 +15,4 @@ etcd_deployment_type: host
 deploy_netchecker: true
 kubedns_min_replicas: 1
 cloud_provider: gce
+kube_encrypt_secret_data: true