Merge pull request #2251 from woopstar/metrics-server-patch-2

Adding metrics-server support for K8s version 1.9

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -102,6 +102,16 @@ spec:
 {% if kube_feature_gates %}
     - --feature-gates={{ kube_feature_gates|join(',') }}
 {% endif %}
+{% if kube_version | version_compare('1.9', '>=') %}
+    - --requestheader-client-ca-file={{ kube_cert_dir }}/ca.pem
+    - --requestheader-allowed-names=front-proxy-client
+    - --requestheader-extra-headers-prefix=X-Remote-Extra-
+    - --requestheader-group-headers=X-Remote-Group
+    - --requestheader-username-headers=X-Remote-User
+    - --enable-aggregator-routing={{ kube_api_aggregator_routing }}
+    - --proxy-client-cert-file={{ kube_cert_dir }}/front-proxy-client.pem
+    - --proxy-client-key-file={{ kube_cert_dir }}/front-proxy-client-key.pem
+{% endif %}
 {% if apiserver_custom_flags is string %}
     - {{ apiserver_custom_flags }}
 {% else %}

roles/kubernetes/secrets/files/make-ssl.sh

@@ -93,6 +93,8 @@ if [ -n "$MASTERS" ]; then
     gen_key_and_cert "kube-scheduler" "/CN=system:kube-scheduler"
     # kube-controller-manager
     gen_key_and_cert "kube-controller-manager" "/CN=system:kube-controller-manager"
+    # metrics aggregator
+    gen_key_and_cert "front-proxy-client" "/CN=front-proxy-client"
 
     for host in $MASTERS; do
         cn="${host%%.*}"

roles/kubernetes/secrets/tasks/check-certs.yml

@@ -26,6 +26,8 @@
     - kube-scheduler-key.pem
     - kube-controller-manager.pem
     - kube-controller-manager-key.pem
+    - front-proxy-client.pem
+    - front-proxy-client-key.pem
     - admin-{{ inventory_hostname }}.pem
     - admin-{{ inventory_hostname }}-key.pem
     - node-{{ inventory_hostname }}.pem
@@ -46,6 +48,8 @@
        '{{ kube_cert_dir }}/kube-scheduler-key.pem',
        '{{ kube_cert_dir }}/kube-controller-manager.pem',
        '{{ kube_cert_dir }}/kube-controller-manager-key.pem',
+       '{{ kube_cert_dir }}/front-proxy-client.pem',
+       '{{ kube_cert_dir }}/front-proxy-client-key.pem',
        {% for host in groups['kube-master'] %}
        '{{ kube_cert_dir }}/admin-{{ host }}.pem'
        '{{ kube_cert_dir }}/admin-{{ host }}-key.pem'
@@ -64,9 +68,10 @@
     gen_master_certs: |-
       {%- set gen = False -%}
       {% set existing_certs = kubecert_master.files|map(attribute='path')|list|sort %}
-      {% for cert in ['apiserver.pem', 'apiserver-key.pem', 'kube-scheduler.pem',
-                      'kube-scheduler-key.pem', 'kube-controller-manager.pem',
-                      'kube-controller-manager-key.pem'] -%}
+      {% for cert in ['apiserver.pem', 'apiserver-key.pem',
+                      'kube-scheduler.pem','kube-scheduler-key.pem',
+                      'kube-controller-manager.pem','kube-controller-manager-key.pem',
+                      'front-proxy-client.pem','front-proxy-client-key.pem'] -%}
         {% set cert_file = "%s/%s.pem"|format(kube_cert_dir, cert) %}
         {% if not cert_file in existing_certs -%}
         {%- set gen = True -%}
@@ -101,6 +106,7 @@
       {% if gen_node_certs[inventory_hostname] or
         (not kubecert_node.results[0].stat.exists|default(False)) or
           (not kubecert_node.results[10].stat.exists|default(False)) or
+            (not kubecert_node.results[7].stat.exists|default(False)) or
               (kubecert_node.results[10].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[10].stat.path)|map(attribute="checksum")|first|default('')) -%}
                 {%- set _ = certs.update({'sync': True}) -%}
       {% endif %}

roles/kubernetes/secrets/tasks/gen_certs_script.yml

@@ -73,6 +73,8 @@
                        'kube-scheduler-key.pem',
                        'kube-controller-manager.pem',
                        'kube-controller-manager-key.pem',
+                       'front-proxy-client.pem',
+                       'front-proxy-client-key.pem',
                        {% for node in groups['kube-master'] %}
                        'admin-{{ node }}.pem',
                        'admin-{{ node }}-key.pem',
@@ -82,6 +84,8 @@
                       'admin-{{ inventory_hostname }}-key.pem',
                       'apiserver.pem',
                       'apiserver-key.pem',
+                      'front-proxy-client.pem',
+                      'front-proxy-client-key.pem',
                       'kube-scheduler.pem',
                       'kube-scheduler-key.pem',
                       'kube-controller-manager.pem',

roles/kubernetes/secrets/tasks/gen_certs_vault.yml

@@ -93,3 +93,29 @@
     issue_cert_mount_path: "{{ kube_vault_mount_path }}"
   with_items: "{{ kube_proxy_certs_needed|d([]) }}"
   when: inventory_hostname in groups['k8s-cluster']
+
+# Issue front proxy cert to kube-master hosts
+- include_tasks: ../../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_common_name: "front-proxy-client"
+    issue_cert_alt_names: "{{ kube_cert_alt_names }}"
+    issue_cert_file_group: "{{ kube_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_hosts: "{{ groups['kube-master'] }}"
+    issue_cert_ip_sans: >-
+        [
+        {%- for host in groups['kube-master']  -%}
+        "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- if hostvars[host]['ip'] is defined -%}
+        "{{ hostvars[host]['ip'] }}",
+        {%- endif -%}
+        {%- endfor -%}
+        "127.0.0.1","::1","{{ kube_apiserver_ip }}"
+        ]
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: front-proxy-client
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+    issue_cert_mount_path: "{{ kube_vault_mount_path }}"
+  with_items: "{{ kube_master_components_certs_needed|d([]) }}"
+  when: inventory_hostname in groups['kube-master']
+  notify: set secret_changed

roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml

@@ -32,7 +32,7 @@
     sync_file_hosts: "{{ groups['kube-master'] }}"
     sync_file_is_cert: true
     sync_file_owner: kube
-  with_items: ["apiserver.pem", "kube-scheduler.pem", "kube-controller-manager.pem"]
+  with_items: ["apiserver.pem", "kube-scheduler.pem", "kube-controller-manager.pem", "front-proxy-client.pem"]
 
 - name: sync_kube_master_certs | Set facts for kube master components sync_file results
   set_fact:

roles/kubespray-defaults/defaults/main.yaml

@@ -122,6 +122,9 @@ kube_apiserver_port: 6443
 kube_apiserver_insecure_bind_address: 127.0.0.1
 kube_apiserver_insecure_port: 8080
 
+# Aggregator
+kube_api_aggregator_routing: true
+
 # Path used to store Docker data
 docker_daemon_graph: "/var/lib/docker"
 

roles/vault/defaults/main.yml

@@ -164,3 +164,11 @@ vault_pki_mounts:
           allow_any_name: true
           enforce_hostnames: false
           organization: "system:node-proxier"
+      - name: front-proxy-client
+        group: k8s-cluster
+        password: "{{ lookup('password', 'credentials/vault/kube-proxy length=15') }}"
+        policy_rules: default
+        role_options:
+          allow_any_name: true
+          enforce_hostnames: false
+          organization: "system:front-proxy"