From 4fa142be0bc7f70ad37f2fe2c6a871cf7f651116 Mon Sep 17 00:00:00 2001
From: jwfang <54740235@qq.com>
Date: Fri, 16 Jun 2017 14:21:21 +0800
Subject: [PATCH] certs for system:kube-controller-manager
 system:kube-scheduler

---
 roles/kubernetes/master/tasks/main.yml        | 22 +++++++++---
 ...kube-controller-manager-kubeconfig.yaml.j2 | 18 ++++++++++
 .../kube-scheduler-kubeconfig.yaml.j2         | 18 ++++++++++
 .../kube-controller-manager.manifest.j2       | 32 ++++++++++++-----
 .../manifests/kube-scheduler.manifest.j2      | 26 +++++++++++++-
 .../manifests/kube-proxy.manifest.j2          | 22 ++++++------
 roles/kubernetes/secrets/files/make-ssl.sh    | 36 ++++++++++---------
 .../secrets/tasks/gen_certs_script.yml        | 14 ++++++--
 8 files changed, 143 insertions(+), 45 deletions(-)
 create mode 100644 roles/kubernetes/master/templates/kube-controller-manager-kubeconfig.yaml.j2
 create mode 100644 roles/kubernetes/master/templates/kube-scheduler-kubeconfig.yaml.j2

diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml
index dadef4bf5..6922e6a51 100644
--- a/roles/kubernetes/master/tasks/main.yml
+++ b/roles/kubernetes/master/tasks/main.yml
@@ -60,12 +60,11 @@
   when: kubesystem|failed and inventory_hostname == groups['kube-master'][0]
   tags: apps
 
-- name: Write kube-controller-manager manifest
+- name: Write kube-scheduler kubeconfig
   template:
-    src: manifests/kube-controller-manager.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
-  notify: Master | wait for kube-controller-manager
-  tags: kube-controller-manager
+    src: kube-scheduler-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
+  tags: kube-scheduler
 
 - name: Write kube-scheduler manifest
   template:
@@ -74,6 +73,19 @@
   notify: Master | wait for kube-scheduler
   tags: kube-scheduler
 
+- name: Write kube-controller-manager kubeconfig
+  template:
+    src: kube-controller-manager-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
+  tags: kube-controller-manager
+
+- name: Write kube-controller-manager manifest
+  template:
+    src: manifests/kube-controller-manager.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
+  notify: Master | wait for kube-controller-manager
+  tags: kube-controller-manager
+
 - include: post-upgrade.yml
   tags: k8s-post-upgrade
 
diff --git a/roles/kubernetes/master/templates/kube-controller-manager-kubeconfig.yaml.j2 b/roles/kubernetes/master/templates/kube-controller-manager-kubeconfig.yaml.j2
new file mode 100644
index 000000000..887d022c1
--- /dev/null
+++ b/roles/kubernetes/master/templates/kube-controller-manager-kubeconfig.yaml.j2
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.pem
+    server: {{ kube_apiserver_endpoint }}
+users:
+- name: kube-controller-manager
+  user:
+    client-certificate: {{ kube_cert_dir }}/kube-controller-manager.pem
+    client-key: {{ kube_cert_dir }}/kube-controller-manager-key.pem
+contexts:
+- context:
+    cluster: local
+    user: kube-controller-manager
+  name: kube-controller-manager-{{ cluster_name }}
+current-context: kube-controller-manager-{{ cluster_name }}
diff --git a/roles/kubernetes/master/templates/kube-scheduler-kubeconfig.yaml.j2 b/roles/kubernetes/master/templates/kube-scheduler-kubeconfig.yaml.j2
new file mode 100644
index 000000000..974b72427
--- /dev/null
+++ b/roles/kubernetes/master/templates/kube-scheduler-kubeconfig.yaml.j2
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.pem
+    server: {{ kube_apiserver_endpoint }}
+users:
+- name: kube-scheduler
+  user:
+    client-certificate: {{ kube_cert_dir }}/kube-scheduler.pem
+    client-key: {{ kube_cert_dir }}/kube-scheduler-key.pem
+contexts:
+- context:
+    cluster: local
+    user: kube-scheduler
+  name: kube-scheduler-{{ cluster_name }}
+current-context: kube-scheduler-{{ cluster_name }}
diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
index d3f8a23a5..f65bb004c 100644
--- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
@@ -24,7 +24,7 @@ spec:
     command:
     - /hyperkube
     - controller-manager
-    - --master={{ kube_apiserver_endpoint }}
+    - --kubeconfig={{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml
     - --leader-elect=true
     - --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
     - --root-ca-file={{ kube_cert_dir }}/ca.pem
@@ -61,20 +61,36 @@ spec:
       initialDelaySeconds: 30
       timeoutSeconds: 10
     volumeMounts:
-    - mountPath: {{ kube_cert_dir }}
-      name: ssl-certs-kubernetes
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+    - mountPath: "{{kube_config_dir}}/ssl"
+      name: etc-kube-ssl
+      readOnly: true
+    - mountPath: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
+      name: kubeconfig
       readOnly: true
 {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere" ] %}
-    - mountPath: {{ kube_config_dir }}/cloud_config
+    - mountPath: "{{ kube_config_dir }}/cloud_config"
       name: cloudconfig
       readOnly: true
 {% endif %}
   volumes:
-  - hostPath:
-      path: {{ kube_cert_dir }}
-    name: ssl-certs-kubernetes
+  - name: ssl-certs-host
+    hostPath:
+{% if ansible_os_family == 'RedHat' %}
+      path: /etc/pki/tls
+{% else %}
+      path: /usr/share/ca-certificates
+{% endif %}
+  - name: etc-kube-ssl
+    hostPath:
+      path: "{{ kube_config_dir }}/ssl"
+  - name: kubeconfig
+    hostPath:
+      path: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
 {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %}
   - hostPath:
-      path: {{ kube_config_dir }}/cloud_config
+      path: "{{ kube_config_dir }}/cloud_config"
     name: cloudconfig
 {% endif %}
diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
index 441f991eb..1508e60cf 100644
--- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
@@ -25,7 +25,7 @@ spec:
     - /hyperkube
     - scheduler
     - --leader-elect=true
-    - --master={{ kube_apiserver_endpoint }}
+    - --kubeconfig={{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml
     - --v={{ kube_log_level }}
 {% if scheduler_custom_flags is string %}
     - {{ scheduler_custom_flags }}
@@ -41,3 +41,27 @@ spec:
         port: 10251
       initialDelaySeconds: 30
       timeoutSeconds: 10
+    volumeMounts:
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+    - mountPath: "{{ kube_config_dir }}/ssl"
+      name: etc-kube-ssl
+      readOnly: true
+    - mountPath: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
+      name: kubeconfig
+      readOnly: true
+  volumes:
+  - name: ssl-certs-host
+    hostPath:
+{% if ansible_os_family == 'RedHat' %}
+      path: /etc/pki/tls
+{% else %}
+      path: /usr/share/ca-certificates
+{% endif %}
+  - name: etc-kube-ssl
+    hostPath:
+      path: "{{ kube_config_dir }}/ssl"
+  - name: kubeconfig
+    hostPath:
+      path: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
index d584bdd7d..bbb13bc9d 100644
--- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
+++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
@@ -41,14 +41,14 @@ spec:
     - mountPath: /etc/ssl/certs
       name: ssl-certs-host
       readOnly: true
-    - mountPath: {{kube_config_dir}}/kube-proxy-kubeconfig.yaml
-      name: "kubeconfig"
+    - mountPath: "{{ kube_config_dir }}/ssl"
+      name: etc-kube-ssl
       readOnly: true
-    - mountPath: {{kube_config_dir}}/ssl
-      name: "etc-kube-ssl"
+    - mountPath: "{{ kube_config_dir }}/kube-proxy-kubeconfig.yaml"
+      name: kubeconfig
       readOnly: true
     - mountPath: /var/run/dbus
-      name: "var-run-dbus"
+      name: var-run-dbus
       readOnly: false
   volumes:
   - name: ssl-certs-host
@@ -58,12 +58,12 @@ spec:
 {% else %}
       path: /usr/share/ca-certificates
 {% endif %}
-  - name: "kubeconfig"
+  - name: etc-kube-ssl
     hostPath:
-      path: "{{kube_config_dir}}/kube-proxy-kubeconfig.yaml"
-  - name: "etc-kube-ssl"
+      path: "{{ kube_config_dir }}/ssl"
+  - name: kubeconfig
     hostPath:
-      path: "{{kube_config_dir}}/ssl"
-  - name: "var-run-dbus"
+      path: "{{ kube_config_dir }}/kube-proxy-kubeconfig.yaml"
+  - name: var-run-dbus
     hostPath:
-      path: "/var/run/dbus"
+      path: /var/run/dbus
diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh
index dde5873fb..5383e7adb 100755
--- a/roles/kubernetes/secrets/files/make-ssl.sh
+++ b/roles/kubernetes/secrets/files/make-ssl.sh
@@ -72,22 +72,30 @@ else
     openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" > /dev/null 2>&1
 fi
 
+gen_key_and_cert() {
+    local name=$1
+    local subject=$2
+    openssl genrsa -out ${name}-key.pem 2048 > /dev/null 2>&1
+    openssl req -new -key ${name}-key.pem -out ${name}.csr -subj "${subject}" -config ${CONFIG} > /dev/null 2>&1
+    openssl x509 -req -in ${name}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${name}.pem -days 3650 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
+}
+
 if [ ! -e "$SSLDIR/ca-key.pem" ]; then
-    # kube-apiserver key
-    openssl genrsa -out apiserver-key.pem 2048 > /dev/null 2>&1
-    openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config ${CONFIG} > /dev/null 2>&1
-    openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 3650 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
+    # kube-apiserver
+    gen_key_and_cert "apiserver" "/CN=kube-apiserver"
     cat ca.pem >> apiserver.pem
+    # kube-scheduler
+    gen_key_and_cert "kube-scheduler" "/CN=system:kube-scheduler"
+    # kube-controller-manager
+    gen_key_and_cert "kube-controller-manager" "/CN=system:kube-controller-manager"
 fi
 
 # Admins
 if [ -n "$MASTERS" ]; then
     for host in $MASTERS; do
         cn="${host%%.*}"
-        # admin key
-        openssl genrsa -out admin-${host}-key.pem 2048 > /dev/null 2>&1
-        openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=kube-admin-${cn}/O=system:masters" > /dev/null 2>&1
-        openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days 3650 > /dev/null 2>&1
+        # admin
+        gen_key_and_cert "admin-${host}" "/CN=kube-admin-${cn}/O=system:masters"
     done
 fi
 
@@ -95,10 +103,7 @@ fi
 if [ -n "$HOSTS" ]; then
     for host in $HOSTS; do
         cn="${host%%.*}"
-        # node key
-        openssl genrsa -out node-${host}-key.pem 2048 > /dev/null 2>&1
-        openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=kube-node-${cn}/O=system:nodes" > /dev/null 2>&1
-        openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days 3650 > /dev/null 2>&1
+        gen_key_and_cert "node-${host}" "/CN=kube-node-${cn}/O=system:nodes"
     done
 fi
 
@@ -106,13 +111,10 @@ fi
 if [ -n "$HOSTS" ]; then
     for host in $HOSTS; do
         cn="${host%%.*}"
-        # kube-proxy key
-        openssl genrsa -out kube-proxy-${host}-key.pem 2048 > /dev/null 2>&1
-        openssl req -new -key kube-proxy-${host}-key.pem -out kube-proxy-${host}.csr -subj "/CN=system:kube-proxy" > /dev/null 2>&1
-        openssl x509 -req -in kube-proxy-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out kube-proxy-${host}.pem -days 3650 > /dev/null 2>&1
+        # kube-proxy
+        gen_key_and_cert "kube-proxy-${host}" "/CN=system:kube-proxy"
     done
 fi
 
-
 # Install certs
 mv *.pem ${SSLDIR}/
diff --git a/roles/kubernetes/secrets/tasks/gen_certs_script.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml
index 1920b696b..61d9c7826 100644
--- a/roles/kubernetes/secrets/tasks/gen_certs_script.yml
+++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml
@@ -56,17 +56,25 @@
 
 - set_fact:
     all_master_certs: "['ca-key.pem',
+                      'apiserver.pem',
+                      'apiserver-key.pem',
+                      'kube-scheduler.pem',
+                      'kube-scheduler-key.pem',
+                      'kube-controller-manager.pem',
+                      'kube-controller-manager-key.pem',
                       {% for node in groups['kube-master'] %}
                       'admin-{{ node }}.pem',
                       'admin-{{ node }}-key.pem',
-                      'apiserver.pem',
-                      'apiserver-key.pem',
                       {% endfor %}]"
     my_master_certs: ['ca-key.pem',
                      'admin-{{ inventory_hostname }}.pem',
                      'admin-{{ inventory_hostname }}-key.pem',
                      'apiserver.pem',
-                     'apiserver-key.pem'
+                     'apiserver-key.pem',
+                     'kube-scheduler.pem',
+                     'kube-scheduler-key.pem',
+                     'kube-controller-manager.pem',
+                     'kube-controller-manager-key.pem',
                      ]
     all_node_certs: "['ca.pem',
                     {% for node in groups['k8s-cluster'] %}