From f3a4c31e666cbf27b9e432782bd6618017d2a075 Mon Sep 17 00:00:00 2001
From: jwfang <54740235@qq.com>
Date: Thu, 15 Jun 2017 18:15:52 +0800
Subject: [PATCH 1/6] add kube-node to system:nodes group, add
 system:kube-proxy cert for kube-proxy

---
 roles/kubernetes/node/tasks/main.yml           |  7 +++++--
 .../templates/kube-proxy-kubeconfig.yaml.j2    | 18 ++++++++++++++++++
 .../templates/manifests/kube-proxy.manifest.j2 |  6 +++---
 roles/kubernetes/secrets/files/make-ssl.sh     | 11 +++++++++--
 .../secrets/tasks/gen_certs_script.yml         |  9 ++++++++-
 5 files changed, 43 insertions(+), 8 deletions(-)
 create mode 100644 roles/kubernetes/node/templates/kube-proxy-kubeconfig.yaml.j2

diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml
index f09845f76..e0558f8cd 100644
--- a/roles/kubernetes/node/tasks/main.yml
+++ b/roles/kubernetes/node/tasks/main.yml
@@ -30,9 +30,12 @@
 
 - name: write the kubecfg (auth) file for kubelet
   template:
-    src: node-kubeconfig.yaml.j2
-    dest: "{{ kube_config_dir }}/node-kubeconfig.yaml"
+    src: "{{ item }}-kubeconfig.yaml.j2"
+    dest: "{{ kube_config_dir }}/{{ item }}-kubeconfig.yaml"
     backup: yes
+  with_items:
+    - node
+    - kube-proxy
   notify: restart kubelet
   tags: kubelet
 
diff --git a/roles/kubernetes/node/templates/kube-proxy-kubeconfig.yaml.j2 b/roles/kubernetes/node/templates/kube-proxy-kubeconfig.yaml.j2
new file mode 100644
index 000000000..cd305b493
--- /dev/null
+++ b/roles/kubernetes/node/templates/kube-proxy-kubeconfig.yaml.j2
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.pem
+    server: {{ kube_apiserver_endpoint }}
+users:
+- name: kube-proxy
+  user:
+    client-certificate: {{ kube_cert_dir }}/kube-proxy.pem
+    client-key: {{ kube_cert_dir }}/kube-proxy-key.pem
+contexts:
+- context:
+    cluster: local
+    user: kube-proxy
+  name: kube-proxy-{{ cluster_name }}
+current-context: kube-proxy-{{ cluster_name }}
diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
index 9b7d53857..d584bdd7d 100644
--- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
+++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
@@ -27,7 +27,7 @@ spec:
     - --v={{ kube_log_level }}
     - --master={{ kube_apiserver_endpoint }}
 {% if not is_kube_master %}
-    - --kubeconfig={{kube_config_dir}}/node-kubeconfig.yaml
+    - --kubeconfig={{kube_config_dir}}/kube-proxy-kubeconfig.yaml
 {% endif %}
     - --bind-address={{ ip | default(ansible_default_ipv4.address) }}
     - --cluster-cidr={{ kube_pods_subnet }}
@@ -41,7 +41,7 @@ spec:
     - mountPath: /etc/ssl/certs
       name: ssl-certs-host
       readOnly: true
-    - mountPath: {{kube_config_dir}}/node-kubeconfig.yaml
+    - mountPath: {{kube_config_dir}}/kube-proxy-kubeconfig.yaml
       name: "kubeconfig"
       readOnly: true
     - mountPath: {{kube_config_dir}}/ssl
@@ -60,7 +60,7 @@ spec:
 {% endif %}
   - name: "kubeconfig"
     hostPath:
-      path: "{{kube_config_dir}}/node-kubeconfig.yaml"
+      path: "{{kube_config_dir}}/kube-proxy-kubeconfig.yaml"
   - name: "etc-kube-ssl"
     hostPath:
       path: "{{kube_config_dir}}/ssl"
diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh
index 55ea13d1e..8fec4f314 100755
--- a/roles/kubernetes/secrets/files/make-ssl.sh
+++ b/roles/kubernetes/secrets/files/make-ssl.sh
@@ -80,6 +80,7 @@ if [ ! -e "$SSLDIR/ca-key.pem" ]; then
     cat ca.pem >> apiserver.pem
 fi
 
+# Admins
 if [ -n "$MASTERS" ]; then
     for host in $MASTERS; do
         cn="${host%%.*}"
@@ -90,16 +91,22 @@ if [ -n "$MASTERS" ]; then
     done
 fi
 
-# Nodes and Admin
+# Nodes
 if [ -n "$HOSTS" ]; then
     for host in $HOSTS; do
         cn="${host%%.*}"
         # node key
         openssl genrsa -out node-${host}-key.pem 2048 > /dev/null 2>&1
-        openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=kube-node-${cn}" > /dev/null 2>&1
+        openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=kube-node-${cn}/O=system:nodes" > /dev/null 2>&1
         openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days 3650 > /dev/null 2>&1
     done
 fi
 
+# system:kube-proxy
+openssl genrsa -out kube-proxy-key.pem 2048 > /dev/null 2>&1
+openssl req -new -key kube-proxy-key.pem -out kube-proxy.csr -subj "/CN=system:kube-proxy" > /dev/null 2>&1
+openssl x509 -req -in kube-proxy.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out kube-proxy.pem -days 3650 > /dev/null 2>&1
+
+
 # Install certs
 mv *.pem ${SSLDIR}/
diff --git a/roles/kubernetes/secrets/tasks/gen_certs_script.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml
index 8df2195bf..0629e3ea5 100644
--- a/roles/kubernetes/secrets/tasks/gen_certs_script.yml
+++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml
@@ -69,11 +69,18 @@
                      'apiserver-key.pem'
                      ]
     all_node_certs: "['ca.pem',
+                    'kube-proxy.pem',
+                    'kube-proxy-key.pem',
                     {% for node in groups['k8s-cluster'] %}
                     'node-{{ node }}.pem',
                     'node-{{ node }}-key.pem',
                     {% endfor %}]"
-    my_node_certs: ['ca.pem', 'node-{{ inventory_hostname }}.pem', 'node-{{ inventory_hostname }}-key.pem']
+    my_node_certs: ['ca.pem',
+                   'kube-proxy.pem',
+                   'kube-proxy-key.pem',
+                   'node-{{ inventory_hostname }}.pem',
+                   'node-{{ inventory_hostname }}-key.pem'
+                   ]
   tags: facts
 
 - name: Gen_certs | Gather master certs

From 8b58394d8c9e0d394e784b558e6f8d1cd623de93 Mon Sep 17 00:00:00 2001
From: jwfang <54740235@qq.com>
Date: Thu, 15 Jun 2017 19:20:58 +0800
Subject: [PATCH 2/6] seperate kube-proxy certs for each node

---
 .../node/templates/kube-proxy-kubeconfig.yaml.j2     |  4 ++--
 roles/kubernetes/secrets/files/make-ssl.sh           | 12 +++++++++---
 roles/kubernetes/secrets/tasks/gen_certs_script.yml  | 10 +++++-----
 3 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/roles/kubernetes/node/templates/kube-proxy-kubeconfig.yaml.j2 b/roles/kubernetes/node/templates/kube-proxy-kubeconfig.yaml.j2
index cd305b493..18c47cd3e 100644
--- a/roles/kubernetes/node/templates/kube-proxy-kubeconfig.yaml.j2
+++ b/roles/kubernetes/node/templates/kube-proxy-kubeconfig.yaml.j2
@@ -8,8 +8,8 @@ clusters:
 users:
 - name: kube-proxy
   user:
-    client-certificate: {{ kube_cert_dir }}/kube-proxy.pem
-    client-key: {{ kube_cert_dir }}/kube-proxy-key.pem
+    client-certificate: {{ kube_cert_dir }}/kube-proxy-{{ inventory_hostname }}.pem
+    client-key: {{ kube_cert_dir }}/kube-proxy-{{ inventory_hostname }}-key.pem
 contexts:
 - context:
     cluster: local
diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh
index 8fec4f314..dde5873fb 100755
--- a/roles/kubernetes/secrets/files/make-ssl.sh
+++ b/roles/kubernetes/secrets/files/make-ssl.sh
@@ -103,9 +103,15 @@ if [ -n "$HOSTS" ]; then
 fi
 
 # system:kube-proxy
-openssl genrsa -out kube-proxy-key.pem 2048 > /dev/null 2>&1
-openssl req -new -key kube-proxy-key.pem -out kube-proxy.csr -subj "/CN=system:kube-proxy" > /dev/null 2>&1
-openssl x509 -req -in kube-proxy.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out kube-proxy.pem -days 3650 > /dev/null 2>&1
+if [ -n "$HOSTS" ]; then
+    for host in $HOSTS; do
+        cn="${host%%.*}"
+        # kube-proxy key
+        openssl genrsa -out kube-proxy-${host}-key.pem 2048 > /dev/null 2>&1
+        openssl req -new -key kube-proxy-${host}-key.pem -out kube-proxy-${host}.csr -subj "/CN=system:kube-proxy" > /dev/null 2>&1
+        openssl x509 -req -in kube-proxy-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out kube-proxy-${host}.pem -days 3650 > /dev/null 2>&1
+    done
+fi
 
 
 # Install certs
diff --git a/roles/kubernetes/secrets/tasks/gen_certs_script.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml
index 0629e3ea5..1920b696b 100644
--- a/roles/kubernetes/secrets/tasks/gen_certs_script.yml
+++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml
@@ -69,17 +69,17 @@
                      'apiserver-key.pem'
                      ]
     all_node_certs: "['ca.pem',
-                    'kube-proxy.pem',
-                    'kube-proxy-key.pem',
                     {% for node in groups['k8s-cluster'] %}
                     'node-{{ node }}.pem',
                     'node-{{ node }}-key.pem',
+                    'kube-proxy-{{ node }}.pem',
+                    'kube-proxy-{{ node }}-key.pem',
                     {% endfor %}]"
     my_node_certs: ['ca.pem',
-                   'kube-proxy.pem',
-                   'kube-proxy-key.pem',
                    'node-{{ inventory_hostname }}.pem',
-                   'node-{{ inventory_hostname }}-key.pem'
+                   'node-{{ inventory_hostname }}-key.pem',
+                   'kube-proxy-{{ inventory_hostname }}.pem',
+                   'kube-proxy-{{ inventory_hostname }}-key.pem',
                    ]
   tags: facts
 

From 0ee229488ed1e0c30fa4f7e4ff9f97d91ecf938c Mon Sep 17 00:00:00 2001
From: jwfang <54740235@qq.com>
Date: Fri, 16 Jun 2017 14:21:21 +0800
Subject: [PATCH 3/6] certs for system:kube-controller-manager
 system:kube-scheduler

---
 roles/kubernetes/master/tasks/main.yml        | 22 +++++++++---
 ...kube-controller-manager-kubeconfig.yaml.j2 | 18 ++++++++++
 .../kube-scheduler-kubeconfig.yaml.j2         | 18 ++++++++++
 .../kube-controller-manager.manifest.j2       | 32 ++++++++++++-----
 .../manifests/kube-scheduler.manifest.j2      | 26 +++++++++++++-
 .../manifests/kube-proxy.manifest.j2          | 22 ++++++------
 roles/kubernetes/secrets/files/make-ssl.sh    | 36 ++++++++++---------
 .../secrets/tasks/gen_certs_script.yml        | 14 ++++++--
 8 files changed, 143 insertions(+), 45 deletions(-)
 create mode 100644 roles/kubernetes/master/templates/kube-controller-manager-kubeconfig.yaml.j2
 create mode 100644 roles/kubernetes/master/templates/kube-scheduler-kubeconfig.yaml.j2

diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml
index dadef4bf5..6922e6a51 100644
--- a/roles/kubernetes/master/tasks/main.yml
+++ b/roles/kubernetes/master/tasks/main.yml
@@ -60,12 +60,11 @@
   when: kubesystem|failed and inventory_hostname == groups['kube-master'][0]
   tags: apps
 
-- name: Write kube-controller-manager manifest
+- name: Write kube-scheduler kubeconfig
   template:
-    src: manifests/kube-controller-manager.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
-  notify: Master | wait for kube-controller-manager
-  tags: kube-controller-manager
+    src: kube-scheduler-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
+  tags: kube-scheduler
 
 - name: Write kube-scheduler manifest
   template:
@@ -74,6 +73,19 @@
   notify: Master | wait for kube-scheduler
   tags: kube-scheduler
 
+- name: Write kube-controller-manager kubeconfig
+  template:
+    src: kube-controller-manager-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
+  tags: kube-controller-manager
+
+- name: Write kube-controller-manager manifest
+  template:
+    src: manifests/kube-controller-manager.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
+  notify: Master | wait for kube-controller-manager
+  tags: kube-controller-manager
+
 - include: post-upgrade.yml
   tags: k8s-post-upgrade
 
diff --git a/roles/kubernetes/master/templates/kube-controller-manager-kubeconfig.yaml.j2 b/roles/kubernetes/master/templates/kube-controller-manager-kubeconfig.yaml.j2
new file mode 100644
index 000000000..887d022c1
--- /dev/null
+++ b/roles/kubernetes/master/templates/kube-controller-manager-kubeconfig.yaml.j2
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.pem
+    server: {{ kube_apiserver_endpoint }}
+users:
+- name: kube-controller-manager
+  user:
+    client-certificate: {{ kube_cert_dir }}/kube-controller-manager.pem
+    client-key: {{ kube_cert_dir }}/kube-controller-manager-key.pem
+contexts:
+- context:
+    cluster: local
+    user: kube-controller-manager
+  name: kube-controller-manager-{{ cluster_name }}
+current-context: kube-controller-manager-{{ cluster_name }}
diff --git a/roles/kubernetes/master/templates/kube-scheduler-kubeconfig.yaml.j2 b/roles/kubernetes/master/templates/kube-scheduler-kubeconfig.yaml.j2
new file mode 100644
index 000000000..974b72427
--- /dev/null
+++ b/roles/kubernetes/master/templates/kube-scheduler-kubeconfig.yaml.j2
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.pem
+    server: {{ kube_apiserver_endpoint }}
+users:
+- name: kube-scheduler
+  user:
+    client-certificate: {{ kube_cert_dir }}/kube-scheduler.pem
+    client-key: {{ kube_cert_dir }}/kube-scheduler-key.pem
+contexts:
+- context:
+    cluster: local
+    user: kube-scheduler
+  name: kube-scheduler-{{ cluster_name }}
+current-context: kube-scheduler-{{ cluster_name }}
diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
index d3f8a23a5..f65bb004c 100644
--- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
@@ -24,7 +24,7 @@ spec:
     command:
     - /hyperkube
     - controller-manager
-    - --master={{ kube_apiserver_endpoint }}
+    - --kubeconfig={{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml
     - --leader-elect=true
     - --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
     - --root-ca-file={{ kube_cert_dir }}/ca.pem
@@ -61,20 +61,36 @@ spec:
       initialDelaySeconds: 30
       timeoutSeconds: 10
     volumeMounts:
-    - mountPath: {{ kube_cert_dir }}
-      name: ssl-certs-kubernetes
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+    - mountPath: "{{kube_config_dir}}/ssl"
+      name: etc-kube-ssl
+      readOnly: true
+    - mountPath: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
+      name: kubeconfig
       readOnly: true
 {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere" ] %}
-    - mountPath: {{ kube_config_dir }}/cloud_config
+    - mountPath: "{{ kube_config_dir }}/cloud_config"
       name: cloudconfig
       readOnly: true
 {% endif %}
   volumes:
-  - hostPath:
-      path: {{ kube_cert_dir }}
-    name: ssl-certs-kubernetes
+  - name: ssl-certs-host
+    hostPath:
+{% if ansible_os_family == 'RedHat' %}
+      path: /etc/pki/tls
+{% else %}
+      path: /usr/share/ca-certificates
+{% endif %}
+  - name: etc-kube-ssl
+    hostPath:
+      path: "{{ kube_config_dir }}/ssl"
+  - name: kubeconfig
+    hostPath:
+      path: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
 {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %}
   - hostPath:
-      path: {{ kube_config_dir }}/cloud_config
+      path: "{{ kube_config_dir }}/cloud_config"
     name: cloudconfig
 {% endif %}
diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
index 441f991eb..1508e60cf 100644
--- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
@@ -25,7 +25,7 @@ spec:
     - /hyperkube
     - scheduler
     - --leader-elect=true
-    - --master={{ kube_apiserver_endpoint }}
+    - --kubeconfig={{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml
     - --v={{ kube_log_level }}
 {% if scheduler_custom_flags is string %}
     - {{ scheduler_custom_flags }}
@@ -41,3 +41,27 @@ spec:
         port: 10251
       initialDelaySeconds: 30
       timeoutSeconds: 10
+    volumeMounts:
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+    - mountPath: "{{ kube_config_dir }}/ssl"
+      name: etc-kube-ssl
+      readOnly: true
+    - mountPath: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
+      name: kubeconfig
+      readOnly: true
+  volumes:
+  - name: ssl-certs-host
+    hostPath:
+{% if ansible_os_family == 'RedHat' %}
+      path: /etc/pki/tls
+{% else %}
+      path: /usr/share/ca-certificates
+{% endif %}
+  - name: etc-kube-ssl
+    hostPath:
+      path: "{{ kube_config_dir }}/ssl"
+  - name: kubeconfig
+    hostPath:
+      path: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
index d584bdd7d..bbb13bc9d 100644
--- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
+++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2
@@ -41,14 +41,14 @@ spec:
     - mountPath: /etc/ssl/certs
       name: ssl-certs-host
       readOnly: true
-    - mountPath: {{kube_config_dir}}/kube-proxy-kubeconfig.yaml
-      name: "kubeconfig"
+    - mountPath: "{{ kube_config_dir }}/ssl"
+      name: etc-kube-ssl
       readOnly: true
-    - mountPath: {{kube_config_dir}}/ssl
-      name: "etc-kube-ssl"
+    - mountPath: "{{ kube_config_dir }}/kube-proxy-kubeconfig.yaml"
+      name: kubeconfig
       readOnly: true
     - mountPath: /var/run/dbus
-      name: "var-run-dbus"
+      name: var-run-dbus
       readOnly: false
   volumes:
   - name: ssl-certs-host
@@ -58,12 +58,12 @@ spec:
 {% else %}
       path: /usr/share/ca-certificates
 {% endif %}
-  - name: "kubeconfig"
+  - name: etc-kube-ssl
     hostPath:
-      path: "{{kube_config_dir}}/kube-proxy-kubeconfig.yaml"
-  - name: "etc-kube-ssl"
+      path: "{{ kube_config_dir }}/ssl"
+  - name: kubeconfig
     hostPath:
-      path: "{{kube_config_dir}}/ssl"
-  - name: "var-run-dbus"
+      path: "{{ kube_config_dir }}/kube-proxy-kubeconfig.yaml"
+  - name: var-run-dbus
     hostPath:
-      path: "/var/run/dbus"
+      path: /var/run/dbus
diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh
index dde5873fb..5383e7adb 100755
--- a/roles/kubernetes/secrets/files/make-ssl.sh
+++ b/roles/kubernetes/secrets/files/make-ssl.sh
@@ -72,22 +72,30 @@ else
     openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" > /dev/null 2>&1
 fi
 
+gen_key_and_cert() {
+    local name=$1
+    local subject=$2
+    openssl genrsa -out ${name}-key.pem 2048 > /dev/null 2>&1
+    openssl req -new -key ${name}-key.pem -out ${name}.csr -subj "${subject}" -config ${CONFIG} > /dev/null 2>&1
+    openssl x509 -req -in ${name}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${name}.pem -days 3650 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
+}
+
 if [ ! -e "$SSLDIR/ca-key.pem" ]; then
-    # kube-apiserver key
-    openssl genrsa -out apiserver-key.pem 2048 > /dev/null 2>&1
-    openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config ${CONFIG} > /dev/null 2>&1
-    openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 3650 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
+    # kube-apiserver
+    gen_key_and_cert "apiserver" "/CN=kube-apiserver"
     cat ca.pem >> apiserver.pem
+    # kube-scheduler
+    gen_key_and_cert "kube-scheduler" "/CN=system:kube-scheduler"
+    # kube-controller-manager
+    gen_key_and_cert "kube-controller-manager" "/CN=system:kube-controller-manager"
 fi
 
 # Admins
 if [ -n "$MASTERS" ]; then
     for host in $MASTERS; do
         cn="${host%%.*}"
-        # admin key
-        openssl genrsa -out admin-${host}-key.pem 2048 > /dev/null 2>&1
-        openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=kube-admin-${cn}/O=system:masters" > /dev/null 2>&1
-        openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days 3650 > /dev/null 2>&1
+        # admin
+        gen_key_and_cert "admin-${host}" "/CN=kube-admin-${cn}/O=system:masters"
     done
 fi
 
@@ -95,10 +103,7 @@ fi
 if [ -n "$HOSTS" ]; then
     for host in $HOSTS; do
         cn="${host%%.*}"
-        # node key
-        openssl genrsa -out node-${host}-key.pem 2048 > /dev/null 2>&1
-        openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=kube-node-${cn}/O=system:nodes" > /dev/null 2>&1
-        openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days 3650 > /dev/null 2>&1
+        gen_key_and_cert "node-${host}" "/CN=kube-node-${cn}/O=system:nodes"
     done
 fi
 
@@ -106,13 +111,10 @@ fi
 if [ -n "$HOSTS" ]; then
     for host in $HOSTS; do
         cn="${host%%.*}"
-        # kube-proxy key
-        openssl genrsa -out kube-proxy-${host}-key.pem 2048 > /dev/null 2>&1
-        openssl req -new -key kube-proxy-${host}-key.pem -out kube-proxy-${host}.csr -subj "/CN=system:kube-proxy" > /dev/null 2>&1
-        openssl x509 -req -in kube-proxy-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out kube-proxy-${host}.pem -days 3650 > /dev/null 2>&1
+        # kube-proxy
+        gen_key_and_cert "kube-proxy-${host}" "/CN=system:kube-proxy"
     done
 fi
 
-
 # Install certs
 mv *.pem ${SSLDIR}/
diff --git a/roles/kubernetes/secrets/tasks/gen_certs_script.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml
index 1920b696b..61d9c7826 100644
--- a/roles/kubernetes/secrets/tasks/gen_certs_script.yml
+++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml
@@ -56,17 +56,25 @@
 
 - set_fact:
     all_master_certs: "['ca-key.pem',
+                      'apiserver.pem',
+                      'apiserver-key.pem',
+                      'kube-scheduler.pem',
+                      'kube-scheduler-key.pem',
+                      'kube-controller-manager.pem',
+                      'kube-controller-manager-key.pem',
                       {% for node in groups['kube-master'] %}
                       'admin-{{ node }}.pem',
                       'admin-{{ node }}-key.pem',
-                      'apiserver.pem',
-                      'apiserver-key.pem',
                       {% endfor %}]"
     my_master_certs: ['ca-key.pem',
                      'admin-{{ inventory_hostname }}.pem',
                      'admin-{{ inventory_hostname }}-key.pem',
                      'apiserver.pem',
-                     'apiserver-key.pem'
+                     'apiserver-key.pem',
+                     'kube-scheduler.pem',
+                     'kube-scheduler-key.pem',
+                     'kube-controller-manager.pem',
+                     'kube-controller-manager-key.pem',
                      ]
     all_node_certs: "['ca.pem',
                     {% for node in groups['k8s-cluster'] %}

From 765a5ce1ab5189279d71b8c3e356e1618ad8849e Mon Sep 17 00:00:00 2001
From: jwfang <54740235@qq.com>
Date: Fri, 16 Jun 2017 17:15:37 +0800
Subject: [PATCH 4/6] node identified as system:node:<node-name>

---
 roles/kubernetes/secrets/files/make-ssl.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh
index 5383e7adb..e8574cc6b 100755
--- a/roles/kubernetes/secrets/files/make-ssl.sh
+++ b/roles/kubernetes/secrets/files/make-ssl.sh
@@ -103,7 +103,7 @@ fi
 if [ -n "$HOSTS" ]; then
     for host in $HOSTS; do
         cn="${host%%.*}"
-        gen_key_and_cert "node-${host}" "/CN=kube-node-${cn}/O=system:nodes"
+        gen_key_and_cert "node-${host}" "/CN=system:node:${cn}/O=system:nodes"
     done
 fi
 

From acbdfb08ce9b41b56199f3211fb676f374321aeb Mon Sep 17 00:00:00 2001
From: jwfang <54740235@qq.com>
Date: Fri, 16 Jun 2017 18:54:18 +0800
Subject: [PATCH 5/6] run kubedns as
 system:serviceaccount:kube-system:kube-dns; but dns does NOT work

---
 roles/kubernetes-apps/ansible/tasks/main.yml                 | 1 +
 roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml   | 1 +
 .../ansible/templates/kubedns-serviceaccount.yml             | 5 +++++
 3 files changed, 7 insertions(+)
 create mode 100644 roles/kubernetes-apps/ansible/templates/kubedns-serviceaccount.yml

diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml
index ed0d11f28..5225bbda3 100644
--- a/roles/kubernetes-apps/ansible/tasks/main.yml
+++ b/roles/kubernetes-apps/ansible/tasks/main.yml
@@ -13,6 +13,7 @@
     src: "{{item.file}}"
     dest: "{{kube_config_dir}}/{{item.file}}"
   with_items:
+    - {name: kube-dns, file: kubedns-serviceaccount.yml, type: serviceaccount}
     - {name: kubedns, file: kubedns-deploy.yml, type: deployment}
     - {name: kubedns, file: kubedns-svc.yml, type: svc}
     - {name: kubedns-autoscaler, file: kubedns-autoscaler.yml, type: deployment}
diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml
index a2150cc70..4c7a7eec7 100644
--- a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml
+++ b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml
@@ -114,3 +114,4 @@ spec:
         - containerPort: 8080
           protocol: TCP
       dnsPolicy: Default  # Don't use cluster DNS.
+      serviceAccountName: kube-dns
diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-serviceaccount.yml b/roles/kubernetes-apps/ansible/templates/kubedns-serviceaccount.yml
new file mode 100644
index 000000000..8cf41ae23
--- /dev/null
+++ b/roles/kubernetes-apps/ansible/templates/kubedns-serviceaccount.yml
@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-dns
+  namespace: {{ system_namespace }}

From 7c2816ba738887988c476f1a7bb4c3c95659c63c Mon Sep 17 00:00:00 2001
From: jwfang <54740235@qq.com>
Date: Fri, 16 Jun 2017 20:08:19 +0800
Subject: [PATCH 6/6] add label for kube-dns sa

---
 .../templates/{kubedns-serviceaccount.yml => kubedns-sa.yml}    | 2 ++
 1 file changed, 2 insertions(+)
 rename roles/kubernetes-apps/ansible/templates/{kubedns-serviceaccount.yml => kubedns-sa.yml} (65%)

diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-serviceaccount.yml b/roles/kubernetes-apps/ansible/templates/kubedns-sa.yml
similarity index 65%
rename from roles/kubernetes-apps/ansible/templates/kubedns-serviceaccount.yml
rename to roles/kubernetes-apps/ansible/templates/kubedns-sa.yml
index 8cf41ae23..e520ccbfc 100644
--- a/roles/kubernetes-apps/ansible/templates/kubedns-serviceaccount.yml
+++ b/roles/kubernetes-apps/ansible/templates/kubedns-sa.yml
@@ -3,3 +3,5 @@ kind: ServiceAccount
 metadata:
   name: kube-dns
   namespace: {{ system_namespace }}
+  labels:
+    kubernetes.io/cluster-service: "true"