Master and nodes will run the 'node' role, kube-proxy is run under a container, new script for ssl certs

roles/kubernetes/common/files/make-ca-cert.sh

@@ -1,115 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# Caller should set in the ev:
-# MASTER_IP - this may be an ip or things like "_use_gce_external_ip_"
-# DNS_DOMAIN - which will be passed to minions in --cluster_domain
-# SERVICE_CLUSTER_IP_RANGE - where all service IPs are allocated
-# MASTER_NAME - I'm not sure what it is...
-
-# Also the following will be respected
-# CERT_DIR - where to place the finished certs
-# CERT_GROUP - who the group owner of the cert files should be
-
-cert_ip="${MASTER_IP:="${1}"}"
-master_name="${MASTER_NAME:="kubernetes"}"
-service_range="${SERVICE_CLUSTER_IP_RANGE:="10.0.0.0/16"}"
-dns_domain="${DNS_DOMAIN:="cluster.local"}"
-cert_dir="${CERT_DIR:-"/srv/kubernetes"}"
-cert_group="${CERT_GROUP:="kube-cert"}"
-
-# The following certificate pairs are created:
-#
-#  - ca (the cluster's certificate authority)
-#  - server
-#  - kubelet
-#  - kubecfg (for kubectl)
-#
-# TODO(roberthbailey): Replace easyrsa with a simple Go program to generate
-# the certs that we need.
-
-# TODO: Add support for discovery on other providers?
-if [ "$cert_ip" == "_use_gce_external_ip_" ]; then
-  cert_ip=$(curl -s -H Metadata-Flavor:Google http://metadata.google.internal./computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
-fi
-
-if [ "$cert_ip" == "_use_aws_external_ip_" ]; then
-  cert_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)
-fi
-
-if [ "$cert_ip" == "_use_azure_dns_name_" ]; then
-  cert_ip=$(uname -n | awk -F. '{ print $2 }').cloudapp.net
-fi
-
-tmpdir=$(mktemp -d --tmpdir kubernetes_cacert.XXXXXX)
-trap 'rm -rf "${tmpdir}"' EXIT
-cd "${tmpdir}"
-
-# TODO: For now, this is a patched tool that makes subject-alt-name work, when
-# the fix is upstream  move back to the upstream easyrsa.  This is cached in GCS
-# but is originally taken from:
-#   https://github.com/brendandburns/easy-rsa/archive/master.tar.gz
-#
-# To update, do the following:
-# curl -o easy-rsa.tar.gz https://github.com/brendandburns/easy-rsa/archive/master.tar.gz
-# gsutil cp easy-rsa.tar.gz gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz
-# gsutil acl ch -R -g all:R gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz
-#
-# Due to GCS caching of public objects, it may take time for this to be widely
-# distributed.
-
-# Calculate the first ip address in the service range
-octects=($(echo "${service_range}" | sed -e 's|/.*||' -e 's/\./ /g'))
-((octects[3]+=1))
-service_ip=$(echo "${octects[*]}" | sed 's/ /./g')
-
-# Determine appropriete subject alt names
-sans="IP:${cert_ip},IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${dns_domain},DNS:${master_name}"
-
-curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz > /dev/null 2>&1
-tar xzf easy-rsa.tar.gz > /dev/null
-cd easy-rsa-master/easyrsa3
-
-(./easyrsa init-pki > /dev/null 2>&1
- ./easyrsa --batch "--req-cn=${cert_ip}@$(date +%s)" build-ca nopass > /dev/null 2>&1
- ./easyrsa --subject-alt-name="${sans}" build-server-full "${master_name}" nopass > /dev/null 2>&1
- ./easyrsa build-client-full kubelet nopass > /dev/null 2>&1
- ./easyrsa build-client-full kubecfg nopass > /dev/null 2>&1) || {
- # If there was an error in the subshell, just die.
- # TODO(roberthbailey): add better error handling here
- echo "=== Failed to generate certificates: Aborting ==="
- exit 2
- }
-
-mkdir -p "$cert_dir"
-
-cp -p pki/ca.crt "${cert_dir}/ca.crt"
-cp -p "pki/issued/${master_name}.crt" "${cert_dir}/server.crt" > /dev/null 2>&1
-cp -p "pki/private/${master_name}.key" "${cert_dir}/server.key" > /dev/null 2>&1
-cp -p pki/issued/kubecfg.crt "${cert_dir}/kubecfg.crt"
-cp -p pki/private/kubecfg.key "${cert_dir}/kubecfg.key"
-cp -p pki/issued/kubelet.crt "${cert_dir}/kubelet.crt"
-cp -p pki/private/kubelet.key "${cert_dir}/kubelet.key"
-
-CERTS=("ca.crt" "server.key" "server.crt" "kubelet.key" "kubelet.crt" "kubecfg.key" "kubecfg.crt")
-for cert in "${CERTS[@]}"; do
-  chgrp "${cert_group}" "${cert_dir}/${cert}"
-  chmod 660 "${cert_dir}/${cert}"
-done

roles/kubernetes/common/meta/main.yml

@@ -1,3 +0,0 @@
----
-dependencies:
-  - { role: etcd }

roles/kubernetes/common/tasks/gen_certs.yml

@@ -1,42 +0,0 @@
----
-#- name: Get create ca cert script from Kubernetes
-#  get_url:
-#    url=https://raw.githubusercontent.com/GoogleCloudPlatform/kubernetes/master/cluster/saltbase/salt/generate-cert/make-ca-cert.sh
-#    dest={{ kube_script_dir }}/make-ca-cert.sh mode=0500
-#    force=yes
-
-- name: certs | install cert generation script
-  copy:
-    src=make-ca-cert.sh
-    dest={{ kube_script_dir }}
-    mode=0500
-  changed_when: false
-
-# FIXME This only generates a cert for one master...
-- name: certs | run cert generation script
-  command:
-    "{{ kube_script_dir }}/make-ca-cert.sh {{ inventory_hostname }}"
-  args:
-    creates: "{{ kube_cert_dir }}/server.crt"
-  environment:
-    MASTER_IP: "{{ hostvars[inventory_hostname]['ip'] | default(hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}"
-    MASTER_NAME: "{{ inventory_hostname }}"
-    DNS_DOMAIN: "{{ dns_domain }}"
-    SERVICE_CLUSTER_IP_RANGE: "{{ kube_service_addresses }}"
-    CERT_DIR: "{{ kube_cert_dir }}"
-    CERT_GROUP: "{{ kube_cert_group }}"
-
-- name: certs | check certificate permissions
-  file:
-    path={{ item }}
-    group={{ kube_cert_group }}
-    owner=kube
-    mode=0440
-  with_items:
-    - "{{ kube_cert_dir }}/ca.crt"
-    - "{{ kube_cert_dir }}/server.crt"
-    - "{{ kube_cert_dir }}/server.key"
-    - "{{ kube_cert_dir }}/kubecfg.crt"
-    - "{{ kube_cert_dir }}/kubecfg.key"
-    - "{{ kube_cert_dir }}/kubelet.crt"
-    - "{{ kube_cert_dir }}/kubelet.key"

roles/kubernetes/common/tasks/main.yml

@@ -1,29 +0,0 @@
----
-- name: define alias command for kubectl all
-  lineinfile:
-    dest=/etc/bash.bashrc
-    line="alias kball='{{ bin_dir }}/kubectl --all-namespaces -o wide'"
-    regexp='^alias kball=.*$'
-    state=present
-    insertafter=EOF
-    create=True
-
-- name: create kubernetes config directory
-  file: path={{ kube_config_dir }} state=directory
-
-- name: create kubernetes script directory
-  file: path={{ kube_script_dir }} state=directory
-
-- name: Make sure manifest directory exists
-  file: path={{ kube_manifest_dir }} state=directory
-
-- name: write the global config file
-  template:
-    src: config.j2
-    dest: "{{ kube_config_dir }}/config"
-  notify:
-    - restart daemons
-
-- include: secrets.yml
-  tags:
-    - secrets

roles/kubernetes/node/defaults/main.yml

@@ -12,7 +12,7 @@ kube_script_dir: "{{ bin_dir }}/kubernetes-scripts"
 kube_config_dir: /etc/kubernetes
 
 # This is where all the cert scripts and certs will be located
-kube_cert_dir: "{{ kube_config_dir }}/certs"
+kube_cert_dir: "{{ kube_config_dir }}/ssl"
 
 # This is where all of the bearer tokens will be stored
 kube_token_dir: "{{ kube_config_dir }}/tokens"
@@ -32,12 +32,16 @@ dns_domain: "{{ cluster_name }}"
 
 kube_proxy_mode: userspace
 
+hyperkube_image:
+  name: gcr.io/google_containers/hyperkube
+  tag: v1.1.2
+
 # IP address of the DNS server.
 # Kubernetes will create a pod with several containers, serving as the DNS
 # server and expose it under this IP address. The IP address must be from
 # the range specified as kube_service_addresses. This magic will actually
 # pick the 10th ip address in the kube_service_addresses range and use that.
-# dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}"
+dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}"
 
 # kube_api_runtime_config:
 #   - extensions/v1beta1/daemonsets=true

roles/kubernetes/node/files/make-ssl.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+# Author: skahlouc@skahlouc-laptop
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o pipefail
+
+usage()
+{
+    cat << EOF
+Create self signed certificates
+
+Usage : $(basename $0) -f <config> [-c <cloud_provider>] [-d <ssldir>] [-g <ssl_group>]
+      -h | --help         : Show this message
+      -f | --config       : Openssl configuration file
+      -c | --cloud        : Cloud provider (GCE, AWS or AZURE)
+      -d | --ssldir       : Directory where the certificates will be installed
+      -g | --sslgrp       : Group of the certificates
+               
+               ex : 
+               $(basename $0) -f openssl.conf -c GCE -d /srv/ssl -g kube
+EOF
+}
+
+# Options parsing
+while (($#)); do
+    case "$1" in
+        -h | --help)   usage;   exit 0;;
+        -f | --config) CONFIG=${2}; shift 2;;
+        -c | --cloud) CLOUD=${2}; shift 2;;
+        -d | --ssldir) SSLDIR="${2}"; shift 2;; 
+        -g | --group) SSLGRP="${2}"; shift 2;;
+        *)
+            usage
+            echo "ERROR : Unknown option"
+            exit 3
+        ;;
+    esac
+done
+
+if [ -z ${CONFIG} ]; then
+    echo "ERROR: the openssl configuration file is missing. option -f"
+    exit 1
+fi
+if [ -z ${SSLDIR} ]; then
+    SSLDIR="/etc/kubernetes/certs"
+fi
+if [ -z ${SSLGRP} ]; then
+    SSLGRP="kube-cert"
+fi
+
+#echo "config=$CONFIG, cloud=$CLOUD, certdir=$SSLDIR, certgroup=$SSLGRP"
+
+SUPPORTED_CLOUDS="GCE AWS AZURE"
+
+# TODO: Add support for discovery on other providers?
+if [ "${CLOUD}" == "GCE" ]; then
+  CLOUD_IP=$(curl -s -H Metadata-Flavor:Google http://metadata.google.internal./computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
+fi
+
+if [ "${CLOUD}" == "AWS" ]; then
+  CLOUD_IP=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)
+fi
+
+if [ "${CLOUD}" == "AZURE" ]; then
+  CLOUD_IP=$(uname -n | awk -F. '{ print $2 }').cloudapp.net
+fi
+
+tmpdir=$(mktemp -d --tmpdir kubernetes_cacert.XXXXXX)
+trap 'rm -rf "${tmpdir}"' EXIT
+cd "${tmpdir}"
+
+mkdir -p "${SSLDIR}"
+
+# Root CA
+openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
+openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" > /dev/null 2>&1
+
+# Apiserver
+openssl genrsa -out apiserver-key.pem 2048 > /dev/null 2>&1
+openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config ${CONFIG} > /dev/null 2>&1
+openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
+
+# Nodes and Admin
+for i in node admin; do
+    openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1
+    openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1
+    openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 > /dev/null 2>&1
+done
+
+# Install certs
+mv *.pem ${SSLDIR}/
+chgrp ${SSLGRP} ${SSLDIR}/*
+chmod 600 ${SSLDIR}/*-key.pem
+chown root:root ${SSLDIR}/*-key.pem

roles/kubernetes/node/handlers/main.yml

@@ -4,7 +4,6 @@
   notify:
     - reload systemd
     - restart reloaded-kubelet
-    - restart reloaded-proxy
 
 - name: reload systemd
   command: systemctl daemon-reload
@@ -19,14 +18,3 @@
   service:
     name: kubelet
     state: restarted
-
-- name: restart proxy
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-proxy
-
-- name: restart reloaded-proxy
-  service:
-    name: kube-proxy
-    state: restarted

roles/kubernetes/node/meta/main.yml

@@ -1,3 +0,0 @@
----
-dependencies:
-  - { role: kubernetes/common }

roles/kubernetes/node/tasks/config.yml

@@ -1,53 +0,0 @@
----
-- name: Get the node token values
-  slurp:
-    src: "{{ kube_token_dir }}/{{ item }}-{{ inventory_hostname }}.token"
-  with_items:
-    - "system:kubelet"
-    - "system:proxy"
-  register: tokens
-  run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Set token facts
-  set_fact:
-    kubelet_token: "{{ tokens.results[0].content|b64decode }}"
-    proxy_token: "{{ tokens.results[1].content|b64decode }}"
-
-- name: Create kubelet environment vars dir
-  file: path=/etc/systemd/system/kubelet.service.d state=directory
-
-- name: Write kubelet config file
-  template: src=kubelet.j2 dest=/etc/systemd/system/kubelet.service.d/10-kubelet.conf backup=yes
-  notify:
-    - restart kubelet
-
-- name: write the kubecfg (auth) file for kubelet
-  template: src=kubelet.kubeconfig.j2 dest={{ kube_config_dir }}/kubelet.kubeconfig backup=yes
-  notify:
-    - restart kubelet
-
-- name: Create proxy environment vars dir
-  file: path=/etc/systemd/system/kube-proxy.service.d state=directory
-
-- name: Write proxy config file
-  template: src=proxy.j2 dest=/etc/systemd/system/kube-proxy.service.d/10-proxy-cluster.conf backup=yes
-  notify:
-    - restart proxy
-
-- name: write the kubecfg (auth) file for kube-proxy
-  template: src=proxy.kubeconfig.j2 dest={{ kube_config_dir }}/proxy.kubeconfig backup=yes
-  notify:
-    - restart proxy
-
-- name: Enable kubelet
-  service:
-    name: kubelet
-    enabled: yes
-    state: started
-
-- name: Enable proxy
-  service:
-    name: kube-proxy
-    enabled: yes
-    state: started

roles/kubernetes/node/tasks/gen_certs.yml

@@ -0,0 +1,28 @@
+---
+- name: certs | install cert generation script
+  copy:
+    src=make-ssl.sh
+    dest={{ kube_script_dir }}
+    mode=0500
+  changed_when: false
+
+- name: certs | write openssl config
+  template:
+    src: "openssl.conf.j2"
+    dest: "{{ kube_config_dir }}/.openssl.conf"
+
+- name: certs | run cert generation script
+  shell: >
+    {{ kube_script_dir }}/make-ssl.sh
+    -f {{ kube_config_dir }}/.openssl.conf
+    -g {{ kube_cert_group }}
+    -d {{ kube_cert_dir }}
+  args:
+    creates: "{{ kube_cert_dir }}/apiserver.pem"
+
+- name: certs | check certificate permissions
+  file:
+    path={{ kube_cert_dir }}
+    group={{ kube_cert_group }}
+    owner=kube
+    recurse=yes

roles/kubernetes/node/tasks/gen_tokens.yml

@@ -10,21 +10,17 @@
   environment:
     TOKEN_DIR: "{{ kube_token_dir }}"
   with_nested:
-    - [ "system:controller_manager", "system:scheduler", "system:kubectl", 'system:proxy' ]
+    - [ "system:kubectl" ]
-    - "{{ groups['kube-master'][0] }}"
+    - "{{ groups['kube-master'] }}"
   register: gentoken
   changed_when: "'Added' in gentoken.stdout"
-  notify:
-    - restart daemons
 
 - name: tokens | generate tokens for node components
   command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
   environment:
     TOKEN_DIR: "{{ kube_token_dir }}"
   with_nested:
-    - [ 'system:kubelet', 'system:proxy' ]
+    - [ 'system:kubelet' ]
     - "{{ groups['kube-node'] }}"
   register: gentoken
   changed_when: "'Added' in gentoken.stdout"
-  notify:
-    - restart daemons

roles/kubernetes/node/tasks/install.yml

@@ -1,20 +1,13 @@
 ---
-- name: Write kube-proxy systemd init file
-  template: src=systemd-init/kube-proxy.service.j2 dest=/etc/systemd/system/kube-proxy.service backup=yes
-  notify: restart daemons
-
 - name: Write kubelet systemd init file
-  template: src=systemd-init/kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes
+  template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes
-  notify: restart daemons
+  notify: restart kubelet
 
-- name: Install kubernetes binaries
+- name: Install kubelet binary
   copy:
-     src={{ local_release_dir }}/kubernetes/bin/{{ item }}
+     src={{ local_release_dir }}/kubernetes/bin/kubelet
      dest={{ bin_dir }}
      owner=kube
      mode=u+x
-  with_items:
-    - kube-proxy
-    - kubelet
   notify:
-    - restart daemons
+    - restart kubelet

roles/kubernetes/node/tasks/main.yml

@@ -1,4 +1,48 @@
 ---
+- name: create kubernetes config directory
+  file: path={{ kube_config_dir }} state=directory
+
+- name: create kubernetes script directory
+  file: path={{ kube_script_dir }} state=directory
+
+- name: Make sure manifest directory exists
+  file: path={{ kube_manifest_dir }} state=directory
+
+- include: secrets.yml
+  tags:
+    - secrets
+
 - include: install.yml
-- include: config.yml
+
+- name: write the global config file
+  template:
+    src: config.j2
+    dest: "{{ kube_config_dir }}/config"
+  notify:
+    - restart kubelet
+
+- name: Create kubelet environment vars dir
+  file: path=/etc/systemd/system/kubelet.service.d state=directory
+
+- name: Write kubelet config file
+  template: src=kubelet.j2 dest=/etc/systemd/system/kubelet.service.d/10-kubelet.conf backup=yes
+  notify:
+    - restart kubelet
+
+- name: write the kubecfg (auth) file for kubelet
+  template: src=node-kubeconfig.yaml.j2 dest={{ kube_config_dir }}/node-kubeconfig.yaml backup=yes
+  notify:
+    - restart kubelet
+
+- name: Write proxy manifest
+  template: 
+    src: manifests/kube-proxy.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-proxy.manifest"
+
+- name: Enable kubelet
+  service:
+    name: kubelet
+    enabled: yes
+    state: started
+
 - include: temp_workaround.yml

roles/kubernetes/node/tasks/secrets.yml

@@ -29,26 +29,36 @@
   run_once: true
   when: inventory_hostname == groups['kube-master'][0]
 
-- name: Read back the CA certificate
-  slurp:
-    src: "{{ kube_cert_dir }}/ca.crt"
-  register: ca_cert
-  run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: certs | register the CA certificate as a fact for later use
-  set_fact:
-    kube_ca_cert: "{{ ca_cert.content|b64decode }}"
-
-- name: certs | write CA certificate everywhere
-  copy: content="{{ kube_ca_cert }}" dest="{{ kube_cert_dir }}/ca.crt"
-  notify:
-    - restart daemons
-
-- debug: msg="{{groups['kube-master'][0]}} == {{inventory_hostname}}"
-  tags:
-    - debug
-
 - include: gen_tokens.yml
   run_once: true
   when: inventory_hostname == groups['kube-master'][0]
+
+# Sync certs between nodes
+- user:
+    name: '{{ansible_user_id}}'
+    generate_ssh_key: yes
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  run_once: yes
+
+- name: 'get ssh keypair'
+  slurp: path=~/.ssh/id_rsa.pub
+  register: public_key
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: 'setup keypair on nodes'
+  authorized_key:
+    user: '{{ansible_user_id}}'
+    key: "{{public_key.content|b64decode }}"
+
+- name: synchronize certificates for nodes
+  synchronize:
+    src: "{{ item }}"
+    dest: "{{ kube_cert_dir }}"
+    recursive: yes
+    delete: yes
+    rsync_opts: [ '--one-file-system']
+  with_items:
+    - "{{ kube_cert_dir}}/ca.pem"
+    - "{{ kube_cert_dir}}/node.pem"
+    - "{{ kube_cert_dir}}/node-key.pem"
+  delegate_to: "{{ groups['kube-master'][0] }}"

roles/kubernetes/node/tasks/temp_workaround.yml

@@ -1,5 +1,2 @@
-- name: Warning Temporary workaround !!! Disable kubelet and kube-proxy on node startup
+- name: Warning Temporary workaround !!! Disable kubelet on node startup
-  service: name={{ item }} enabled=no
+  service: name=kubelet enabled=no
-  with_items:
-    - kubelet
-    - kube-proxy

roles/kubernetes/node/templates/config.j2

@@ -17,10 +17,10 @@
 KUBE_LOGTOSTDERR="--logtostderr=true"
 
 # journal message level, 0 is debug
-KUBE_LOG_LEVEL="--v=5"
+KUBE_LOG_LEVEL="{{ kube_log_level | default('--v=2') }}"
 
 # Should this cluster be allowed to run privileged docker containers
 KUBE_ALLOW_PRIV="--allow_privileged=true"
 
 # How the replication controller, scheduler, and proxy
-KUBE_MASTER="--master=https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}"
+KUBE_MASTER="--master=https://{{ groups['kube-master'][0] }}:{{ kube_apiserver_port }}"

roles/kubernetes/node/templates/kubelet.j2

@@ -1,18 +1,24 @@
 [Service]
 Environment="KUBE_LOGTOSTDERR=--logtostderr=true"
-Environment="KUBE_LOG_LEVEL=--v=0"
+Environment="KUBE_LOG_LEVEL={{ kube_log_level | default('--v=2') }}"
 Environment="KUBE_ALLOW_PRIV=--allow_privileged=true"
-Environment="KUBE_MASTER=--master=https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}"
+{% if inventory_hostname in groups['kube-master'] %}
+Environment="KUBELET_API_SERVER=--api_servers=http://{{ hostvars[inventory_hostname]['ip'] | default(hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}:{{ kube_apiserver_insecure_port }}"
+{% else %}
+Environment="KUBELET_API_SERVER=--api_servers=https://{{ groups['kube-master'][0]}}:{{ kube_apiserver_port }}"
+{% endif %}
 # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
 Environment="KUBELET_ADDRESS=--address=0.0.0.0"
 # The port for the info server to serve on
 # Environment="KUBELET_PORT=--port=10250"
 # You may leave this blank to use the actual hostname
 Environment="KUBELET_HOSTNAME=--hostname_override={{ inventory_hostname }}"
+{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %}
+Environment="KUBELET_REGISTER_NODE=--register-node=false"
+{% endif %}
 # location of the api-server
-Environment="KUBELET_API_SERVER=--api_servers=https://{{ groups['kube-master'][0]}}:{{ kube_master_port }}"
 {% if dns_setup %}
-Environment="KUBELET_ARGS=--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"
+Environment="KUBELET_ARGS=--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }}"
 {% else %}
 Environment="KUBELET_ARGS=--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"
 {% endif %}

roles/kubernetes/node/templates/kubelet.kubeconfig.j2

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Config
-current-context: kubelet-to-{{ cluster_name }}
-preferences: {}
-clusters:
-- cluster:
-    certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: https://{{ groups['kube-master'][0] }}:{{kube_master_port}}
-  name: {{ cluster_name }}
-contexts:
-- context:
-    cluster: {{ cluster_name }}
-    user: kubelet
-  name: kubelet-to-{{ cluster_name }}
-users:
-- name: kubelet
-  user:
-    token: {{ kubelet_token }}

roles/kubernetes/node/templates/kubelet.service.j2

@@ -19,6 +19,7 @@ ExecStart={{ bin_dir }}/kubelet \
 	    $KUBELET_HOSTNAME \
 	    $KUBE_ALLOW_PRIV \
 	    $KUBELET_ARGS \
+	    $KUBELET_REGISTER_NODE \
 	    $KUBELET_NETWORK_PLUGIN
 Restart=on-failure
 

roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-proxy
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-proxy
+    image: {{ hyperkube_image.name }}:{{ hyperkube_image.tag }}
+    command:
+    - /hyperkube
+    - proxy
+    - --v={{ kube_log_level | default('2') }}
+{% if inventory_hostname in groups['kube-master'] %}
+    - --master=http://127.0.0.1:8080
+{% else %}
+    - --master=https://{{ groups['kube-master'][0] }}:{{kube_apiserver_port }} 
+    - --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml
+{% endif %}
+    securityContext:
+      privileged: true
+    volumeMounts:
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+    - mountPath: /etc/kubernetes/node-kubeconfig.yaml
+      name: "kubeconfig"
+      readOnly: true
+    - mountPath: /etc/kubernetes/ssl
+      name: "etc-kube-ssl"
+      readOnly: true
+  volumes:
+  - name: ssl-certs-host
+    hostPath:
+      path: /usr/share/ca-certificates
+  - name: "kubeconfig"
+    hostPath:
+      path: "/etc/kubernetes/node-kubeconfig.yaml"
+  - name: "etc-kube-ssl"
+    hostPath:
+      path: "/etc/kubernetes/ssl"

roles/kubernetes/node/templates/node-kubeconfig.yaml.j2

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    certificate-authority: {{ kube_cert_dir }}/ca.pem
+users:
+- name: kubelet
+  user:
+    client-certificate: {{ kube_cert_dir }}/node.pem
+    client-key: {{ kube_cert_dir }}/node-key.pem
+contexts:
+- context:
+    cluster: local
+    user: kubelet
+  name: kubelet-{{ cluster_name }}
+current-context: kubelet-{{ cluster_name }}

roles/kubernetes/node/templates/openssl.conf.j2

@@ -0,0 +1,21 @@
+[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+[req_distinguished_name]
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = kubernetes
+DNS.2 = kubernetes.default
+DNS.3 = kubernetes.default.svc.{{ dns_domain }}
+{% for host in groups['kube-master'] %}
+IP.{{ loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+{% endfor %}
+{% set idx =  groups['kube-master'] | length | int + 1 %}
+IP.{{ idx | string }} = {{ kube_apiserver_ip }}
+{% if kube_loadbalancer_ip is defined | default('') %}                                                                                      
+{% set idx =  idx | int + 1 %}
+IP.{{ idx | string }} = {{ kube_loadbalancer }}
+{% endif %}

roles/kubernetes/node/templates/proxy.j2

@@ -1,6 +0,0 @@
-###
-# kubernetes proxy config
-
-# default config should be adequate
-[Service]
-Environment="KUBE_PROXY_ARGS=--kubeconfig={{ kube_config_dir }}/proxy.kubeconfig --proxy-mode={{kube_proxy_mode}}"

roles/kubernetes/node/templates/proxy.kubeconfig.j2

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Config
-current-context: proxy-to-{{ cluster_name }}
-preferences: {}
-contexts:
-- context:
-    cluster: {{ cluster_name }}
-    user: proxy
-  name: proxy-to-{{ cluster_name }}
-clusters:
-- cluster:
-    certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
-  name: {{ cluster_name }}
-users:
-- name: proxy
-  user:
-    token: {{ proxy_token }}

roles/kubernetes/node/templates/systemd-init/kube-proxy.service.j2

@@ -1,22 +0,0 @@
-[Unit]
-Description=Kubernetes Kube-Proxy Server
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
-After=docker.service calico-node.service
-{% else %}
-After=docker.service
-{% endif %}
-
-[Service]
-EnvironmentFile=/etc/kubernetes/config
-EnvironmentFile=/etc/network-environment
-ExecStart={{ bin_dir }}/kube-proxy \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_MASTER \
-	    $KUBE_PROXY_ARGS
-Restart=on-failure
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target