diff --git a/roles/kubernetes/common/files/make-ca-cert.sh b/roles/kubernetes/common/files/make-ca-cert.sh deleted file mode 100755 index 3950eec91..000000000 --- a/roles/kubernetes/common/files/make-ca-cert.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash - -# Copyright 2014 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail - -# Caller should set in the ev: -# MASTER_IP - this may be an ip or things like "_use_gce_external_ip_" -# DNS_DOMAIN - which will be passed to minions in --cluster_domain -# SERVICE_CLUSTER_IP_RANGE - where all service IPs are allocated -# MASTER_NAME - I'm not sure what it is... - -# Also the following will be respected -# CERT_DIR - where to place the finished certs -# CERT_GROUP - who the group owner of the cert files should be - -cert_ip="${MASTER_IP:="${1}"}" -master_name="${MASTER_NAME:="kubernetes"}" -service_range="${SERVICE_CLUSTER_IP_RANGE:="10.0.0.0/16"}" -dns_domain="${DNS_DOMAIN:="cluster.local"}" -cert_dir="${CERT_DIR:-"/srv/kubernetes"}" -cert_group="${CERT_GROUP:="kube-cert"}" - -# The following certificate pairs are created: -# -# - ca (the cluster's certificate authority) -# - server -# - kubelet -# - kubecfg (for kubectl) -# -# TODO(roberthbailey): Replace easyrsa with a simple Go program to generate -# the certs that we need. - -# TODO: Add support for discovery on other providers? -if [ "$cert_ip" == "_use_gce_external_ip_" ]; then - cert_ip=$(curl -s -H Metadata-Flavor:Google http://metadata.google.internal./computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip) -fi - -if [ "$cert_ip" == "_use_aws_external_ip_" ]; then - cert_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) -fi - -if [ "$cert_ip" == "_use_azure_dns_name_" ]; then - cert_ip=$(uname -n | awk -F. '{ print $2 }').cloudapp.net -fi - -tmpdir=$(mktemp -d --tmpdir kubernetes_cacert.XXXXXX) -trap 'rm -rf "${tmpdir}"' EXIT -cd "${tmpdir}" - -# TODO: For now, this is a patched tool that makes subject-alt-name work, when -# the fix is upstream move back to the upstream easyrsa. This is cached in GCS -# but is originally taken from: -# https://github.com/brendandburns/easy-rsa/archive/master.tar.gz -# -# To update, do the following: -# curl -o easy-rsa.tar.gz https://github.com/brendandburns/easy-rsa/archive/master.tar.gz -# gsutil cp easy-rsa.tar.gz gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz -# gsutil acl ch -R -g all:R gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz -# -# Due to GCS caching of public objects, it may take time for this to be widely -# distributed. - -# Calculate the first ip address in the service range -octects=($(echo "${service_range}" | sed -e 's|/.*||' -e 's/\./ /g')) -((octects[3]+=1)) -service_ip=$(echo "${octects[*]}" | sed 's/ /./g') - -# Determine appropriete subject alt names -sans="IP:${cert_ip},IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${dns_domain},DNS:${master_name}" - -curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz > /dev/null 2>&1 -tar xzf easy-rsa.tar.gz > /dev/null -cd easy-rsa-master/easyrsa3 - -(./easyrsa init-pki > /dev/null 2>&1 - ./easyrsa --batch "--req-cn=${cert_ip}@$(date +%s)" build-ca nopass > /dev/null 2>&1 - ./easyrsa --subject-alt-name="${sans}" build-server-full "${master_name}" nopass > /dev/null 2>&1 - ./easyrsa build-client-full kubelet nopass > /dev/null 2>&1 - ./easyrsa build-client-full kubecfg nopass > /dev/null 2>&1) || { - # If there was an error in the subshell, just die. - # TODO(roberthbailey): add better error handling here - echo "=== Failed to generate certificates: Aborting ===" - exit 2 - } - -mkdir -p "$cert_dir" - -cp -p pki/ca.crt "${cert_dir}/ca.crt" -cp -p "pki/issued/${master_name}.crt" "${cert_dir}/server.crt" > /dev/null 2>&1 -cp -p "pki/private/${master_name}.key" "${cert_dir}/server.key" > /dev/null 2>&1 -cp -p pki/issued/kubecfg.crt "${cert_dir}/kubecfg.crt" -cp -p pki/private/kubecfg.key "${cert_dir}/kubecfg.key" -cp -p pki/issued/kubelet.crt "${cert_dir}/kubelet.crt" -cp -p pki/private/kubelet.key "${cert_dir}/kubelet.key" - -CERTS=("ca.crt" "server.key" "server.crt" "kubelet.key" "kubelet.crt" "kubecfg.key" "kubecfg.crt") -for cert in "${CERTS[@]}"; do - chgrp "${cert_group}" "${cert_dir}/${cert}" - chmod 660 "${cert_dir}/${cert}" -done diff --git a/roles/kubernetes/common/meta/main.yml b/roles/kubernetes/common/meta/main.yml deleted file mode 100644 index 87756afe1..000000000 --- a/roles/kubernetes/common/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - { role: etcd } diff --git a/roles/kubernetes/common/tasks/gen_certs.yml b/roles/kubernetes/common/tasks/gen_certs.yml deleted file mode 100644 index 74fd4458c..000000000 --- a/roles/kubernetes/common/tasks/gen_certs.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -#- name: Get create ca cert script from Kubernetes -# get_url: -# url=https://raw.githubusercontent.com/GoogleCloudPlatform/kubernetes/master/cluster/saltbase/salt/generate-cert/make-ca-cert.sh -# dest={{ kube_script_dir }}/make-ca-cert.sh mode=0500 -# force=yes - -- name: certs | install cert generation script - copy: - src=make-ca-cert.sh - dest={{ kube_script_dir }} - mode=0500 - changed_when: false - -# FIXME This only generates a cert for one master... -- name: certs | run cert generation script - command: - "{{ kube_script_dir }}/make-ca-cert.sh {{ inventory_hostname }}" - args: - creates: "{{ kube_cert_dir }}/server.crt" - environment: - MASTER_IP: "{{ hostvars[inventory_hostname]['ip'] | default(hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}" - MASTER_NAME: "{{ inventory_hostname }}" - DNS_DOMAIN: "{{ dns_domain }}" - SERVICE_CLUSTER_IP_RANGE: "{{ kube_service_addresses }}" - CERT_DIR: "{{ kube_cert_dir }}" - CERT_GROUP: "{{ kube_cert_group }}" - -- name: certs | check certificate permissions - file: - path={{ item }} - group={{ kube_cert_group }} - owner=kube - mode=0440 - with_items: - - "{{ kube_cert_dir }}/ca.crt" - - "{{ kube_cert_dir }}/server.crt" - - "{{ kube_cert_dir }}/server.key" - - "{{ kube_cert_dir }}/kubecfg.crt" - - "{{ kube_cert_dir }}/kubecfg.key" - - "{{ kube_cert_dir }}/kubelet.crt" - - "{{ kube_cert_dir }}/kubelet.key" diff --git a/roles/kubernetes/common/tasks/main.yml b/roles/kubernetes/common/tasks/main.yml deleted file mode 100644 index 76d3bbc80..000000000 --- a/roles/kubernetes/common/tasks/main.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -- name: define alias command for kubectl all - lineinfile: - dest=/etc/bash.bashrc - line="alias kball='{{ bin_dir }}/kubectl --all-namespaces -o wide'" - regexp='^alias kball=.*$' - state=present - insertafter=EOF - create=True - -- name: create kubernetes config directory - file: path={{ kube_config_dir }} state=directory - -- name: create kubernetes script directory - file: path={{ kube_script_dir }} state=directory - -- name: Make sure manifest directory exists - file: path={{ kube_manifest_dir }} state=directory - -- name: write the global config file - template: - src: config.j2 - dest: "{{ kube_config_dir }}/config" - notify: - - restart daemons - -- include: secrets.yml - tags: - - secrets diff --git a/roles/kubernetes/common/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml similarity index 88% rename from roles/kubernetes/common/defaults/main.yml rename to roles/kubernetes/node/defaults/main.yml index 69d619ae0..d48d72b66 100644 --- a/roles/kubernetes/common/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -12,7 +12,7 @@ kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" kube_config_dir: /etc/kubernetes # This is where all the cert scripts and certs will be located -kube_cert_dir: "{{ kube_config_dir }}/certs" +kube_cert_dir: "{{ kube_config_dir }}/ssl" # This is where all of the bearer tokens will be stored kube_token_dir: "{{ kube_config_dir }}/tokens" @@ -32,12 +32,16 @@ dns_domain: "{{ cluster_name }}" kube_proxy_mode: userspace +hyperkube_image: + name: gcr.io/google_containers/hyperkube + tag: v1.1.2 + # IP address of the DNS server. # Kubernetes will create a pod with several containers, serving as the DNS # server and expose it under this IP address. The IP address must be from # the range specified as kube_service_addresses. This magic will actually # pick the 10th ip address in the kube_service_addresses range and use that. -# dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}" +dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}" # kube_api_runtime_config: # - extensions/v1beta1/daemonsets=true diff --git a/roles/kubernetes/common/files/kube-gen-token.sh b/roles/kubernetes/node/files/kube-gen-token.sh similarity index 100% rename from roles/kubernetes/common/files/kube-gen-token.sh rename to roles/kubernetes/node/files/kube-gen-token.sh diff --git a/roles/kubernetes/node/files/make-ssl.sh b/roles/kubernetes/node/files/make-ssl.sh new file mode 100644 index 000000000..9ab0a49df --- /dev/null +++ b/roles/kubernetes/node/files/make-ssl.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +# Author: skahlouc@skahlouc-laptop +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o pipefail + +usage() +{ + cat << EOF +Create self signed certificates + +Usage : $(basename $0) -f [-c ] [-d ] [-g ] + -h | --help : Show this message + -f | --config : Openssl configuration file + -c | --cloud : Cloud provider (GCE, AWS or AZURE) + -d | --ssldir : Directory where the certificates will be installed + -g | --sslgrp : Group of the certificates + + ex : + $(basename $0) -f openssl.conf -c GCE -d /srv/ssl -g kube +EOF +} + +# Options parsing +while (($#)); do + case "$1" in + -h | --help) usage; exit 0;; + -f | --config) CONFIG=${2}; shift 2;; + -c | --cloud) CLOUD=${2}; shift 2;; + -d | --ssldir) SSLDIR="${2}"; shift 2;; + -g | --group) SSLGRP="${2}"; shift 2;; + *) + usage + echo "ERROR : Unknown option" + exit 3 + ;; + esac +done + +if [ -z ${CONFIG} ]; then + echo "ERROR: the openssl configuration file is missing. option -f" + exit 1 +fi +if [ -z ${SSLDIR} ]; then + SSLDIR="/etc/kubernetes/certs" +fi +if [ -z ${SSLGRP} ]; then + SSLGRP="kube-cert" +fi + +#echo "config=$CONFIG, cloud=$CLOUD, certdir=$SSLDIR, certgroup=$SSLGRP" + +SUPPORTED_CLOUDS="GCE AWS AZURE" + +# TODO: Add support for discovery on other providers? +if [ "${CLOUD}" == "GCE" ]; then + CLOUD_IP=$(curl -s -H Metadata-Flavor:Google http://metadata.google.internal./computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip) +fi + +if [ "${CLOUD}" == "AWS" ]; then + CLOUD_IP=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4) +fi + +if [ "${CLOUD}" == "AZURE" ]; then + CLOUD_IP=$(uname -n | awk -F. '{ print $2 }').cloudapp.net +fi + +tmpdir=$(mktemp -d --tmpdir kubernetes_cacert.XXXXXX) +trap 'rm -rf "${tmpdir}"' EXIT +cd "${tmpdir}" + +mkdir -p "${SSLDIR}" + +# Root CA +openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1 +openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" > /dev/null 2>&1 + +# Apiserver +openssl genrsa -out apiserver-key.pem 2048 > /dev/null 2>&1 +openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config ${CONFIG} > /dev/null 2>&1 +openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1 + +# Nodes and Admin +for i in node admin; do + openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1 + openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1 + openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 > /dev/null 2>&1 +done + +# Install certs +mv *.pem ${SSLDIR}/ +chgrp ${SSLGRP} ${SSLDIR}/* +chmod 600 ${SSLDIR}/*-key.pem +chown root:root ${SSLDIR}/*-key.pem diff --git a/roles/kubernetes/node/handlers/main.yml b/roles/kubernetes/node/handlers/main.yml index 9abb8ff25..162c4cde1 100644 --- a/roles/kubernetes/node/handlers/main.yml +++ b/roles/kubernetes/node/handlers/main.yml @@ -4,7 +4,6 @@ notify: - reload systemd - restart reloaded-kubelet - - restart reloaded-proxy - name: reload systemd command: systemctl daemon-reload @@ -19,14 +18,3 @@ service: name: kubelet state: restarted - -- name: restart proxy - command: /bin/true - notify: - - reload systemd - - restart reloaded-proxy - -- name: restart reloaded-proxy - service: - name: kube-proxy - state: restarted diff --git a/roles/kubernetes/node/meta/main.yml b/roles/kubernetes/node/meta/main.yml deleted file mode 100644 index 31675692c..000000000 --- a/roles/kubernetes/node/meta/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -dependencies: - - { role: kubernetes/common } diff --git a/roles/kubernetes/node/tasks/config.yml b/roles/kubernetes/node/tasks/config.yml deleted file mode 100644 index c1d5f29b2..000000000 --- a/roles/kubernetes/node/tasks/config.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -- name: Get the node token values - slurp: - src: "{{ kube_token_dir }}/{{ item }}-{{ inventory_hostname }}.token" - with_items: - - "system:kubelet" - - "system:proxy" - register: tokens - run_once: true - delegate_to: "{{ groups['kube-master'][0] }}" - -- name: Set token facts - set_fact: - kubelet_token: "{{ tokens.results[0].content|b64decode }}" - proxy_token: "{{ tokens.results[1].content|b64decode }}" - -- name: Create kubelet environment vars dir - file: path=/etc/systemd/system/kubelet.service.d state=directory - -- name: Write kubelet config file - template: src=kubelet.j2 dest=/etc/systemd/system/kubelet.service.d/10-kubelet.conf backup=yes - notify: - - restart kubelet - -- name: write the kubecfg (auth) file for kubelet - template: src=kubelet.kubeconfig.j2 dest={{ kube_config_dir }}/kubelet.kubeconfig backup=yes - notify: - - restart kubelet - -- name: Create proxy environment vars dir - file: path=/etc/systemd/system/kube-proxy.service.d state=directory - -- name: Write proxy config file - template: src=proxy.j2 dest=/etc/systemd/system/kube-proxy.service.d/10-proxy-cluster.conf backup=yes - notify: - - restart proxy - -- name: write the kubecfg (auth) file for kube-proxy - template: src=proxy.kubeconfig.j2 dest={{ kube_config_dir }}/proxy.kubeconfig backup=yes - notify: - - restart proxy - -- name: Enable kubelet - service: - name: kubelet - enabled: yes - state: started - -- name: Enable proxy - service: - name: kube-proxy - enabled: yes - state: started diff --git a/roles/kubernetes/node/tasks/gen_certs.yml b/roles/kubernetes/node/tasks/gen_certs.yml new file mode 100644 index 000000000..a4f70ce54 --- /dev/null +++ b/roles/kubernetes/node/tasks/gen_certs.yml @@ -0,0 +1,28 @@ +--- +- name: certs | install cert generation script + copy: + src=make-ssl.sh + dest={{ kube_script_dir }} + mode=0500 + changed_when: false + +- name: certs | write openssl config + template: + src: "openssl.conf.j2" + dest: "{{ kube_config_dir }}/.openssl.conf" + +- name: certs | run cert generation script + shell: > + {{ kube_script_dir }}/make-ssl.sh + -f {{ kube_config_dir }}/.openssl.conf + -g {{ kube_cert_group }} + -d {{ kube_cert_dir }} + args: + creates: "{{ kube_cert_dir }}/apiserver.pem" + +- name: certs | check certificate permissions + file: + path={{ kube_cert_dir }} + group={{ kube_cert_group }} + owner=kube + recurse=yes diff --git a/roles/kubernetes/common/tasks/gen_tokens.yml b/roles/kubernetes/node/tasks/gen_tokens.yml similarity index 74% rename from roles/kubernetes/common/tasks/gen_tokens.yml rename to roles/kubernetes/node/tasks/gen_tokens.yml index cf77d4399..f2e5625f9 100644 --- a/roles/kubernetes/common/tasks/gen_tokens.yml +++ b/roles/kubernetes/node/tasks/gen_tokens.yml @@ -10,21 +10,17 @@ environment: TOKEN_DIR: "{{ kube_token_dir }}" with_nested: - - [ "system:controller_manager", "system:scheduler", "system:kubectl", 'system:proxy' ] - - "{{ groups['kube-master'][0] }}" + - [ "system:kubectl" ] + - "{{ groups['kube-master'] }}" register: gentoken changed_when: "'Added' in gentoken.stdout" - notify: - - restart daemons - name: tokens | generate tokens for node components command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}" environment: TOKEN_DIR: "{{ kube_token_dir }}" with_nested: - - [ 'system:kubelet', 'system:proxy' ] + - [ 'system:kubelet' ] - "{{ groups['kube-node'] }}" register: gentoken changed_when: "'Added' in gentoken.stdout" - notify: - - restart daemons diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index 0772393ff..e1f45460a 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -1,20 +1,13 @@ --- -- name: Write kube-proxy systemd init file - template: src=systemd-init/kube-proxy.service.j2 dest=/etc/systemd/system/kube-proxy.service backup=yes - notify: restart daemons - - name: Write kubelet systemd init file - template: src=systemd-init/kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes - notify: restart daemons + template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes + notify: restart kubelet -- name: Install kubernetes binaries +- name: Install kubelet binary copy: - src={{ local_release_dir }}/kubernetes/bin/{{ item }} + src={{ local_release_dir }}/kubernetes/bin/kubelet dest={{ bin_dir }} owner=kube mode=u+x - with_items: - - kube-proxy - - kubelet notify: - - restart daemons + - restart kubelet diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index e0efbaf73..7b5e29da9 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -1,4 +1,48 @@ --- +- name: create kubernetes config directory + file: path={{ kube_config_dir }} state=directory + +- name: create kubernetes script directory + file: path={{ kube_script_dir }} state=directory + +- name: Make sure manifest directory exists + file: path={{ kube_manifest_dir }} state=directory + +- include: secrets.yml + tags: + - secrets + - include: install.yml -- include: config.yml + +- name: write the global config file + template: + src: config.j2 + dest: "{{ kube_config_dir }}/config" + notify: + - restart kubelet + +- name: Create kubelet environment vars dir + file: path=/etc/systemd/system/kubelet.service.d state=directory + +- name: Write kubelet config file + template: src=kubelet.j2 dest=/etc/systemd/system/kubelet.service.d/10-kubelet.conf backup=yes + notify: + - restart kubelet + +- name: write the kubecfg (auth) file for kubelet + template: src=node-kubeconfig.yaml.j2 dest={{ kube_config_dir }}/node-kubeconfig.yaml backup=yes + notify: + - restart kubelet + +- name: Write proxy manifest + template: + src: manifests/kube-proxy.manifest.j2 + dest: "{{ kube_manifest_dir }}/kube-proxy.manifest" + +- name: Enable kubelet + service: + name: kubelet + enabled: yes + state: started + - include: temp_workaround.yml diff --git a/roles/kubernetes/common/tasks/secrets.yml b/roles/kubernetes/node/tasks/secrets.yml similarity index 54% rename from roles/kubernetes/common/tasks/secrets.yml rename to roles/kubernetes/node/tasks/secrets.yml index c61e17d9b..1fdb99f98 100644 --- a/roles/kubernetes/common/tasks/secrets.yml +++ b/roles/kubernetes/node/tasks/secrets.yml @@ -29,26 +29,36 @@ run_once: true when: inventory_hostname == groups['kube-master'][0] -- name: Read back the CA certificate - slurp: - src: "{{ kube_cert_dir }}/ca.crt" - register: ca_cert - run_once: true - delegate_to: "{{ groups['kube-master'][0] }}" - -- name: certs | register the CA certificate as a fact for later use - set_fact: - kube_ca_cert: "{{ ca_cert.content|b64decode }}" - -- name: certs | write CA certificate everywhere - copy: content="{{ kube_ca_cert }}" dest="{{ kube_cert_dir }}/ca.crt" - notify: - - restart daemons - -- debug: msg="{{groups['kube-master'][0]}} == {{inventory_hostname}}" - tags: - - debug - - include: gen_tokens.yml run_once: true when: inventory_hostname == groups['kube-master'][0] + +# Sync certs between nodes +- user: + name: '{{ansible_user_id}}' + generate_ssh_key: yes + delegate_to: "{{ groups['kube-master'][0] }}" + run_once: yes + +- name: 'get ssh keypair' + slurp: path=~/.ssh/id_rsa.pub + register: public_key + delegate_to: "{{ groups['kube-master'][0] }}" + +- name: 'setup keypair on nodes' + authorized_key: + user: '{{ansible_user_id}}' + key: "{{public_key.content|b64decode }}" + +- name: synchronize certificates for nodes + synchronize: + src: "{{ item }}" + dest: "{{ kube_cert_dir }}" + recursive: yes + delete: yes + rsync_opts: [ '--one-file-system'] + with_items: + - "{{ kube_cert_dir}}/ca.pem" + - "{{ kube_cert_dir}}/node.pem" + - "{{ kube_cert_dir}}/node-key.pem" + delegate_to: "{{ groups['kube-master'][0] }}" diff --git a/roles/kubernetes/node/tasks/temp_workaround.yml b/roles/kubernetes/node/tasks/temp_workaround.yml index 8dcefe5e8..a6ef09f4d 100644 --- a/roles/kubernetes/node/tasks/temp_workaround.yml +++ b/roles/kubernetes/node/tasks/temp_workaround.yml @@ -1,5 +1,2 @@ -- name: Warning Temporary workaround !!! Disable kubelet and kube-proxy on node startup - service: name={{ item }} enabled=no - with_items: - - kubelet - - kube-proxy +- name: Warning Temporary workaround !!! Disable kubelet on node startup + service: name=kubelet enabled=no diff --git a/roles/kubernetes/common/templates/config.j2 b/roles/kubernetes/node/templates/config.j2 similarity index 89% rename from roles/kubernetes/common/templates/config.j2 rename to roles/kubernetes/node/templates/config.j2 index 526160a7b..f68dffb3d 100644 --- a/roles/kubernetes/common/templates/config.j2 +++ b/roles/kubernetes/node/templates/config.j2 @@ -17,10 +17,10 @@ KUBE_LOGTOSTDERR="--logtostderr=true" # journal message level, 0 is debug -KUBE_LOG_LEVEL="--v=5" +KUBE_LOG_LEVEL="{{ kube_log_level | default('--v=2') }}" # Should this cluster be allowed to run privileged docker containers KUBE_ALLOW_PRIV="--allow_privileged=true" # How the replication controller, scheduler, and proxy -KUBE_MASTER="--master=https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}" +KUBE_MASTER="--master=https://{{ groups['kube-master'][0] }}:{{ kube_apiserver_port }}" diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index 0a516b5cc..b062a055a 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -1,18 +1,24 @@ [Service] Environment="KUBE_LOGTOSTDERR=--logtostderr=true" -Environment="KUBE_LOG_LEVEL=--v=0" +Environment="KUBE_LOG_LEVEL={{ kube_log_level | default('--v=2') }}" Environment="KUBE_ALLOW_PRIV=--allow_privileged=true" -Environment="KUBE_MASTER=--master=https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}" +{% if inventory_hostname in groups['kube-master'] %} +Environment="KUBELET_API_SERVER=--api_servers=http://{{ hostvars[inventory_hostname]['ip'] | default(hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}:{{ kube_apiserver_insecure_port }}" +{% else %} +Environment="KUBELET_API_SERVER=--api_servers=https://{{ groups['kube-master'][0]}}:{{ kube_apiserver_port }}" +{% endif %} # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces) Environment="KUBELET_ADDRESS=--address=0.0.0.0" # The port for the info server to serve on # Environment="KUBELET_PORT=--port=10250" # You may leave this blank to use the actual hostname Environment="KUBELET_HOSTNAME=--hostname_override={{ inventory_hostname }}" +{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %} +Environment="KUBELET_REGISTER_NODE=--register-node=false" +{% endif %} # location of the api-server -Environment="KUBELET_API_SERVER=--api_servers=https://{{ groups['kube-master'][0]}}:{{ kube_master_port }}" {% if dns_setup %} -Environment="KUBELET_ARGS=--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}" +Environment="KUBELET_ARGS=--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }}" {% else %} Environment="KUBELET_ARGS=--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}" {% endif %} diff --git a/roles/kubernetes/node/templates/kubelet.kubeconfig.j2 b/roles/kubernetes/node/templates/kubelet.kubeconfig.j2 deleted file mode 100644 index 28eda1e03..000000000 --- a/roles/kubernetes/node/templates/kubelet.kubeconfig.j2 +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Config -current-context: kubelet-to-{{ cluster_name }} -preferences: {} -clusters: -- cluster: - certificate-authority: {{ kube_cert_dir }}/ca.crt - server: https://{{ groups['kube-master'][0] }}:{{kube_master_port}} - name: {{ cluster_name }} -contexts: -- context: - cluster: {{ cluster_name }} - user: kubelet - name: kubelet-to-{{ cluster_name }} -users: -- name: kubelet - user: - token: {{ kubelet_token }} diff --git a/roles/kubernetes/node/templates/systemd-init/kubelet.service.j2 b/roles/kubernetes/node/templates/kubelet.service.j2 similarity index 95% rename from roles/kubernetes/node/templates/systemd-init/kubelet.service.j2 rename to roles/kubernetes/node/templates/kubelet.service.j2 index 338b4b23c..c09ff795d 100644 --- a/roles/kubernetes/node/templates/systemd-init/kubelet.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.service.j2 @@ -19,6 +19,7 @@ ExecStart={{ bin_dir }}/kubelet \ $KUBELET_HOSTNAME \ $KUBE_ALLOW_PRIV \ $KUBELET_ARGS \ + $KUBELET_REGISTER_NODE \ $KUBELET_NETWORK_PLUGIN Restart=on-failure diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 new file mode 100644 index 000000000..5d8aef5c0 --- /dev/null +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kube-proxy + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: kube-proxy + image: {{ hyperkube_image.name }}:{{ hyperkube_image.tag }} + command: + - /hyperkube + - proxy + - --v={{ kube_log_level | default('2') }} +{% if inventory_hostname in groups['kube-master'] %} + - --master=http://127.0.0.1:8080 +{% else %} + - --master=https://{{ groups['kube-master'][0] }}:{{kube_apiserver_port }} + - --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml +{% endif %} + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/ssl/certs + name: ssl-certs-host + readOnly: true + - mountPath: /etc/kubernetes/node-kubeconfig.yaml + name: "kubeconfig" + readOnly: true + - mountPath: /etc/kubernetes/ssl + name: "etc-kube-ssl" + readOnly: true + volumes: + - name: ssl-certs-host + hostPath: + path: /usr/share/ca-certificates + - name: "kubeconfig" + hostPath: + path: "/etc/kubernetes/node-kubeconfig.yaml" + - name: "etc-kube-ssl" + hostPath: + path: "/etc/kubernetes/ssl" diff --git a/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 new file mode 100644 index 000000000..d21b8eef3 --- /dev/null +++ b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Config +clusters: +- name: local + cluster: + certificate-authority: {{ kube_cert_dir }}/ca.pem +users: +- name: kubelet + user: + client-certificate: {{ kube_cert_dir }}/node.pem + client-key: {{ kube_cert_dir }}/node-key.pem +contexts: +- context: + cluster: local + user: kubelet + name: kubelet-{{ cluster_name }} +current-context: kubelet-{{ cluster_name }} diff --git a/roles/kubernetes/node/templates/openssl.conf.j2 b/roles/kubernetes/node/templates/openssl.conf.j2 new file mode 100644 index 000000000..05015651f --- /dev/null +++ b/roles/kubernetes/node/templates/openssl.conf.j2 @@ -0,0 +1,21 @@ +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names +[alt_names] +DNS.1 = kubernetes +DNS.2 = kubernetes.default +DNS.3 = kubernetes.default.svc.{{ dns_domain }} +{% for host in groups['kube-master'] %} +IP.{{ loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} +{% endfor %} +{% set idx = groups['kube-master'] | length | int + 1 %} +IP.{{ idx | string }} = {{ kube_apiserver_ip }} +{% if kube_loadbalancer_ip is defined | default('') %} +{% set idx = idx | int + 1 %} +IP.{{ idx | string }} = {{ kube_loadbalancer }} +{% endif %} diff --git a/roles/kubernetes/node/templates/proxy.j2 b/roles/kubernetes/node/templates/proxy.j2 deleted file mode 100644 index f529d7d5e..000000000 --- a/roles/kubernetes/node/templates/proxy.j2 +++ /dev/null @@ -1,6 +0,0 @@ -### -# kubernetes proxy config - -# default config should be adequate -[Service] -Environment="KUBE_PROXY_ARGS=--kubeconfig={{ kube_config_dir }}/proxy.kubeconfig --proxy-mode={{kube_proxy_mode}}" diff --git a/roles/kubernetes/node/templates/proxy.kubeconfig.j2 b/roles/kubernetes/node/templates/proxy.kubeconfig.j2 deleted file mode 100644 index 78d181631..000000000 --- a/roles/kubernetes/node/templates/proxy.kubeconfig.j2 +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Config -current-context: proxy-to-{{ cluster_name }} -preferences: {} -contexts: -- context: - cluster: {{ cluster_name }} - user: proxy - name: proxy-to-{{ cluster_name }} -clusters: -- cluster: - certificate-authority: {{ kube_cert_dir }}/ca.crt - server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }} - name: {{ cluster_name }} -users: -- name: proxy - user: - token: {{ proxy_token }} diff --git a/roles/kubernetes/node/templates/systemd-init/kube-proxy.service.j2 b/roles/kubernetes/node/templates/systemd-init/kube-proxy.service.j2 deleted file mode 100644 index b1170c5d8..000000000 --- a/roles/kubernetes/node/templates/systemd-init/kube-proxy.service.j2 +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=Kubernetes Kube-Proxy Server -Documentation=https://github.com/GoogleCloudPlatform/kubernetes -{% if kube_network_plugin is defined and kube_network_plugin == "calico" %} -After=docker.service calico-node.service -{% else %} -After=docker.service -{% endif %} - -[Service] -EnvironmentFile=/etc/kubernetes/config -EnvironmentFile=/etc/network-environment -ExecStart={{ bin_dir }}/kube-proxy \ - $KUBE_LOGTOSTDERR \ - $KUBE_LOG_LEVEL \ - $KUBE_MASTER \ - $KUBE_PROXY_ARGS -Restart=on-failure -LimitNOFILE=65536 - -[Install] -WantedBy=multi-user.target