From 0cdbc13f1efc4a69b1ae2b7082eb3d0538aa0f66 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 8 Jul 2016 13:59:21 +0400 Subject: [PATCH] Deploy kubelet and kube-apiserver as containers kubelet via docker kube-apiserver as a static pod Fixed etcd service start to be more tolerant of slow start. Workaround for kube_version to stay in download role, but not download an files by creating a new "nothing" download entry. --- roles/download/defaults/main.yml | 20 +-- roles/download/vars/kube_versions.yml | 10 -- roles/kubernetes/master/meta/main.yml | 2 - roles/kubernetes/master/tasks/main.yml | 66 +++------- roles/kubernetes/master/tasks/pre-upgrade.yml | 25 ++++ roles/kubernetes/master/tasks/start.yml | 22 ---- .../templates/deb-kube-apiserver.initd.j2 | 118 ------------------ .../master/templates/kube-apiserver.j2 | 58 --------- .../templates/kube-apiserver.service.j2 | 30 ----- .../manifests/kube-apiserver.manifest.j2 | 21 ++-- roles/kubernetes/node/meta/main.yml | 4 +- roles/kubernetes/node/tasks/install.yml | 6 +- roles/kubernetes/node/tasks/main.yml | 2 +- .../node/templates/kubelet-container.j2 | 15 +++ roles/kubernetes/node/templates/kubelet.j2 | 7 +- .../node/templates/kubelet.service.j2 | 5 +- .../calico/templates/calicoctl-container.j2 | 2 +- roles/uploads/defaults/main.yml | 18 --- roles/uploads/vars/kube_versions.yml | 5 - 19 files changed, 88 insertions(+), 348 deletions(-) create mode 100644 roles/kubernetes/master/tasks/pre-upgrade.yml delete mode 100644 roles/kubernetes/master/tasks/start.yml delete mode 100644 roles/kubernetes/master/templates/deb-kube-apiserver.initd.j2 delete mode 100644 roles/kubernetes/master/templates/kube-apiserver.j2 delete mode 100644 roles/kubernetes/master/templates/kube-apiserver.service.j2 create mode 100644 roles/kubernetes/node/templates/kubelet-container.j2 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index c402b1efa..2353beca3 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -15,8 +15,6 @@ calico_cni_version: v1.3.1 weave_version: v1.5.0 # Download URL's -kubelet_download_url: "https://storage.googleapis.com/kargo/{{kube_version}}_kubernetes-kubelet" -apiserver_download_url: "https://storage.googleapis.com/kargo/{{kube_version}}_kubernetes-apiserver" kubectl_download_url: "https://storage.googleapis.com/kargo/{{kube_version}}_kubernetes-kubectl" etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd" @@ -64,14 +62,6 @@ downloads: unarchive: true owner: "etcd" mode: "0755" - kubernetes_kubelet: - version: "{{kube_version}}" - dest: kubernetes/bin/kubelet - sha256: "{{vars['kube_checksum'][kube_version]['kubelet']}}" - source_url: "{{ kubelet_download_url }}" - url: "{{ kubelet_download_url }}" - owner: "kube" - mode: "0755" kubernetes_kubectl: dest: kubernetes/bin/kubectl version: "{{kube_version}}" @@ -80,14 +70,8 @@ downloads: url: "{{ kubectl_download_url }}" owner: "kube" mode: "0755" - kubernetes_apiserver: - dest: kubernetes/bin/kube-apiserver - version: "{{kube_version}}" - sha256: "{{vars['kube_checksum'][kube_version]['kube_apiserver']}}" - source_url: "{{ apiserver_download_url }}" - url: "{{ apiserver_download_url }}" - owner: "kube" - mode: "0755" + nothing: + enabled: false download: enabled: "{{ file.enabled|default('true') }}" diff --git a/roles/download/vars/kube_versions.yml b/roles/download/vars/kube_versions.yml index 5b5f64a42..9b1a162c8 100644 --- a/roles/download/vars/kube_versions.yml +++ b/roles/download/vars/kube_versions.yml @@ -1,22 +1,12 @@ kube_checksum: v1.2.2: - kube_apiserver: eb1bfd8b877052cbd1991b8c429a1d06661f4cb019905e20e128174f724e16de kubectl: 473e6924569fba30d4a50cecdc2cae5f31d97d1f662463e85b74a472105dcff4 - kubelet: f16827dc7e7c82f0e215f0fc73eb01e2dfe91a2ec83f9cbcaf8d37c91b64fd3b v1.2.3: - kube_apiserver_checksum: ebaeeeb72cb29b358337b330617a96355ff2d08a5a523fc1a81beba36cc9d6f9 kubectl_checksum: 394853edd409a721bcafe4f1360009ef9f845050719fe7d6fc7176f45cc92a8c - kubelet_checksum: 633bb41c51c5c0df0645dd60ba82b12eba39d009eb87bae9227de7d9a89c0797 v1.2.4: - kube_apiserver: 6ac99b36b02968459e026fcfc234207c66064b5e11816b69dd8fc234b2ffec1e kubectl: dac61fbd506f7a17540feca691cd8a9d9d628d59661eebce788a50511f578897 - kubelet: 4adaf40592248eef6fd4fa126464915ea41e624a70dc77178089760ed235e341 v1.2.5: - kube_apiserver: fbe8296ad4b194c06f6802a126d35cd2887dc1aded308d4da2b580f270412b33 kubectl: 5526a496a84701015485e32c86486e2f23599f7a865164f546e619c6a62f7f19 - kubelet: cd15b929f0190876216f397c2c6e7aa8c08d3b047fd90b4980cd68c8f4896211 v1.3.0: - kube_apiserver: 431cd312984a29f45590138e990d5c4d537b069b71f2587a72414fabc4fcffdd kubectl: f40b2d0ff33984e663a0dea4916f1cb9041abecc09b11f9372cdb8049ded95dc - kubelet: bd5f10ccb95fe6e95ddf7ad8a119195c27cb2bce4be6f80c1810ff1a2111496d kube_version: v1.3.0 diff --git a/roles/kubernetes/master/meta/main.yml b/roles/kubernetes/master/meta/main.yml index 11f02f99d..bd1008ae6 100644 --- a/roles/kubernetes/master/meta/main.yml +++ b/roles/kubernetes/master/meta/main.yml @@ -2,7 +2,5 @@ dependencies: - role: download file: "{{ downloads.kubernetes_kubectl }}" - - role: download - file: "{{ downloads.kubernetes_apiserver }}" - { role: etcd } - { role: kubernetes/node } diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index deaf017f3..2d9221ba1 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -1,52 +1,34 @@ --- +- include: pre-upgrade.yml + - name: Copy kubectl bash completion copy: src: kubectl_bash_completion.sh dest: /etc/bash_completion.d/kubectl.sh when: ansible_os_family in ["Debian","RedHat"] -- name: Copy kube-apiserver binary - command: rsync -piu "{{ local_release_dir }}/kubernetes/bin/kube-apiserver" "{{ bin_dir }}/kube-apiserver" - register: kube_apiserver_copy - changed_when: false - - name: Copy kubectl binary command: rsync -piu "{{ local_release_dir }}/kubernetes/bin/kubectl" "{{ bin_dir }}/kubectl" changed_when: false -- name: install | Write kube-apiserver systemd init file - template: - src: "kube-apiserver.service.j2" - dest: "/etc/systemd/system/kube-apiserver.service" - backup: yes - when: ansible_service_mgr == "systemd" - notify: restart kube-apiserver - -- name: install | Write kube-apiserver initd script - template: - src: "deb-kube-apiserver.initd.j2" - dest: "/etc/init.d/kube-apiserver" - owner: root - mode: 0755 - backup: yes - when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "Debian" - -- name: Write kube-apiserver config file - template: - src: "kube-apiserver.j2" - dest: "{{ kube_config_dir }}/kube-apiserver.env" - backup: yes - notify: restart kube-apiserver - -- name: Allow apiserver to bind on both secure and insecure ports - shell: setcap cap_net_bind_service+ep {{ bin_dir }}/kube-apiserver - changed_when: false - - meta: flush_handlers -- include: start.yml - with_items: "{{ groups['kube-master'] }}" - when: "{{ hostvars[item].inventory_hostname == inventory_hostname }}" +- name: Write kube-apiserver manifest + template: + src: manifests/kube-apiserver.manifest.j2 + dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest" + register: apiserver_manifest + +- name: restart kubelet + service: + name: kubelet + state: restarted + when: apiserver_manifest.changed + +- name: wait for the apiserver to be running + wait_for: + port: "{{kube_apiserver_insecure_port}}" + timeout: 60 # Create kube-system namespace - name: copy 'kube-system' namespace manifest @@ -61,17 +43,13 @@ failed_when: False run_once: yes -- name: wait for the apiserver to be running - wait_for: - port: "{{kube_apiserver_insecure_port}}" - timeout: 60 - name: Create 'kube-system' namespace command: "{{ bin_dir }}/kubectl create -f /etc/kubernetes/kube-system-ns.yml" changed_when: False when: kubesystem|failed and inventory_hostname == groups['kube-master'][0] -# Write manifests +# Write other manifests - name: Write kube-controller-manager manifest template: src: manifests/kube-controller-manager.manifest.j2 @@ -81,9 +59,3 @@ template: src: manifests/kube-scheduler.manifest.j2 dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest" - -- name: restart kubelet - service: - name: kubelet - state: restarted - changed_when: false diff --git a/roles/kubernetes/master/tasks/pre-upgrade.yml b/roles/kubernetes/master/tasks/pre-upgrade.yml new file mode 100644 index 000000000..3b9f26de1 --- /dev/null +++ b/roles/kubernetes/master/tasks/pre-upgrade.yml @@ -0,0 +1,25 @@ +--- +- name: "Pre-upgrade | check for kube-apiserver unit file" + stat: + path: /etc/systemd/system/kube-apiserver.service + register: kube_apiserver_service_file + +- name: "Pre-upgrade | check for kube-apiserver init script" + stat: + path: /etc/init.d/kube-apiserver + register: kube_apiserver_init_script + +- name: "Pre-upgrade | stop kube-apiserver if service defined" + service: + name: kube-apiserver + state: stopped + when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False)) + +- name: "Pre-upgrade | remove kube-apiserver service definition" + file: + path: "{{ item }}" + state: absent + when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False)) + with_items: + - /etc/systemd/system/kube-apiserver.service + - /etc/init.d/kube-apiserver diff --git a/roles/kubernetes/master/tasks/start.yml b/roles/kubernetes/master/tasks/start.yml deleted file mode 100644 index 9cd247c42..000000000 --- a/roles/kubernetes/master/tasks/start.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- name: Pause - pause: seconds=10 - -- name: reload systemd - command: systemctl daemon-reload - when: ansible_service_mgr == "systemd" and restart_apimaster is defined and restart_apimaster == True - -- name: reload kube-apiserver - service: - name: kube-apiserver - state: restarted - enabled: yes - when: ( restart_apimaster is defined and restart_apimaster == True) or - secret_changed | default(false) - -- name: Enable apiserver - service: - name: kube-apiserver - enabled: yes - state: started - when: restart_apimaster is not defined or restart_apimaster == False diff --git a/roles/kubernetes/master/templates/deb-kube-apiserver.initd.j2 b/roles/kubernetes/master/templates/deb-kube-apiserver.initd.j2 deleted file mode 100644 index 576c70128..000000000 --- a/roles/kubernetes/master/templates/deb-kube-apiserver.initd.j2 +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash -# -### BEGIN INIT INFO -# Provides: kube-apiserver -# Required-Start: $local_fs $network $syslog -# Required-Stop: -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: The Kubernetes apiserver -# Description: -# The Kubernetes apiserver. -### END INIT INFO - - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/sbin:/usr/sbin:/bin:/usr/bin -DESC="The Kubernetes apiserver" -NAME=kube-apiserver -DAEMON={{ bin_dir }}/kube-apiserver -DAEMON_LOG_FILE=/var/log/$NAME.log -PIDFILE=/var/run/$NAME.pid -SCRIPTNAME=/etc/init.d/$NAME -DAEMON_USER=root - -# Exit if the package is not installed -[ -x "$DAEMON" ] || exit 0 - -# Read configuration variable file if it is present -[ -r /etc/kubernetes/$NAME.env ] && . /etc/kubernetes/$NAME.env - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.2-14) to ensure that this file is present -# and status_of_proc is working. -. /lib/lsb/init-functions - -# -# Function that starts the daemon/service -# -do_start() -{ - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - start-stop-daemon --start --quiet --background --no-close \ - --make-pidfile --pidfile $PIDFILE \ - --exec $DAEMON -c $DAEMON_USER --test > /dev/null \ - || return 1 - start-stop-daemon --start --quiet --background --no-close \ - --make-pidfile --pidfile $PIDFILE \ - --exec $DAEMON -c $DAEMON_USER -- \ - $DAEMON_ARGS >> $DAEMON_LOG_FILE 2>&1 \ - || return 2 -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME - RETVAL="$?" - [ "$RETVAL" = 2 ] && return 2 - # Many daemons don't delete their pidfiles when they exit. - rm -f $PIDFILE - return "$RETVAL" -} - - -case "$1" in - start) - log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) log_end_msg 0 || exit 0 ;; - 2) log_end_msg 1 || exit 1 ;; - esac - ;; - stop) - log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) log_end_msg 0 ;; - 2) exit 1 ;; - esac - ;; - status) - status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $? - ;; - - restart|force-reload) - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 ;; # Old process is still running - *) log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 - ;; - esac - ;; - *) - echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 - exit 3 - ;; -esac diff --git a/roles/kubernetes/master/templates/kube-apiserver.j2 b/roles/kubernetes/master/templates/kube-apiserver.j2 deleted file mode 100644 index 0e3a2710d..000000000 --- a/roles/kubernetes/master/templates/kube-apiserver.j2 +++ /dev/null @@ -1,58 +0,0 @@ -### -# kubernetes system config -# -# The following values are used to configure the kube-apiserver - -{% if ansible_service_mgr in ["sysvinit","upstart"] %} -# Logging directory -KUBE_LOGGING="--log-dir={{ kube_log_dir }} --logtostderr=true" -{% else %} -# logging to stderr means we get it in the systemd journal -KUBE_LOGGING="--logtostderr=true" -{% endif %} - -# Apiserver Log level, 0 is debug -KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}" - -# Should this cluster be allowed to run privileged docker containers -KUBE_ALLOW_PRIV="--allow_privileged=true" - -# The port on the local server to listen on. -KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{ kube_apiserver_port }}" - -# Insecure API address (default is localhost) -KUBE_API_INSECURE_BIND="--insecure-bind-address={{ kube_apiserver_insecure_bind_address | default('127.0.0.1') }}" - -# Address range to use for services -KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}" - -# Location of the etcd cluster -KUBE_ETCD_SERVERS="--etcd_servers={{ etcd_access_endpoint }}" - -# Bind address for secure endpoint -KUBE_API_ADDRESS="--bind-address={{ ip | default(ansible_default_ipv4.address) }}" - -# default admission control policies -KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota" - -# RUNTIME API CONFIGURATION (e.g. enable extensions) -KUBE_RUNTIME_CONFIG="{% if kube_api_runtime_config is defined %}{% for conf in kube_api_runtime_config %}--runtime-config={{ conf }} {% endfor %}{% endif %}" - -# TLS CONFIGURATION -KUBE_TLS_CONFIG="--tls_cert_file={{ kube_cert_dir }}/apiserver.pem --tls_private_key_file={{ kube_cert_dir }}/apiserver-key.pem --client_ca_file={{ kube_cert_dir }}/ca.pem" - -# Add you own! -KUBE_API_ARGS="--token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/apiserver-key.pem --advertise-address={{ ip | default(ansible_default_ipv4.address) }}" - -{% if cloud_provider is defined and cloud_provider == "openstack" %} -KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config" -{% else %} -{# TODO: gce and aws don't need the cloud provider to be set? #} -KUBELET_CLOUDPROVIDER="" -{% endif %} - -{% if ansible_service_mgr in ["sysvinit","upstart"] %} -DAEMON_ARGS="$KUBE_LOGGING $KUBE_LOG_LEVEL $KUBE_ALLOW_PRIV $KUBE_API_PORT $KUBE_API_INSECURE_BIND \ -$KUBE_SERVICE_ADDRESSES $KUBE_ETCD_SERVERS $KUBE_ADMISSION_CONTROL $KUBE_RUNTIME_CONFIG \ -$KUBE_TLS_CONFIG $KUBE_API_ARGS $KUBELET_CLOUDPROVIDER" -{% endif %} diff --git a/roles/kubernetes/master/templates/kube-apiserver.service.j2 b/roles/kubernetes/master/templates/kube-apiserver.service.j2 deleted file mode 100644 index 785cfd097..000000000 --- a/roles/kubernetes/master/templates/kube-apiserver.service.j2 +++ /dev/null @@ -1,30 +0,0 @@ -[Unit] -Description=Kubernetes API Server -Documentation=https://github.com/GoogleCloudPlatform/kubernetes -Wants=etcd-proxy.service -After=etcd-proxy.service - -[Service] -EnvironmentFile=/etc/kubernetes/kube-apiserver.env -User=kube -ExecStart={{ bin_dir }}/kube-apiserver \ - $KUBE_LOGTOSTDERR \ - $KUBE_LOG_LEVEL \ - $KUBE_ETCD_SERVERS \ - $KUBE_API_ADDRESS \ - $KUBE_API_PORT \ - $KUBE_API_INSECURE_BIND \ - $KUBELET_PORT \ - $KUBE_ALLOW_PRIV \ - $KUBE_SERVICE_ADDRESSES \ - $KUBE_ADMISSION_CONTROL \ - $KUBE_RUNTIME_CONFIG \ - $KUBE_TLS_CONFIG \ - $KUBE_API_ARGS \ - $KUBELET_CLOUDPROVIDER -Restart=on-failure -Type=notify -LimitNOFILE=65536 - -[Install] -WantedBy=multi-user.target diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 853a76cae..eaecb011b 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -2,6 +2,7 @@ apiVersion: v1 kind: Pod metadata: name: kube-apiserver + namespace: kube-system spec: hostNetwork: true containers: @@ -12,12 +13,14 @@ spec: - apiserver - --advertise-address={{ ip | default(ansible_default_ipv4.address) }} - --etcd-servers={{ etcd_access_endpoint }} + - --insecure-bind-address={{ kube_apiserver_insecure_bind_address | default('127.0.0.1') }} - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota - --service-cluster-ip-range={{ kube_service_addresses }} - --client-ca-file={{ kube_cert_dir }}/ca.pem - --basic-auth-file={{ kube_users_dir }}/known_users.csv - --tls-cert-file={{ kube_cert_dir }}/apiserver.pem - --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem + - --token-auth-file={{ kube_token_dir }}/known_tokens.csv - --service-account-key-file={{ kube_cert_dir }}/apiserver-key.pem - --secure-port={{ kube_apiserver_port }} - --insecure-port={{ kube_apiserver_insecure_port }} @@ -26,16 +29,13 @@ spec: - --runtime-config={{ conf }} {% endfor %} {% endif %} - - --token-auth-file={{ kube_token_dir }}/known_tokens.csv - --v={{ kube_log_level | default('2') }} - --allow-privileged=true - ports: - - containerPort: {{ kube_apiserver_port }} - hostPort: {{ kube_apiserver_port }} - name: https - - containerPort: {{ kube_apiserver_insecure_port }} - hostPort: {{ kube_apiserver_insecure_port }} - name: local +{% if cloud_provider is defined and cloud_provider == "openstack" %} + - --cloud-provider={{ cloud_provider }} + - --cloud-config={{ kube_config_dir }}/cloud_config +{% endif %} + - 2>&1 >> {{ kube_log_dir }}/kube-apiserver.log volumeMounts: - mountPath: {{ kube_config_dir }} name: kubernetes-config @@ -43,6 +43,8 @@ spec: - mountPath: /etc/ssl/certs name: ssl-certs-host readOnly: true + - mountPath: /var/log/ + name: logfile volumes: - hostPath: path: {{ kube_config_dir }} @@ -50,3 +52,6 @@ spec: - hostPath: path: /etc/ssl/certs/ name: ssl-certs-host + - hostPath: + path: /var/log/ + name: logfile diff --git a/roles/kubernetes/node/meta/main.yml b/roles/kubernetes/node/meta/main.yml index a277c7d8a..c65c68393 100644 --- a/roles/kubernetes/node/meta/main.yml +++ b/roles/kubernetes/node/meta/main.yml @@ -1,5 +1,5 @@ --- dependencies: - - role: download - file: "{{ downloads.kubernetes_kubelet }}" + - role: download #For kube_version + file: "{{ downloads.nothing }}" - role: kubernetes/secrets diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index 4fabf1c88..23e482bc7 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -14,7 +14,7 @@ when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "RedHat" notify: restart kubelet -- name: install | Install kubelet binary - command: rsync -piu "{{ local_release_dir }}/kubernetes/bin/kubelet" "{{ bin_dir }}/kubelet" - register: kubelet_copy +- name: install | Install kubelet launch script + template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes + register: kubelet_launcher changed_when: false diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 803c9251b..0aa1957ea 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -26,7 +26,7 @@ - name: Restart kubelet if binary changed command: /bin/true notify: restart kubelet - when: kubelet_copy.stdout_lines + when: kubelet_launcher.changed # reload-systemd - meta: flush_handlers diff --git a/roles/kubernetes/node/templates/kubelet-container.j2 b/roles/kubernetes/node/templates/kubelet-container.j2 new file mode 100644 index 000000000..2fcc7307f --- /dev/null +++ b/roles/kubernetes/node/templates/kubelet-container.j2 @@ -0,0 +1,15 @@ +#!/bin/bash +/usr/bin/docker run --privileged --rm \ +--net=host --pid=host --name=kubelet \ +-v /etc/cni:/etc/cni:ro \ +-v /opt/cni:/opt/cni:ro \ +-v /etc/kubernetes:/etc/kubernetes \ +-v /sys:/sys \ +-v /dev:/dev \ +-v /var/lib/docker:/var/lib/docker \ +-v /var/run:/var/run \ +-v /var/lib/kubelet:/var/lib/kubelet \ +{{ hyperkube_image_repo }}:{{ hyperkube_image_tag}} \ +nsenter --target=1 --mount --wd=. -- \ +./hyperkube kubelet \ +$@ diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index 20e521f73..7eee44993 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -6,7 +6,6 @@ KUBE_LOGGING="--log-dir={{ kube_log_dir }} --logtostderr=true" KUBE_LOGGING="--logtostderr=true" {% endif %} KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}" -KUBE_ALLOW_PRIV="--allow_privileged=true" {% if inventory_hostname in groups['kube-node'] %} KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}" {% endif %} @@ -15,7 +14,7 @@ KUBELET_ADDRESS="--address={{ ip | default("0.0.0.0") }}" # The port for the info server to serve on # KUBELET_PORT="--port=10250" # You may leave this blank to use the actual hostname -KUBELET_HOSTNAME="--hostname_override={{ inventory_hostname }}" +KUBELET_HOSTNAME="--hostname-override={{ inventory_hostname }}" {% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %} KUBELET_REGISTER_NODE="--register-node=false" {% endif %} @@ -26,12 +25,12 @@ KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} - KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}" {% endif %} {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave"] %} -KUBELET_NETWORK_PLUGIN="--network_plugin=cni --network-plugin-dir=/etc/cni/net.d" +KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d" {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %} DOCKER_SOCKET="--docker-endpoint=unix:/var/run/weave/weave.sock" {% endif %} # Should this cluster be allowed to run privileged docker containers -KUBE_ALLOW_PRIV="--allow_privileged=true" +KUBE_ALLOW_PRIV="--allow-privileged=true" {% if cloud_provider is defined and cloud_provider == "openstack" %} KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config" {% else %} diff --git a/roles/kubernetes/node/templates/kubelet.service.j2 b/roles/kubernetes/node/templates/kubelet.service.j2 index 9fa47bf13..acad42e1f 100644 --- a/roles/kubernetes/node/templates/kubelet.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.service.j2 @@ -22,7 +22,10 @@ ExecStart={{ bin_dir }}/kubelet \ $KUBELET_REGISTER_NODE \ $KUBELET_NETWORK_PLUGIN \ $KUBELET_CLOUDPROVIDER -Restart=on-failure +ExecStartPre=-/usr/bin/docker rm -f kubelet +ExecReload=/usr/bin/docker restart kubelet +Restart=always +RestartSec=10s [Install] WantedBy=multi-user.target diff --git a/roles/network_plugin/calico/templates/calicoctl-container.j2 b/roles/network_plugin/calico/templates/calicoctl-container.j2 index a6bf88896..466f1df93 100644 --- a/roles/network_plugin/calico/templates/calicoctl-container.j2 +++ b/roles/network_plugin/calico/templates/calicoctl-container.j2 @@ -1,6 +1,6 @@ #!/bin/bash /usr/bin/docker run --privileged --rm \ ---net=host -e ETCD_AUTHORITY={{ etcd_authority }} \ +--net=host --pid=host -e ETCD_AUTHORITY={{ etcd_authority }} \ -v /usr/bin/docker:/usr/bin/docker \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /var/run/calico:/var/run/calico \ diff --git a/roles/uploads/defaults/main.yml b/roles/uploads/defaults/main.yml index 2eb76bb16..db6ba5485 100644 --- a/roles/uploads/defaults/main.yml +++ b/roles/uploads/defaults/main.yml @@ -60,15 +60,6 @@ downloads: owner: "etcd" mode: "0755" - - name: kubernetes-kubelet - version: "{{kube_version}}" - dest: kubernetes/bin/kubelet - sha256: "{{vars['kube_checksum'][kube_version]['kubelet']}}" - source_url: "{{ kube_download_url }}/kubelet" - url: "{{ kube_download_url }}/kubelet" - owner: "kube" - mode: "0755" - - name: kubernetes-kubectl dest: kubernetes/bin/kubectl version: "{{kube_version}}" @@ -77,12 +68,3 @@ downloads: url: "{{ kube_download_url }}/kubectl" owner: "kube" mode: "0755" - - - name: kubernetes-apiserver - dest: kubernetes/bin/kube-apiserver - version: "{{kube_version}}" - sha256: "{{vars['kube_checksum'][kube_version]['kube_apiserver']}}" - source_url: "{{ kube_download_url }}/kube-apiserver" - url: "{{ kube_download_url }}/kube-apiserver" - owner: "kube" - mode: "0755" diff --git a/roles/uploads/vars/kube_versions.yml b/roles/uploads/vars/kube_versions.yml index 5b5f64a42..627cdc60b 100644 --- a/roles/uploads/vars/kube_versions.yml +++ b/roles/uploads/vars/kube_versions.yml @@ -2,21 +2,16 @@ kube_checksum: v1.2.2: kube_apiserver: eb1bfd8b877052cbd1991b8c429a1d06661f4cb019905e20e128174f724e16de kubectl: 473e6924569fba30d4a50cecdc2cae5f31d97d1f662463e85b74a472105dcff4 - kubelet: f16827dc7e7c82f0e215f0fc73eb01e2dfe91a2ec83f9cbcaf8d37c91b64fd3b v1.2.3: kube_apiserver_checksum: ebaeeeb72cb29b358337b330617a96355ff2d08a5a523fc1a81beba36cc9d6f9 kubectl_checksum: 394853edd409a721bcafe4f1360009ef9f845050719fe7d6fc7176f45cc92a8c - kubelet_checksum: 633bb41c51c5c0df0645dd60ba82b12eba39d009eb87bae9227de7d9a89c0797 v1.2.4: kube_apiserver: 6ac99b36b02968459e026fcfc234207c66064b5e11816b69dd8fc234b2ffec1e kubectl: dac61fbd506f7a17540feca691cd8a9d9d628d59661eebce788a50511f578897 - kubelet: 4adaf40592248eef6fd4fa126464915ea41e624a70dc77178089760ed235e341 v1.2.5: kube_apiserver: fbe8296ad4b194c06f6802a126d35cd2887dc1aded308d4da2b580f270412b33 kubectl: 5526a496a84701015485e32c86486e2f23599f7a865164f546e619c6a62f7f19 - kubelet: cd15b929f0190876216f397c2c6e7aa8c08d3b047fd90b4980cd68c8f4896211 v1.3.0: kube_apiserver: 431cd312984a29f45590138e990d5c4d537b069b71f2587a72414fabc4fcffdd kubectl: f40b2d0ff33984e663a0dea4916f1cb9041abecc09b11f9372cdb8049ded95dc - kubelet: bd5f10ccb95fe6e95ddf7ad8a119195c27cb2bce4be6f80c1810ff1a2111496d kube_version: v1.3.0