Move kube-apiserver to static pod

Added pre-upgrade.yml for kubernetes/master role to
purge previously configured kube-apiserver service.

roles/download/defaults/main.yml

@@ -14,7 +14,6 @@ weave_version: v1.5.0
 
 # Download URL's
 kubelet_download_url: "https://storage.googleapis.com/kargo/{{kube_version}}_kubernetes-kubelet"
-apiserver_download_url: "https://storage.googleapis.com/kargo/{{kube_version}}_kubernetes-apiserver"
 kubectl_download_url: "https://storage.googleapis.com/kargo/{{kube_version}}_kubernetes-kubectl"
 
 etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd"
@@ -88,14 +87,6 @@ downloads:
     url: "{{ kubectl_download_url }}"
     owner: "kube"
     mode: "0755"
-  kubernetes_apiserver:
-    dest: kubernetes/bin/kube-apiserver
-    version: "{{kube_version}}"
-    sha256: "{{vars['kube_checksum'][kube_version]['kube_apiserver']}}"
-    source_url: "{{ apiserver_download_url }}"
-    url: "{{ apiserver_download_url }}"
-    owner: "kube"
-    mode: "0755"
 
 download:
   enabled: "{{ file.enabled|default('true') }}"

roles/download/vars/kube_versions.yml

@@ -1,22 +1,17 @@
 kube_checksum:
     v1.2.2:
-        kube_apiserver: eb1bfd8b877052cbd1991b8c429a1d06661f4cb019905e20e128174f724e16de
         kubectl: 473e6924569fba30d4a50cecdc2cae5f31d97d1f662463e85b74a472105dcff4
         kubelet: f16827dc7e7c82f0e215f0fc73eb01e2dfe91a2ec83f9cbcaf8d37c91b64fd3b
     v1.2.3:
-        kube_apiserver_checksum: ebaeeeb72cb29b358337b330617a96355ff2d08a5a523fc1a81beba36cc9d6f9
         kubectl_checksum: 394853edd409a721bcafe4f1360009ef9f845050719fe7d6fc7176f45cc92a8c
         kubelet_checksum: 633bb41c51c5c0df0645dd60ba82b12eba39d009eb87bae9227de7d9a89c0797
     v1.2.4:
-        kube_apiserver: 6ac99b36b02968459e026fcfc234207c66064b5e11816b69dd8fc234b2ffec1e
         kubectl: dac61fbd506f7a17540feca691cd8a9d9d628d59661eebce788a50511f578897
         kubelet: 4adaf40592248eef6fd4fa126464915ea41e624a70dc77178089760ed235e341
     v1.2.5:
-        kube_apiserver: fbe8296ad4b194c06f6802a126d35cd2887dc1aded308d4da2b580f270412b33
         kubectl: 5526a496a84701015485e32c86486e2f23599f7a865164f546e619c6a62f7f19
         kubelet: cd15b929f0190876216f397c2c6e7aa8c08d3b047fd90b4980cd68c8f4896211
     v1.3.0:
-        kube_apiserver: 431cd312984a29f45590138e990d5c4d537b069b71f2587a72414fabc4fcffdd
         kubectl: f40b2d0ff33984e663a0dea4916f1cb9041abecc09b11f9372cdb8049ded95dc
         kubelet: bd5f10ccb95fe6e95ddf7ad8a119195c27cb2bce4be6f80c1810ff1a2111496d
 kube_version: v1.3.0

roles/kubernetes/master/handlers/main.yml

@@ -1,4 +0,0 @@
----
-- name: restart kube-apiserver
-  set_fact:
-    restart_apimaster: True

roles/kubernetes/master/meta/main.yml

@@ -2,7 +2,5 @@
 dependencies:
   - role: download
     file: "{{ downloads.kubernetes_kubectl }}"
-  - role: download
-    file: "{{ downloads.kubernetes_apiserver }}"
   - { role: etcd }
   - { role: kubernetes/node }

roles/kubernetes/master/tasks/main.yml

@@ -1,59 +1,34 @@
 ---
+- include: pre-upgrade.yml
+
 - name: Copy kubectl bash completion
   copy:
     src: kubectl_bash_completion.sh
     dest: /etc/bash_completion.d/kubectl.sh
   when: ansible_os_family in ["Debian","RedHat"]
 
-- name: Copy kube-apiserver binary
-  command: rsync -piu "{{ local_release_dir }}/kubernetes/bin/kube-apiserver" "{{ bin_dir }}/kube-apiserver"
-  register: kube_apiserver_copy
-  changed_when: false
-
 - name: Copy kubectl binary
   command: rsync -piu "{{ local_release_dir }}/kubernetes/bin/kubectl" "{{ bin_dir }}/kubectl"
   changed_when: false
 
-- name: install | Write kube-apiserver systemd init file
-  template:
-    src: "kube-apiserver.service.j2"
-    dest: "/etc/systemd/system/kube-apiserver.service"
-    backup: yes
-  when: ansible_service_mgr == "systemd"
-  notify: restart kube-apiserver
-
-- name: install | Write kube-apiserver initd script
-  template:
-    src: "deb-kube-apiserver.initd.j2"
-    dest: "/etc/init.d/kube-apiserver"
-    owner: root
-    mode: 0755
-    backup: yes
-  when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "Debian"
-
-- name: Write kube-apiserver config file
-  template:
-    src: "kube-apiserver.j2"
-    dest: "{{ kube_config_dir }}/kube-apiserver.env"
-    backup: yes
-  notify: restart kube-apiserver
-
-- name: Allow apiserver to bind on both secure and insecure ports
-  shell: setcap cap_net_bind_service+ep {{ bin_dir }}/kube-apiserver
-  changed_when: false
-
 - meta: flush_handlers
 
-- include: start.yml
-  with_items: "{{ groups['kube-master'] }}"
-  when: "{{ hostvars[item].inventory_hostname == inventory_hostname }}"
-
 # Create kube-system namespace
 - name: copy 'kube-system' namespace manifest
   copy: src=namespace.yml dest=/etc/kubernetes/kube-system-ns.yml
   run_once: yes
   when: inventory_hostname == groups['kube-master'][0]
 
+- name: Write kube-apiserver manifest
+  template:
+    src: manifests/kube-apiserver.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
+
+- name: wait for the apiserver to be running
+  wait_for:
+    port: "{{kube_apiserver_insecure_port}}"
+    timeout: 60
+
 - name: Check if kube-system exists
   command: "{{ bin_dir }}/kubectl get ns kube-system"
   register: 'kubesystem'
@@ -61,17 +36,12 @@
   failed_when: False
   run_once: yes
 
-- name: wait for the apiserver to be running
-  wait_for:
-    port: "{{kube_apiserver_insecure_port}}"
-    timeout: 60
-
 - name: Create 'kube-system' namespace
   command: "{{ bin_dir }}/kubectl create -f /etc/kubernetes/kube-system-ns.yml"
   changed_when: False
   when: kubesystem|failed and inventory_hostname == groups['kube-master'][0]
 
-# Write manifests
+# Write other manifests
 - name: Write kube-controller-manager manifest
   template:
     src: manifests/kube-controller-manager.manifest.j2

roles/kubernetes/master/tasks/pre-upgrade.yml

@@ -0,0 +1,27 @@
+---
+- name: "Pre-upgrade | check for kube-apiserver unit file"
+  stat:
+    path: /etc/systemd/system/kube-apiserver.service
+  register: kube_apiserver_service_file
+  when: ansible_service_mgr == "systemd"
+
+- name: "Pre-upgrade | check for kube-apiserver init script"
+  stat:
+    path: /etc/init.d/kube-apiserver
+  register: kube_apiserver_init_script
+  when: ansible_service_mgr in ["sysvinit","upstart"]
+
+- name: "Pre-upgrade | stop kube-apiserver if service defined"
+  service:
+    name: kube-apiserver
+    state: stopped
+  when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
+
+- name: "Pre-upgrade | remove kube-apiserver service definition"
+  file:
+    path: "{{ item }}"
+    state: absent
+  when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
+  with_items:
+    - /etc/systemd/system/kube-apiserver.service
+    - /etc/init.d/kube-apiserver

roles/kubernetes/master/tasks/start.yml

@@ -1,22 +0,0 @@
----
-- name: Pause
-  pause: seconds=10
-
-- name: reload systemd
-  command: systemctl daemon-reload
-  when: ansible_service_mgr == "systemd" and restart_apimaster is defined and restart_apimaster == True
-
-- name: reload kube-apiserver
-  service:
-    name: kube-apiserver
-    state: restarted
-    enabled: yes
-  when: ( restart_apimaster is defined and restart_apimaster == True) or
-        secret_changed | default(false)
-
-- name: Enable apiserver
-  service:
-    name: kube-apiserver
-    enabled: yes
-    state: started
-  when: restart_apimaster is not defined or restart_apimaster == False

roles/kubernetes/master/templates/deb-kube-apiserver.initd.j2

@@ -1,118 +0,0 @@
-#!/bin/bash
-#
-### BEGIN INIT INFO
-# Provides:   kube-apiserver 
-# Required-Start:    $local_fs $network $syslog
-# Required-Stop:
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: The Kubernetes apiserver
-# Description:
-#   The Kubernetes apiserver.
-### END INIT INFO
-
-
-# PATH should only include /usr/* if it runs after the mountnfs.sh script
-PATH=/sbin:/usr/sbin:/bin:/usr/bin
-DESC="The Kubernetes apiserver"
-NAME=kube-apiserver
-DAEMON={{ bin_dir }}/kube-apiserver
-DAEMON_LOG_FILE=/var/log/$NAME.log
-PIDFILE=/var/run/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-DAEMON_USER=root
-
-# Exit if the package is not installed
-[ -x "$DAEMON" ] || exit 0
-
-# Read configuration variable file if it is present
-[ -r /etc/kubernetes/$NAME.env ] && . /etc/kubernetes/$NAME.env
-
-# Define LSB log_* functions.
-# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
-# and status_of_proc is working.
-. /lib/lsb/init-functions
-
-#
-# Function that starts the daemon/service
-#
-do_start()
-{
-        # Return
-        #   0 if daemon has been started
-        #   1 if daemon was already running
-        #   2 if daemon could not be started
-        start-stop-daemon --start --quiet --background --no-close \
-                --make-pidfile --pidfile $PIDFILE \
-                --exec $DAEMON -c $DAEMON_USER --test > /dev/null \
-                || return 1
-        start-stop-daemon --start --quiet --background --no-close \
-                --make-pidfile --pidfile $PIDFILE \
-                --exec $DAEMON -c $DAEMON_USER -- \
-                $DAEMON_ARGS >> $DAEMON_LOG_FILE 2>&1 \
-                || return 2
-}
-
-#
-# Function that stops the daemon/service
-#
-do_stop()
-{
-        # Return
-        #   0 if daemon has been stopped
-        #   1 if daemon was already stopped
-        #   2 if daemon could not be stopped
-        #   other if a failure occurred
-        start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME
-        RETVAL="$?"
-        [ "$RETVAL" = 2 ] && return 2
-        # Many daemons don't delete their pidfiles when they exit.
-        rm -f $PIDFILE
-        return "$RETVAL"
-}
-
-
-case "$1" in
-  start)
-        log_daemon_msg "Starting $DESC" "$NAME"
-        do_start
-        case "$?" in
-                0|1) log_end_msg 0 || exit 0 ;;
-                2) log_end_msg 1 || exit 1 ;;
-        esac
-        ;;
-  stop)
-        log_daemon_msg "Stopping $DESC" "$NAME"
-        do_stop
-        case "$?" in
-                0|1) log_end_msg 0 ;;
-                2) exit 1 ;;
-        esac
-        ;;
-  status)
-        status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $?
-        ;;
-
-  restart|force-reload)
-        log_daemon_msg "Restarting $DESC" "$NAME"
-        do_stop
-        case "$?" in
-          0|1)
-                do_start
-                case "$?" in
-                        0) log_end_msg 0 ;;
-                        1) log_end_msg 1 ;; # Old process is still running
-                        *) log_end_msg 1 ;; # Failed to start
-                esac
-                ;;
-          *)
-                # Failed to stop
-                log_end_msg 1
-                ;;
-        esac
-        ;;
-  *)
-        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
-        exit 3
-        ;;
-esac

roles/kubernetes/master/templates/kube-apiserver.j2

@@ -1,58 +0,0 @@
-###
-# kubernetes system config
-#
-# The following values are used to configure the kube-apiserver
-
-{% if ansible_service_mgr in ["sysvinit","upstart"] %}
-# Logging directory
-KUBE_LOGGING="--log-dir={{ kube_log_dir }} --logtostderr=true"
-{% else %}
-# logging to stderr means we get it in the systemd journal
-KUBE_LOGGING="--logtostderr=true"
-{% endif %}
-
-# Apiserver Log level, 0 is debug
-KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}"
-
-# Should this cluster be allowed to run privileged docker containers
-KUBE_ALLOW_PRIV="--allow_privileged=true"
-
-# The port on the local server to listen on.
-KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{ kube_apiserver_port }}"
-
-# Insecure API address (default is localhost)
-KUBE_API_INSECURE_BIND="--insecure-bind-address={{ kube_apiserver_insecure_bind_address | default('127.0.0.1') }}"
-
-# Address range to use for services
-KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}"
-
-# Location of the etcd cluster
-KUBE_ETCD_SERVERS="--etcd_servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %}"
-
-# Bind address for secure endpoint
-KUBE_API_ADDRESS="--bind-address={{ ip | default(ansible_default_ipv4.address) }}"
-
-# default admission control policies
-KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
-
-# RUNTIME API CONFIGURATION (e.g. enable extensions)
-KUBE_RUNTIME_CONFIG="{% if kube_api_runtime_config is defined %}{% for conf in kube_api_runtime_config %}--runtime-config={{ conf }} {% endfor %}{% endif %}"
-
-# TLS CONFIGURATION
-KUBE_TLS_CONFIG="--tls_cert_file={{ kube_cert_dir }}/apiserver.pem --tls_private_key_file={{ kube_cert_dir }}/apiserver-key.pem --client_ca_file={{ kube_cert_dir }}/ca.pem"
-
-# Add you own!
-KUBE_API_ARGS="--token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/apiserver-key.pem --advertise-address={{ ip | default(ansible_default_ipv4.address) }}"
-
-{% if cloud_provider is defined and cloud_provider == "openstack" %}
-KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"
-{% else %}
-{# TODO: gce and aws don't need the cloud provider to be set? #}
-KUBELET_CLOUDPROVIDER=""
-{% endif %}
-
-{% if ansible_service_mgr in ["sysvinit","upstart"] %}
-DAEMON_ARGS="$KUBE_LOGGING $KUBE_LOG_LEVEL $KUBE_ALLOW_PRIV $KUBE_API_PORT $KUBE_API_INSECURE_BIND \
-$KUBE_SERVICE_ADDRESSES $KUBE_ETCD_SERVERS $KUBE_ADMISSION_CONTROL $KUBE_RUNTIME_CONFIG \
-$KUBE_TLS_CONFIG $KUBE_API_ARGS $KUBELET_CLOUDPROVIDER"
-{% endif %}

roles/kubernetes/master/templates/kube-apiserver.service.j2

@@ -1,30 +0,0 @@
-[Unit]
-Description=Kubernetes API Server
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-Requires=etcd.service
-After=etcd.service
-
-[Service]
-EnvironmentFile=/etc/kubernetes/kube-apiserver.env
-User=kube
-ExecStart={{ bin_dir }}/kube-apiserver \
-        $KUBE_LOGTOSTDERR \
-        $KUBE_LOG_LEVEL \
-        $KUBE_ETCD_SERVERS \
-        $KUBE_API_ADDRESS \
-        $KUBE_API_PORT \
-        $KUBE_API_INSECURE_BIND \
-        $KUBELET_PORT \
-        $KUBE_ALLOW_PRIV \
-        $KUBE_SERVICE_ADDRESSES \
-        $KUBE_ADMISSION_CONTROL \
-        $KUBE_RUNTIME_CONFIG \
-        $KUBE_TLS_CONFIG \
-        $KUBE_API_ARGS \
-        $KUBELET_CLOUDPROVIDER
-Restart=on-failure
-Type=notify
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -11,13 +11,16 @@ spec:
     - /hyperkube
     - apiserver
     - --advertise-address={{ ip | default(ansible_default_ipv4.address) }}
-    - --etcd-servers={% for srv in groups['etcd'] %}http://{{ hostvars[srv]['access_ip'] | default(hostvars[srv]['ip']|default(hostvars[srv]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %}
+    - --insecure-bind-address={{ kube_apiserver_insecure_bind_address | default('127.0.0.1') }}
+    - --etcd-servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %}
+
     - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
     - --service-cluster-ip-range={{ kube_service_addresses }}
     - --client-ca-file={{ kube_cert_dir }}/ca.pem
     - --basic-auth-file={{ kube_users_dir }}/known_users.csv
     - --tls-cert-file={{ kube_cert_dir }}/apiserver.pem
     - --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
+    - --token-auth-file={{ kube_token_dir }}/known_tokens.csv
     - --service-account-key-file={{ kube_cert_dir }}/apiserver-key.pem
     - --secure-port={{ kube_apiserver_port }}
     - --insecure-port={{ kube_apiserver_insecure_port }}
@@ -26,9 +29,13 @@ spec:
     - --runtime-config={{ conf }}
 {%   endfor %}
 {% endif %}
-    - --token-auth-file={{ kube_token_dir }}/known_tokens.csv
     - --v={{ kube_log_level | default('2') }}
     - --allow-privileged=true
+{% if cloud_provider is defined and cloud_provider == "openstack" %}
+    - --cloud-provider={{ cloud_provider }}
+    - --cloud-config={{ kube_config_dir }}/cloud_config
+    - 2>&1 >> {{ kube_log_dir }}/kube-apiserver.log
+{% endif %}
     ports:
     - containerPort: {{ kube_apiserver_port }}
       hostPort: {{ kube_apiserver_port }}
@@ -43,6 +50,8 @@ spec:
     - mountPath: /etc/ssl/certs
       name: ssl-certs-host
       readOnly: true
+    - mountPath: /var/log/
+      name: logfile
   volumes:
   - hostPath:
       path: {{ kube_config_dir }}
@@ -50,3 +59,6 @@ spec:
   - hostPath:
       path: /etc/ssl/certs/
     name: ssl-certs-host
+  - hostPath:
+      path: /var/log/
+    name: logfile

roles/uploads/defaults/main.yml

@@ -88,12 +88,3 @@ downloads:
     url: "{{ kube_download_url }}/kubectl"
     owner: "kube"
     mode: "0755"
-
-  - name: kubernetes-apiserver
-    dest: kubernetes/bin/kube-apiserver
-    version: "{{kube_version}}"
-    sha256: "{{vars['kube_checksum'][kube_version]['kube_apiserver']}}"
-    source_url: "{{ kube_download_url }}/kube-apiserver"
-    url: "{{ kube_download_url }}/kube-apiserver"
-    owner: "kube"
-    mode: "0755"

roles/uploads/vars/kube_versions.yml

@@ -1,22 +1,17 @@
 kube_checksum:
     v1.2.2:
-        kube_apiserver: eb1bfd8b877052cbd1991b8c429a1d06661f4cb019905e20e128174f724e16de
         kubectl: 473e6924569fba30d4a50cecdc2cae5f31d97d1f662463e85b74a472105dcff4
         kubelet: f16827dc7e7c82f0e215f0fc73eb01e2dfe91a2ec83f9cbcaf8d37c91b64fd3b
     v1.2.3:
-        kube_apiserver_checksum: ebaeeeb72cb29b358337b330617a96355ff2d08a5a523fc1a81beba36cc9d6f9
         kubectl_checksum: 394853edd409a721bcafe4f1360009ef9f845050719fe7d6fc7176f45cc92a8c
         kubelet_checksum: 633bb41c51c5c0df0645dd60ba82b12eba39d009eb87bae9227de7d9a89c0797
     v1.2.4:
-        kube_apiserver: 6ac99b36b02968459e026fcfc234207c66064b5e11816b69dd8fc234b2ffec1e
         kubectl: dac61fbd506f7a17540feca691cd8a9d9d628d59661eebce788a50511f578897
         kubelet: 4adaf40592248eef6fd4fa126464915ea41e624a70dc77178089760ed235e341
     v1.2.5:
-        kube_apiserver: fbe8296ad4b194c06f6802a126d35cd2887dc1aded308d4da2b580f270412b33
         kubectl: 5526a496a84701015485e32c86486e2f23599f7a865164f546e619c6a62f7f19
         kubelet: cd15b929f0190876216f397c2c6e7aa8c08d3b047fd90b4980cd68c8f4896211
     v1.3.0:
-        kube_apiserver: 431cd312984a29f45590138e990d5c4d537b069b71f2587a72414fabc4fcffdd
         kubectl: f40b2d0ff33984e663a0dea4916f1cb9041abecc09b11f9372cdb8049ded95dc
         kubelet: bd5f10ccb95fe6e95ddf7ad8a119195c27cb2bce4be6f80c1810ff1a2111496d
 kube_version: v1.3.0