diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 6501a1e1b..58baccc9f 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -14,7 +14,6 @@ weave_version: v1.5.0 # Download URL's kubelet_download_url: "https://storage.googleapis.com/kargo/{{kube_version}}_kubernetes-kubelet" -apiserver_download_url: "https://storage.googleapis.com/kargo/{{kube_version}}_kubernetes-apiserver" kubectl_download_url: "https://storage.googleapis.com/kargo/{{kube_version}}_kubernetes-kubectl" etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd" @@ -88,14 +87,6 @@ downloads: url: "{{ kubectl_download_url }}" owner: "kube" mode: "0755" - kubernetes_apiserver: - dest: kubernetes/bin/kube-apiserver - version: "{{kube_version}}" - sha256: "{{vars['kube_checksum'][kube_version]['kube_apiserver']}}" - source_url: "{{ apiserver_download_url }}" - url: "{{ apiserver_download_url }}" - owner: "kube" - mode: "0755" download: enabled: "{{ file.enabled|default('true') }}" diff --git a/roles/download/vars/kube_versions.yml b/roles/download/vars/kube_versions.yml index 5b5f64a42..f454eb2dc 100644 --- a/roles/download/vars/kube_versions.yml +++ b/roles/download/vars/kube_versions.yml @@ -1,22 +1,17 @@ kube_checksum: v1.2.2: - kube_apiserver: eb1bfd8b877052cbd1991b8c429a1d06661f4cb019905e20e128174f724e16de kubectl: 473e6924569fba30d4a50cecdc2cae5f31d97d1f662463e85b74a472105dcff4 kubelet: f16827dc7e7c82f0e215f0fc73eb01e2dfe91a2ec83f9cbcaf8d37c91b64fd3b v1.2.3: - kube_apiserver_checksum: ebaeeeb72cb29b358337b330617a96355ff2d08a5a523fc1a81beba36cc9d6f9 kubectl_checksum: 394853edd409a721bcafe4f1360009ef9f845050719fe7d6fc7176f45cc92a8c kubelet_checksum: 633bb41c51c5c0df0645dd60ba82b12eba39d009eb87bae9227de7d9a89c0797 v1.2.4: - kube_apiserver: 6ac99b36b02968459e026fcfc234207c66064b5e11816b69dd8fc234b2ffec1e kubectl: dac61fbd506f7a17540feca691cd8a9d9d628d59661eebce788a50511f578897 kubelet: 4adaf40592248eef6fd4fa126464915ea41e624a70dc77178089760ed235e341 v1.2.5: - kube_apiserver: fbe8296ad4b194c06f6802a126d35cd2887dc1aded308d4da2b580f270412b33 kubectl: 5526a496a84701015485e32c86486e2f23599f7a865164f546e619c6a62f7f19 kubelet: cd15b929f0190876216f397c2c6e7aa8c08d3b047fd90b4980cd68c8f4896211 v1.3.0: - kube_apiserver: 431cd312984a29f45590138e990d5c4d537b069b71f2587a72414fabc4fcffdd kubectl: f40b2d0ff33984e663a0dea4916f1cb9041abecc09b11f9372cdb8049ded95dc kubelet: bd5f10ccb95fe6e95ddf7ad8a119195c27cb2bce4be6f80c1810ff1a2111496d kube_version: v1.3.0 diff --git a/roles/kubernetes/master/handlers/main.yml b/roles/kubernetes/master/handlers/main.yml deleted file mode 100644 index a4082887b..000000000 --- a/roles/kubernetes/master/handlers/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: restart kube-apiserver - set_fact: - restart_apimaster: True diff --git a/roles/kubernetes/master/meta/main.yml b/roles/kubernetes/master/meta/main.yml index 11f02f99d..bd1008ae6 100644 --- a/roles/kubernetes/master/meta/main.yml +++ b/roles/kubernetes/master/meta/main.yml @@ -2,7 +2,5 @@ dependencies: - role: download file: "{{ downloads.kubernetes_kubectl }}" - - role: download - file: "{{ downloads.kubernetes_apiserver }}" - { role: etcd } - { role: kubernetes/node } diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index deaf017f3..b5248d3e3 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -1,59 +1,34 @@ --- +- include: pre-upgrade.yml + - name: Copy kubectl bash completion copy: src: kubectl_bash_completion.sh dest: /etc/bash_completion.d/kubectl.sh when: ansible_os_family in ["Debian","RedHat"] -- name: Copy kube-apiserver binary - command: rsync -piu "{{ local_release_dir }}/kubernetes/bin/kube-apiserver" "{{ bin_dir }}/kube-apiserver" - register: kube_apiserver_copy - changed_when: false - - name: Copy kubectl binary command: rsync -piu "{{ local_release_dir }}/kubernetes/bin/kubectl" "{{ bin_dir }}/kubectl" changed_when: false -- name: install | Write kube-apiserver systemd init file - template: - src: "kube-apiserver.service.j2" - dest: "/etc/systemd/system/kube-apiserver.service" - backup: yes - when: ansible_service_mgr == "systemd" - notify: restart kube-apiserver - -- name: install | Write kube-apiserver initd script - template: - src: "deb-kube-apiserver.initd.j2" - dest: "/etc/init.d/kube-apiserver" - owner: root - mode: 0755 - backup: yes - when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "Debian" - -- name: Write kube-apiserver config file - template: - src: "kube-apiserver.j2" - dest: "{{ kube_config_dir }}/kube-apiserver.env" - backup: yes - notify: restart kube-apiserver - -- name: Allow apiserver to bind on both secure and insecure ports - shell: setcap cap_net_bind_service+ep {{ bin_dir }}/kube-apiserver - changed_when: false - - meta: flush_handlers -- include: start.yml - with_items: "{{ groups['kube-master'] }}" - when: "{{ hostvars[item].inventory_hostname == inventory_hostname }}" - # Create kube-system namespace - name: copy 'kube-system' namespace manifest copy: src=namespace.yml dest=/etc/kubernetes/kube-system-ns.yml run_once: yes when: inventory_hostname == groups['kube-master'][0] +- name: Write kube-apiserver manifest + template: + src: manifests/kube-apiserver.manifest.j2 + dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest" + +- name: wait for the apiserver to be running + wait_for: + port: "{{kube_apiserver_insecure_port}}" + timeout: 60 + - name: Check if kube-system exists command: "{{ bin_dir }}/kubectl get ns kube-system" register: 'kubesystem' @@ -61,17 +36,12 @@ failed_when: False run_once: yes -- name: wait for the apiserver to be running - wait_for: - port: "{{kube_apiserver_insecure_port}}" - timeout: 60 - - name: Create 'kube-system' namespace command: "{{ bin_dir }}/kubectl create -f /etc/kubernetes/kube-system-ns.yml" changed_when: False when: kubesystem|failed and inventory_hostname == groups['kube-master'][0] -# Write manifests +# Write other manifests - name: Write kube-controller-manager manifest template: src: manifests/kube-controller-manager.manifest.j2 diff --git a/roles/kubernetes/master/tasks/pre-upgrade.yml b/roles/kubernetes/master/tasks/pre-upgrade.yml new file mode 100644 index 000000000..f84090d13 --- /dev/null +++ b/roles/kubernetes/master/tasks/pre-upgrade.yml @@ -0,0 +1,27 @@ +--- +- name: "Pre-upgrade | check for kube-apiserver unit file" + stat: + path: /etc/systemd/system/kube-apiserver.service + register: kube_apiserver_service_file + when: ansible_service_mgr == "systemd" + +- name: "Pre-upgrade | check for kube-apiserver init script" + stat: + path: /etc/init.d/kube-apiserver + register: kube_apiserver_init_script + when: ansible_service_mgr in ["sysvinit","upstart"] + +- name: "Pre-upgrade | stop kube-apiserver if service defined" + service: + name: kube-apiserver + state: stopped + when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False)) + +- name: "Pre-upgrade | remove kube-apiserver service definition" + file: + path: "{{ item }}" + state: absent + when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False)) + with_items: + - /etc/systemd/system/kube-apiserver.service + - /etc/init.d/kube-apiserver diff --git a/roles/kubernetes/master/tasks/start.yml b/roles/kubernetes/master/tasks/start.yml deleted file mode 100644 index 9cd247c42..000000000 --- a/roles/kubernetes/master/tasks/start.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- name: Pause - pause: seconds=10 - -- name: reload systemd - command: systemctl daemon-reload - when: ansible_service_mgr == "systemd" and restart_apimaster is defined and restart_apimaster == True - -- name: reload kube-apiserver - service: - name: kube-apiserver - state: restarted - enabled: yes - when: ( restart_apimaster is defined and restart_apimaster == True) or - secret_changed | default(false) - -- name: Enable apiserver - service: - name: kube-apiserver - enabled: yes - state: started - when: restart_apimaster is not defined or restart_apimaster == False diff --git a/roles/kubernetes/master/templates/deb-kube-apiserver.initd.j2 b/roles/kubernetes/master/templates/deb-kube-apiserver.initd.j2 deleted file mode 100644 index 576c70128..000000000 --- a/roles/kubernetes/master/templates/deb-kube-apiserver.initd.j2 +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash -# -### BEGIN INIT INFO -# Provides: kube-apiserver -# Required-Start: $local_fs $network $syslog -# Required-Stop: -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: The Kubernetes apiserver -# Description: -# The Kubernetes apiserver. -### END INIT INFO - - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/sbin:/usr/sbin:/bin:/usr/bin -DESC="The Kubernetes apiserver" -NAME=kube-apiserver -DAEMON={{ bin_dir }}/kube-apiserver -DAEMON_LOG_FILE=/var/log/$NAME.log -PIDFILE=/var/run/$NAME.pid -SCRIPTNAME=/etc/init.d/$NAME -DAEMON_USER=root - -# Exit if the package is not installed -[ -x "$DAEMON" ] || exit 0 - -# Read configuration variable file if it is present -[ -r /etc/kubernetes/$NAME.env ] && . /etc/kubernetes/$NAME.env - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.2-14) to ensure that this file is present -# and status_of_proc is working. -. /lib/lsb/init-functions - -# -# Function that starts the daemon/service -# -do_start() -{ - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - start-stop-daemon --start --quiet --background --no-close \ - --make-pidfile --pidfile $PIDFILE \ - --exec $DAEMON -c $DAEMON_USER --test > /dev/null \ - || return 1 - start-stop-daemon --start --quiet --background --no-close \ - --make-pidfile --pidfile $PIDFILE \ - --exec $DAEMON -c $DAEMON_USER -- \ - $DAEMON_ARGS >> $DAEMON_LOG_FILE 2>&1 \ - || return 2 -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME - RETVAL="$?" - [ "$RETVAL" = 2 ] && return 2 - # Many daemons don't delete their pidfiles when they exit. - rm -f $PIDFILE - return "$RETVAL" -} - - -case "$1" in - start) - log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) log_end_msg 0 || exit 0 ;; - 2) log_end_msg 1 || exit 1 ;; - esac - ;; - stop) - log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) log_end_msg 0 ;; - 2) exit 1 ;; - esac - ;; - status) - status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $? - ;; - - restart|force-reload) - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 ;; # Old process is still running - *) log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 - ;; - esac - ;; - *) - echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 - exit 3 - ;; -esac diff --git a/roles/kubernetes/master/templates/kube-apiserver.j2 b/roles/kubernetes/master/templates/kube-apiserver.j2 deleted file mode 100644 index 61cb561ab..000000000 --- a/roles/kubernetes/master/templates/kube-apiserver.j2 +++ /dev/null @@ -1,58 +0,0 @@ -### -# kubernetes system config -# -# The following values are used to configure the kube-apiserver - -{% if ansible_service_mgr in ["sysvinit","upstart"] %} -# Logging directory -KUBE_LOGGING="--log-dir={{ kube_log_dir }} --logtostderr=true" -{% else %} -# logging to stderr means we get it in the systemd journal -KUBE_LOGGING="--logtostderr=true" -{% endif %} - -# Apiserver Log level, 0 is debug -KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}" - -# Should this cluster be allowed to run privileged docker containers -KUBE_ALLOW_PRIV="--allow_privileged=true" - -# The port on the local server to listen on. -KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{ kube_apiserver_port }}" - -# Insecure API address (default is localhost) -KUBE_API_INSECURE_BIND="--insecure-bind-address={{ kube_apiserver_insecure_bind_address | default('127.0.0.1') }}" - -# Address range to use for services -KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}" - -# Location of the etcd cluster -KUBE_ETCD_SERVERS="--etcd_servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %}" - -# Bind address for secure endpoint -KUBE_API_ADDRESS="--bind-address={{ ip | default(ansible_default_ipv4.address) }}" - -# default admission control policies -KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota" - -# RUNTIME API CONFIGURATION (e.g. enable extensions) -KUBE_RUNTIME_CONFIG="{% if kube_api_runtime_config is defined %}{% for conf in kube_api_runtime_config %}--runtime-config={{ conf }} {% endfor %}{% endif %}" - -# TLS CONFIGURATION -KUBE_TLS_CONFIG="--tls_cert_file={{ kube_cert_dir }}/apiserver.pem --tls_private_key_file={{ kube_cert_dir }}/apiserver-key.pem --client_ca_file={{ kube_cert_dir }}/ca.pem" - -# Add you own! -KUBE_API_ARGS="--token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/apiserver-key.pem --advertise-address={{ ip | default(ansible_default_ipv4.address) }}" - -{% if cloud_provider is defined and cloud_provider == "openstack" %} -KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config" -{% else %} -{# TODO: gce and aws don't need the cloud provider to be set? #} -KUBELET_CLOUDPROVIDER="" -{% endif %} - -{% if ansible_service_mgr in ["sysvinit","upstart"] %} -DAEMON_ARGS="$KUBE_LOGGING $KUBE_LOG_LEVEL $KUBE_ALLOW_PRIV $KUBE_API_PORT $KUBE_API_INSECURE_BIND \ -$KUBE_SERVICE_ADDRESSES $KUBE_ETCD_SERVERS $KUBE_ADMISSION_CONTROL $KUBE_RUNTIME_CONFIG \ -$KUBE_TLS_CONFIG $KUBE_API_ARGS $KUBELET_CLOUDPROVIDER" -{% endif %} diff --git a/roles/kubernetes/master/templates/kube-apiserver.service.j2 b/roles/kubernetes/master/templates/kube-apiserver.service.j2 deleted file mode 100644 index 699797171..000000000 --- a/roles/kubernetes/master/templates/kube-apiserver.service.j2 +++ /dev/null @@ -1,30 +0,0 @@ -[Unit] -Description=Kubernetes API Server -Documentation=https://github.com/GoogleCloudPlatform/kubernetes -Requires=etcd.service -After=etcd.service - -[Service] -EnvironmentFile=/etc/kubernetes/kube-apiserver.env -User=kube -ExecStart={{ bin_dir }}/kube-apiserver \ - $KUBE_LOGTOSTDERR \ - $KUBE_LOG_LEVEL \ - $KUBE_ETCD_SERVERS \ - $KUBE_API_ADDRESS \ - $KUBE_API_PORT \ - $KUBE_API_INSECURE_BIND \ - $KUBELET_PORT \ - $KUBE_ALLOW_PRIV \ - $KUBE_SERVICE_ADDRESSES \ - $KUBE_ADMISSION_CONTROL \ - $KUBE_RUNTIME_CONFIG \ - $KUBE_TLS_CONFIG \ - $KUBE_API_ARGS \ - $KUBELET_CLOUDPROVIDER -Restart=on-failure -Type=notify -LimitNOFILE=65536 - -[Install] -WantedBy=multi-user.target diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 48b013e6c..f14f3900a 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -11,13 +11,16 @@ spec: - /hyperkube - apiserver - --advertise-address={{ ip | default(ansible_default_ipv4.address) }} - - --etcd-servers={% for srv in groups['etcd'] %}http://{{ hostvars[srv]['access_ip'] | default(hostvars[srv]['ip']|default(hostvars[srv]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %} + - --insecure-bind-address={{ kube_apiserver_insecure_bind_address | default('127.0.0.1') }} + - --etcd-servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %} + - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota - --service-cluster-ip-range={{ kube_service_addresses }} - --client-ca-file={{ kube_cert_dir }}/ca.pem - --basic-auth-file={{ kube_users_dir }}/known_users.csv - --tls-cert-file={{ kube_cert_dir }}/apiserver.pem - --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem + - --token-auth-file={{ kube_token_dir }}/known_tokens.csv - --service-account-key-file={{ kube_cert_dir }}/apiserver-key.pem - --secure-port={{ kube_apiserver_port }} - --insecure-port={{ kube_apiserver_insecure_port }} @@ -26,9 +29,13 @@ spec: - --runtime-config={{ conf }} {% endfor %} {% endif %} - - --token-auth-file={{ kube_token_dir }}/known_tokens.csv - --v={{ kube_log_level | default('2') }} - --allow-privileged=true +{% if cloud_provider is defined and cloud_provider == "openstack" %} + - --cloud-provider={{ cloud_provider }} + - --cloud-config={{ kube_config_dir }}/cloud_config + - 2>&1 >> {{ kube_log_dir }}/kube-apiserver.log +{% endif %} ports: - containerPort: {{ kube_apiserver_port }} hostPort: {{ kube_apiserver_port }} @@ -43,6 +50,8 @@ spec: - mountPath: /etc/ssl/certs name: ssl-certs-host readOnly: true + - mountPath: /var/log/ + name: logfile volumes: - hostPath: path: {{ kube_config_dir }} @@ -50,3 +59,6 @@ spec: - hostPath: path: /etc/ssl/certs/ name: ssl-certs-host + - hostPath: + path: /var/log/ + name: logfile diff --git a/roles/uploads/defaults/main.yml b/roles/uploads/defaults/main.yml index 563d96b34..a624d5e79 100644 --- a/roles/uploads/defaults/main.yml +++ b/roles/uploads/defaults/main.yml @@ -88,12 +88,3 @@ downloads: url: "{{ kube_download_url }}/kubectl" owner: "kube" mode: "0755" - - - name: kubernetes-apiserver - dest: kubernetes/bin/kube-apiserver - version: "{{kube_version}}" - sha256: "{{vars['kube_checksum'][kube_version]['kube_apiserver']}}" - source_url: "{{ kube_download_url }}/kube-apiserver" - url: "{{ kube_download_url }}/kube-apiserver" - owner: "kube" - mode: "0755" diff --git a/roles/uploads/vars/kube_versions.yml b/roles/uploads/vars/kube_versions.yml index 5b5f64a42..f454eb2dc 100644 --- a/roles/uploads/vars/kube_versions.yml +++ b/roles/uploads/vars/kube_versions.yml @@ -1,22 +1,17 @@ kube_checksum: v1.2.2: - kube_apiserver: eb1bfd8b877052cbd1991b8c429a1d06661f4cb019905e20e128174f724e16de kubectl: 473e6924569fba30d4a50cecdc2cae5f31d97d1f662463e85b74a472105dcff4 kubelet: f16827dc7e7c82f0e215f0fc73eb01e2dfe91a2ec83f9cbcaf8d37c91b64fd3b v1.2.3: - kube_apiserver_checksum: ebaeeeb72cb29b358337b330617a96355ff2d08a5a523fc1a81beba36cc9d6f9 kubectl_checksum: 394853edd409a721bcafe4f1360009ef9f845050719fe7d6fc7176f45cc92a8c kubelet_checksum: 633bb41c51c5c0df0645dd60ba82b12eba39d009eb87bae9227de7d9a89c0797 v1.2.4: - kube_apiserver: 6ac99b36b02968459e026fcfc234207c66064b5e11816b69dd8fc234b2ffec1e kubectl: dac61fbd506f7a17540feca691cd8a9d9d628d59661eebce788a50511f578897 kubelet: 4adaf40592248eef6fd4fa126464915ea41e624a70dc77178089760ed235e341 v1.2.5: - kube_apiserver: fbe8296ad4b194c06f6802a126d35cd2887dc1aded308d4da2b580f270412b33 kubectl: 5526a496a84701015485e32c86486e2f23599f7a865164f546e619c6a62f7f19 kubelet: cd15b929f0190876216f397c2c6e7aa8c08d3b047fd90b4980cd68c8f4896211 v1.3.0: - kube_apiserver: 431cd312984a29f45590138e990d5c4d537b069b71f2587a72414fabc4fcffdd kubectl: f40b2d0ff33984e663a0dea4916f1cb9041abecc09b11f9372cdb8049ded95dc kubelet: bd5f10ccb95fe6e95ddf7ad8a119195c27cb2bce4be6f80c1810ff1a2111496d kube_version: v1.3.0