Merge pull request #87 from Smana/apiserver_on_host

Apiserver on host

roles/download/defaults/main.yml

@@ -19,6 +19,7 @@ calico_plugin_checksum: "032f582f5eeec6fb26191d2fbcbf8bca4da3b14abb579db7baa7b35
 etcd_checksum: "6c4e5cdeaaac1a70b8f06b5dd6b82c37ff19993c9bca81248975610e555c4b9b"
 kubectl_checksum: "873ba19926d17a3287dc8639ea1434fe3cd0cb4e61d82101ba754922cfc7a633"
 kubelet_checksum: "f2d1eae3fa6e304f6cbc9b2621e4b86fc3bcb4e74a15d35f58bf00e45c706e0a"
+kube_apiserver_checksum: "bb3814c4df65f1587a3650140437392ce3fb4b64f51d459457456691c99f1202"
 
 downloads:
   - name: calico
@@ -56,3 +57,10 @@ downloads:
     url: "{{ kube_download_url }}/kubectl"
     owner: "kube"
     mode: "0755"
+
+  - name: kubernetes-apiserver
+    dest: kubernetes/bin/kube-apiserver
+    sha256: "{{kube_apiserver_checksum}}"
+    url: "{{ kube_download_url }}/kube-apiserver"
+    owner: "kube"
+    mode: "0755"

roles/etcd/tasks/configure.yml

@@ -17,7 +17,7 @@
   when: init_system == "sysvinit" and ansible_os_family == "Debian"
   notify: restart etcd
 
-- name: Configure |  Create etcd environment vars file
+- name: Configure |  Create etcd config file
   template:
     src: etcd.j2
     dest: /etc/etcd.env

roles/kubernetes/master/handlers/main.yml

@@ -1,14 +1,25 @@
 ---
-- name: reload systemd
-  command: systemctl daemon-reload
-
 - name: restart systemd-kubelet
   command: /bin/true
   notify:
     - reload systemd
     - restart kubelet
 
+- name: restart systemd-kube-apiserver
+  command: /bin/true
+  notify:
+    - reload systemd
+    - restart kube-apiserver
+
+- name: reload systemd
+  command: systemctl daemon-reload
+
 - name: restart kubelet
   service:
     name: kubelet
     state: restarted
+
+- name: restart kube-apiserver
+  service:
+    name: kube-apiserver
+    state: restarted

roles/kubernetes/master/tasks/gen_kube_tokens.yml

@@ -0,0 +1,24 @@
+---
+- name: tokens | generate tokens for master components
+  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
+  environment:
+    TOKEN_DIR: "{{ kube_token_dir }}"
+  with_nested:
+    - [ "system:kubectl" ]
+    - "{{ groups['kube-master'] }}"
+  register: gentoken_master
+  changed_when: "'Added' in gentoken_master.stdout"
+  when: inventory_hostname == groups['kube-master'][0]
+  notify: restart kube-apiserver
+
+- name: tokens | generate tokens for node components
+  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
+  environment:
+    TOKEN_DIR: "{{ kube_token_dir }}"
+  with_nested:
+    - [ 'system:kubelet' ]
+    - "{{ groups['kube-node'] }}"
+  register: gentoken_node
+  changed_when: "'Added' in gentoken_node.stdout"
+  when: inventory_hostname == groups['kube-master'][0]
+  notify: restart kube-apiserver

roles/kubernetes/master/tasks/main.yml

@@ -1,13 +1,18 @@
 ---
+- include: gen_kube_tokens.yml
+  tags: tokens
+
 - name: Copy kubectl bash completion
   copy:
     src: kubectl_bash_completion.sh
     dest: /etc/bash_completion.d/kubectl.sh
 
-- name: Install kubectl binary
+- name: Install kubernetes binaries
-  command: cp -pf "{{ local_release_dir }}/kubernetes/bin/kubectl" "{{ bin_dir }}/kubectl"
+  command: cp -pf "{{ local_release_dir }}/kubernetes/bin/{{ item }}" "{{ bin_dir }}/{{ item }}"
   changed_when: false
-  notify: restart kubelet
+  with_items:
+    - kubectl
+    - kube-apiserver
 
 - name: populate users for basic auth in API
   lineinfile:
@@ -16,6 +21,7 @@
     line: '{{ item.value.pass }},{{ item.key }},{{ item.value.role }}'
     backup: yes
   with_dict: "{{ kube_users }}"
+  notify: restart kube-apiserver
 
 # Sync masters
 - name: synchronize auth directories for masters
@@ -33,21 +39,48 @@
   delegate_to: "{{ groups['kube-master'][0] }}"
   when: inventory_hostname != "{{ groups['kube-master'][0] }}"
 
-# Write manifests
+- name: install | Write kube-apiserver systemd init file
-- name: Write kube-apiserver manifest
   template:
-    src: manifests/kube-apiserver.manifest.j2
+    src: "kube-apiserver.service.j2"
-    dest: "{{ kube_manifest_dir }}/kube-apisever.manifest"
+    dest: "/etc/systemd/system/kube-apiserver.service"
-  notify: restart kubelet
+    backup: yes
+  when: init_system == "systemd"
+  notify: restart systemd-kube-apiserver
 
-- meta: flush_handlers
+- name: install | Write kube-apiserver initd script
+  template:
+    src: "deb-kube-apiserver.initd.j2"
+    dest: "/etc/init.d/kube-apiserver"
+    owner: root
+    mode: 755
+    backup: yes
+  when: init_system == "sysvinit" and ansible_os_family == "Debian"
+  notify: restart kube-apiserver
 
-- name: wait for the apiserver to be running (pulling image and running container)
+- name: Allow apiserver to bind on both secure and insecure ports
-  wait_for:
+  shell: setcap cap_net_bind_service+ep {{ bin_dir }}/kube-apiserver
-    port: "{{kube_apiserver_insecure_port}}"
+  changed_when: false
-    delay: 10
-    timeout: 60
 
+- name: Write kube-apiserver config file
+  template:
+    src: "kube-apiserver.j2"
+    dest: "{{ kube_config_dir }}/kube-apiserver.env"
+    backup: yes
+  notify: restart kube-apiserver
+
+# restart apiserver if calico tokens list has changed
+- name: Reload tokens (restart apiserver)
+  command: /bin/true
+  notify: restart kube-apiserver
+  changed_when: is_gentoken_calico|default(false)
+
+- name: Enable apiserver
+  service:
+    name: kube-apiserver
+    enabled: yes
+    state: started
+
+# Create kube-system namespace
 - name: copy 'kube-system' namespace manifest
   copy: src=namespace.yml dest=/etc/kubernetes/kube-system-ns.yml
   run_once: yes
@@ -60,12 +93,18 @@
   ignore_errors: yes
   run_once: yes
 
+- name: wait for the apiserver to be running
+  wait_for:
+    port: "{{kube_apiserver_insecure_port}}"
+    delay: 10
+    timeout: 60
+
 - name: Create 'kube-system' namespace
   command: kubectl create -f /etc/kubernetes/kube-system-ns.yml
-  when: kubesystem|failed
   changed_when: False
-  run_once: yes
+  when: kubesystem|failed and inventory_hostname == groups['kube-master'][0]
 
+# Write manifests
 - name: Write kube-controller-manager manifest
   template:
     src: manifests/kube-controller-manager.manifest.j2

roles/kubernetes/master/templates/deb-kube-apiserver.initd.j2

@@ -0,0 +1,118 @@
+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides:   kube-apiserver 
+# Required-Start:    $local_fs $network $syslog
+# Required-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: The Kubernetes apiserver
+# Description:
+#   The Kubernetes apiserver.
+### END INIT INFO
+
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="The Kubernetes apiserver"
+NAME=kube-apiserver
+DAEMON={{ bin_dir }}/kube-apiserver
+DAEMON_LOG_FILE=/var/log/$NAME.log
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMON_USER=root
+
+# Exit if the package is not installed
+[ -x "$DAEMON" ] || exit 0
+
+# Read configuration variable file if it is present
+[ -r /etc/kubernetes/$NAME.env ] && . /etc/kubernetes/$NAME.env
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
+# and status_of_proc is working.
+. /lib/lsb/init-functions
+
+#
+# Function that starts the daemon/service
+#
+do_start()
+{
+        # Return
+        #   0 if daemon has been started
+        #   1 if daemon was already running
+        #   2 if daemon could not be started
+        start-stop-daemon --start --quiet --background --no-close \
+                --make-pidfile --pidfile $PIDFILE \
+                --exec $DAEMON -c $DAEMON_USER --test > /dev/null \
+                || return 1
+        start-stop-daemon --start --quiet --background --no-close \
+                --make-pidfile --pidfile $PIDFILE \
+                --exec $DAEMON -c $DAEMON_USER -- \
+                $DAEMON_ARGS >> $DAEMON_LOG_FILE 2>&1 \
+                || return 2
+}
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+        # Return
+        #   0 if daemon has been stopped
+        #   1 if daemon was already stopped
+        #   2 if daemon could not be stopped
+        #   other if a failure occurred
+        start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME
+        RETVAL="$?"
+        [ "$RETVAL" = 2 ] && return 2
+        # Many daemons don't delete their pidfiles when they exit.
+        rm -f $PIDFILE
+        return "$RETVAL"
+}
+
+
+case "$1" in
+  start)
+        log_daemon_msg "Starting $DESC" "$NAME"
+        do_start
+        case "$?" in
+                0|1) log_end_msg 0 || exit 0 ;;
+                2) log_end_msg 1 || exit 1 ;;
+        esac
+        ;;
+  stop)
+        log_daemon_msg "Stopping $DESC" "$NAME"
+        do_stop
+        case "$?" in
+                0|1) log_end_msg 0 ;;
+                2) exit 1 ;;
+        esac
+        ;;
+  status)
+        status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $?
+        ;;
+
+  restart|force-reload)
+        log_daemon_msg "Restarting $DESC" "$NAME"
+        do_stop
+        case "$?" in
+          0|1)
+                do_start
+                case "$?" in
+                        0) log_end_msg 0 ;;
+                        1) log_end_msg 1 ;; # Old process is still running
+                        *) log_end_msg 1 ;; # Failed to start
+                esac
+                ;;
+          *)
+                # Failed to stop
+                log_end_msg 1
+                ;;
+        esac
+        ;;
+  *)
+        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
+        exit 3
+        ;;
+esac

roles/kubernetes/master/templates/kube-apiserver.j2

@@ -0,0 +1,44 @@
+###
+# kubernetes system config
+#
+# The following values are used to configure the kube-apiserver
+
+{% if init_system == "sysvinit" %}
+# Logging directory
+KUBE_LOGGING="--log-dir={{ kube_log_dir }} --logtostderr=true"
+{% else %}
+# logging to stderr means we get it in the systemd journal
+KUBE_LOGGING="--logtostderr=true"
+{% endif %}
+
+# Apiserver Log level, 0 is debug
+KUBE_LOG_LEVEL="{{ kube_log_level | default('--v=2') }}"
+
+# Should this cluster be allowed to run privileged docker containers
+KUBE_ALLOW_PRIV="--allow_privileged=true"
+
+# The port on the local server to listen on.
+KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{ kube_apiserver_port }}"
+
+# Address range to use for services
+KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}"
+
+# Location of the etcd cluster
+KUBE_ETCD_SERVERS="--etcd_servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2379{% if not loop.last %},{% endif %}{% endfor %}"
+
+# default admission control policies
+KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
+
+# RUNTIME API CONFIGURATION (e.g. enable extensions)
+KUBE_RUNTIME_CONFIG="{% if kube_api_runtime_config is defined %}{% for conf in kube_api_runtime_config %}--runtime-config={{ conf }} {% endfor %}{% endif %}"
+
+# TLS CONFIGURATION
+KUBE_TLS_CONFIG="--tls_cert_file={{ kube_cert_dir }}/apiserver.pem --tls_private_key_file={{ kube_cert_dir }}/apiserver-key.pem --client_ca_file={{ kube_cert_dir }}/ca.pem"
+
+# Add you own!
+KUBE_API_ARGS="--token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/apiserver-key.pem"
+
+{% if init_system == "sysvinit" %}
+DAEMON_ARGS="$KUBE_LOGGING $KUBE_LOG_LEVEL $KUBE_ALLOW_PRIV $KUBE_API_PORT $KUBE_SERVICE_ADDRESSES \
+$KUBE_ETCD_SERVERS $KUBE_ADMISSION_CONTROL $KUBE_RUNTIME_CONFIG $KUBE_TLS_CONFIG $KUBE_API_ARGS"
+{% endif %}

roles/kubernetes/master/templates/kube-apiserver.service.j2

@@ -0,0 +1,28 @@
+[Unit]
+Description=Kubernetes API Server
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+Requires=kube-apiserver.service
+After=kube-apiserver.service
+
+[Service]
+EnvironmentFile=/etc/kubernetes/kube-apiserver.env
+User=kube
+ExecStart={{ bin_dir }}/kube-apiserver \
+	    $KUBE_LOGTOSTDERR \
+	    $KUBE_LOG_LEVEL \
+	    $KUBE_ETCD_SERVERS \
+	    $KUBE_API_ADDRESS \
+	    $KUBE_API_PORT \
+	    $KUBELET_PORT \
+	    $KUBE_ALLOW_PRIV \
+	    $KUBE_SERVICE_ADDRESSES \
+	    $KUBE_ADMISSION_CONTROL \
+	    $KUBE_RUNTIME_CONFIG \
+        $KUBE_TLS_CONFIG \
+	    $KUBE_API_ARGS
+Restart=on-failure
+Type=notify
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

roles/kubernetes/node/defaults/main.yml

@@ -24,6 +24,9 @@ kube_users_dir: "{{ kube_config_dir }}/users"
 # pods on startup
 kube_manifest_dir: "{{ kube_config_dir }}/manifests"
 
+# Logging directory (sysvinit systems)
+kube_log_dir: "/var/log/kubernetes"
+
 dns_domain: "{{ cluster_name }}"
 
 kube_proxy_mode: userspace

roles/kubernetes/node/handlers/main.yml

@@ -1,12 +1,16 @@
 ---
+- name: reload systemd
+  command: systemctl daemon-reload
+
 - name: restart systemd-kubelet
   command: /bin/true
   notify:
     - reload systemd
     - restart kubelet
 
-- name: reload systemd
+- name: set is_gentoken_calico fact
-  command: systemctl daemon-reload
+  set_fact:
+    is_gentoken_calico: true
 
 - name: restart kubelet
   service:

roles/kubernetes/node/tasks/gen_calico_tokens.yml

@@ -0,0 +1,27 @@
+---
+- name: tokens | copy the token gen script
+  copy:
+    src=kube-gen-token.sh
+    dest={{ kube_script_dir }}
+    mode=u+x
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: tokens | generate tokens for calico
+  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
+  environment:
+    TOKEN_DIR: "{{ kube_token_dir }}"
+  with_nested:
+    - [ "system:calico" ]
+    - "{{ groups['k8s-cluster'] }}"
+  register: gentoken_calico
+  changed_when: "'Added' in gentoken_calico.stdout"
+  when: kube_network_plugin == "calico"
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  notify: set is_gentoken_calico fact
+
+- name: tokens | get the calico token values
+  slurp:
+    src: "{{ kube_token_dir }}/system:calico-{{ inventory_hostname }}.token"
+  register: calico_token
+  when: kube_network_plugin == "calico"
+  delegate_to: "{{ groups['kube-master'][0] }}"

roles/kubernetes/node/tasks/gen_tokens.yml

@@ -1,48 +0,0 @@
----
-- name: tokens | copy the token gen script
-  copy:
-    src=kube-gen-token.sh
-    dest={{ kube_script_dir }}
-    mode=u+x
-  when: inventory_hostname == groups['kube-master'][0]
-
-- name: tokens | generate tokens for master components
-  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
-  environment:
-    TOKEN_DIR: "{{ kube_token_dir }}"
-  with_nested:
-    - [ "system:kubectl" ]
-    - "{{ groups['kube-master'] }}"
-  register: gentoken
-  changed_when: "'Added' in gentoken.stdout"
-  when: inventory_hostname == groups['kube-master'][0]
-
-- name: tokens | generate tokens for node components
-  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
-  environment:
-    TOKEN_DIR: "{{ kube_token_dir }}"
-  with_nested:
-    - [ 'system:kubelet' ]
-    - "{{ groups['kube-node'] }}"
-  register: gentoken
-  changed_when: "'Added' in gentoken.stdout"
-  when: inventory_hostname == groups['kube-master'][0]
-
-- name: tokens | generate tokens for calico
-  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
-  environment:
-    TOKEN_DIR: "{{ kube_token_dir }}"
-  with_nested:
-    - [ "system:calico" ]
-    - "{{ groups['k8s-cluster'] }}"
-  register: gentoken
-  changed_when: "'Added' in gentoken.stdout"
-  when: kube_network_plugin == "calico"
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: tokens | get the calico token values
-  slurp:
-    src: "{{ kube_token_dir }}/system:calico-{{ inventory_hostname }}.token"
-  register: calico_token
-  when: kube_network_plugin == "calico"
-  delegate_to: "{{ groups['kube-master'][0] }}"

roles/kubernetes/node/tasks/install.yml

@@ -1,7 +1,4 @@
 ---
-- debug: msg="{{init_system == "systemd"}}"
-- debug: msg="{{init_system}}"
-
 - name: install | Write kubelet systemd init file
   template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes
   when: init_system == "systemd"

roles/kubernetes/node/tasks/main.yml

@@ -1,12 +1,28 @@
 ---
-- name: create kubernetes config directory
+- name: Create kubernetes config directory
-  file: path={{ kube_config_dir }} state=directory
+  file:
+    path: "{{ kube_config_dir }}"
+    state: directory
+    owner: kube
 
-- name: create kubernetes script directory
+- name: Create kubernetes script directory
-  file: path={{ kube_script_dir }} state=directory
+  file:
+    path: "{{ kube_script_dir }}"
+    state: directory
+    owner: kube
 
-- name: Make sure manifest directory exists
+- name: Create kubernetes manifests directory
-  file: path={{ kube_manifest_dir }} state=directory
+  file:
+    path: "{{ kube_manifest_dir }}"
+    state: directory
+    owner: kube
+
+- name: Create kubernetes logs directory
+  file:
+    path: "{{ kube_log_dir }}"
+    state: directory
+    owner: kube
+  when: init_system == "sysvinit"
 
 - include: secrets.yml
   tags:
@@ -15,7 +31,7 @@
 - include: install.yml
 
 - name: Write kubelet config file
-  template: src=kubelet.j2 dest={{ kube_config_dir }}/kubelet backup=yes
+  template: src=kubelet.j2 dest={{ kube_config_dir }}/kubelet.env backup=yes
   notify:
     - restart kubelet
 

roles/kubernetes/node/tasks/secrets.yml

@@ -16,7 +16,7 @@
 - include: gen_certs.yml
   when: inventory_hostname == groups['kube-master'][0]
 
-- include: gen_tokens.yml
+- include: gen_calico_tokens.yml
 
 # Sync certs between nodes
 - name: Secrets | create user

roles/kubernetes/node/templates/deb-kubelet.initd.j2

@@ -27,7 +27,7 @@ DAEMON_USER=root
 [ -x "$DAEMON" ] || exit 0
 
 # Read configuration variable file if it is present
-[ -r /etc/kubernetes/$NAME ] && . /etc/kubernetes/$NAME
+[ -r /etc/kubernetes/$NAME.env ] && . /etc/kubernetes/$NAME.env
 
 # Define LSB log_* functions.
 # Depend on lsb-base (>= 3.2-14) to ensure that this file is present

roles/kubernetes/node/templates/kubelet.j2

@@ -1,4 +1,10 @@
-KUBE_LOGTOSTDERR="--logtostderr=true"
+{% if init_system == "sysvinit" %}
+# Logging directory
+KUBE_LOGGING="--log-dir={{ kube_log_dir }} --logtostderr=true"
+{% else %}
+# logging to stderr means we get it in the systemd journal
+KUBE_LOGGING="--logtostderr=true"
+{% endif %}
 KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}"
 KUBE_ALLOW_PRIV="--allow_privileged=true"
 KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}"
@@ -23,6 +29,6 @@ KUBELET_NETWORK_PLUGIN="--network_plugin={{ kube_network_plugin }}"
 # Should this cluster be allowed to run privileged docker containers
 KUBE_ALLOW_PRIV="--allow_privileged=true"
 {% if init_system == "sysvinit" %}
-DAEMON_ARGS="$KUBE_LOGTOSTDERR $KUBE_LOG_LEVEL $KUBE_ALLOW_PRIV $KUBELET_API_SERVER $KUBELET_ADDRESS \
+DAEMON_ARGS="$KUBE_LOGGING $KUBE_LOG_LEVEL $KUBE_ALLOW_PRIV $KUBELET_API_SERVER $KUBELET_ADDRESS \
 $KUBELET_HOSTNAME $KUBELET_REGISTER_NODE $KUBELET_ARGS $KUBELET_ARGS $KUBELET_NETWORK_PLUGIN"
 {% endif %}

roles/kubernetes/node/templates/kubelet.service.j2

@@ -8,7 +8,7 @@ After=docker.service
 {% endif %}
 
 [Service]
-EnvironmentFile=/etc/kubernetes/kubelet
+EnvironmentFile=/etc/kubernetes/kubelet.env
 ExecStart={{ bin_dir }}/kubelet \
 	    $KUBE_LOGTOSTDERR \
 	    $KUBE_LOG_LEVEL \

roles/kubernetes/node/templates/rh-kubelet.initd.j2

@@ -27,7 +27,7 @@ pidfile="/var/run/$prog.pid"
 lockfile="/var/lock/subsys/$prog"
 logfile="/var/log/$prog"
 
-[ -e /etc/kubernetes/$prog ] && . /etc/kubernetes/$prog
+[ -e /etc/kubernetes/$prog.env ] && . /etc/kubernetes/$prog.env
 
 start() {
     if [ ! -x $exec ]; then