From 94596388f7ebdf4c46545f4204434a4bbaaebd6b Mon Sep 17 00:00:00 2001
From: Spencer Smith <robertspencersmith@gmail.com>
Date: Fri, 14 Apr 2017 17:33:04 -0400
Subject: [PATCH 1/5] add ability for custom flags

---
 docs/vars.md                                       | 14 ++++++++++++++
 roles/kubernetes/master/defaults/main.yml          |  7 +++++++
 .../templates/manifests/kube-apiserver.manifest.j2 |  3 +++
 .../manifests/kube-controller-manager.manifest.j2  |  3 +++
 .../templates/manifests/kube-scheduler.manifest.j2 |  3 +++
 roles/kubernetes/node/defaults/main.yml            |  3 +++
 roles/kubernetes/node/templates/kubelet.j2         |  2 +-
 7 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/docs/vars.md b/docs/vars.md
index 966b3ffc8..603a614b2 100644
--- a/docs/vars.md
+++ b/docs/vars.md
@@ -98,6 +98,20 @@ Stack](https://github.com/kubernetes-incubator/kargo/blob/master/docs/dns-stack.
   loaded by preinstall kubernetes processes.  For example, ceph and rbd backed volumes.  Set this variable to
   true to let kubelet load kernel modules.
 
+##### Custom flags for Kube Components
+For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. Example:
+```
+kubelet_custom_flags:
+  - "--eviction-hard=memory.available<100Mi"
+  - "--eviction-soft-grace-period=memory.available=30s"
+  - "--eviction-soft=memory.available<300Mi"
+```
+The possible vars are:
+* *apiserver_custom_flags*
+* *controller_mgr_custom_flags*
+* *scheduler_custom_flags*
+* *kubelet_custom_flags*
+
 #### User accounts
 
 Kargo sets up two Kubernetes accounts by default: ``root`` and ``kube``. Their
diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml
index 016df0c64..bd5461239 100644
--- a/roles/kubernetes/master/defaults/main.yml
+++ b/roles/kubernetes/master/defaults/main.yml
@@ -51,3 +51,10 @@ kube_oidc_auth: false
 # kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem
 # kube_oidc_username_claim: sub
 # kube_oidc_groups_claim: groups
+
+##Variables for custom flags
+apiserver_custom_flags: []
+
+controller_mgr_custom_flags: []
+
+scheduler_custom_flags: []
\ No newline at end of file
diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
index 967f0a9cb..721474466 100644
--- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
@@ -81,6 +81,9 @@ spec:
 {% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=')  %}
     - --anonymous-auth={{ kube_api_anonymous_auth }}
 {% endif %}
+{% for flag in apiserver_custom_flags %}
+    - {{ flag }}
+{% endfor %}
     livenessProbe:
       httpGet:
         host: 127.0.0.1
diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
index 477d6a64f..0f66509ad 100644
--- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
@@ -46,6 +46,9 @@ spec:
     - --configure-cloud-routes=true
     - --cluster-cidr={{ kube_pods_subnet }}
 {% endif %}
+{% for flag in controller_mgr_custom_flags %}
+    - {{ flag }}
+{% endfor %}
     livenessProbe:
       httpGet:
         host: 127.0.0.1
diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
index 7431ddf3d..a549d5296 100644
--- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
@@ -27,6 +27,9 @@ spec:
     - --leader-elect=true
     - --master={{ kube_apiserver_endpoint }}
     - --v={{ kube_log_level }}
+{% for flag in scheduler_custom_flags %}
+    - {{ flag }}
+{% endfor %}
     livenessProbe:
       httpGet:
         host: 127.0.0.1
diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml
index 952214179..7f1e6f4a0 100644
--- a/roles/kubernetes/node/defaults/main.yml
+++ b/roles/kubernetes/node/defaults/main.yml
@@ -45,3 +45,6 @@ etcd_config_dir: /etc/ssl/etcd
 kube_apiserver_node_port_range: "30000-32767"
 
 kubelet_load_modules: false
+
+##Support custom flags to be passed to kubelet
+kubelet_custom_flags: []
\ No newline at end of file
diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2
index ba02e5eb9..d2ca95ad4 100644
--- a/roles/kubernetes/node/templates/kubelet.j2
+++ b/roles/kubernetes/node/templates/kubelet.j2
@@ -44,7 +44,7 @@ KUBELET_HOSTNAME="--hostname-override={{ ansible_hostname }}"
 {%   set node_labels %}--node-labels=node-role.kubernetes.io/node=true{% endset %}
 {% endif %}
 
-KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }}"
+KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}"
 {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave", "canal"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %}

From f9d4a1c1d87f87e6daf27757cfc70c4bcb7f5dfe Mon Sep 17 00:00:00 2001
From: Spencer Smith <robertspencersmith@gmail.com>
Date: Mon, 17 Apr 2017 11:09:34 -0400
Subject: [PATCH 2/5] update to safeguard against accidentally passing string
 instead of list

---
 .../master/templates/manifests/kube-apiserver.manifest.j2 | 8 ++++++--
 .../manifests/kube-controller-manager.manifest.j2         | 8 ++++++--
 .../master/templates/manifests/kube-scheduler.manifest.j2 | 8 ++++++--
 roles/kubernetes/node/templates/kubelet.j2                | 2 +-
 4 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
index 721474466..c0ddf329b 100644
--- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
@@ -81,9 +81,13 @@ spec:
 {% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=')  %}
     - --anonymous-auth={{ kube_api_anonymous_auth }}
 {% endif %}
-{% for flag in apiserver_custom_flags %}
+{% if apiserver_custom_flags is string %}
+    - {{ apiserver_custom_flags }}
+{% else % }
+{%   for flag in apiserver_custom_flags %}
     - {{ flag }}
-{% endfor %}
+{%   endfor %}
+{% endif %}
     livenessProbe:
       httpGet:
         host: 127.0.0.1
diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
index 0f66509ad..1bdcc4324 100644
--- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
@@ -46,9 +46,13 @@ spec:
     - --configure-cloud-routes=true
     - --cluster-cidr={{ kube_pods_subnet }}
 {% endif %}
-{% for flag in controller_mgr_custom_flags %}
+{% if controller_mgr_custom_flags is string %}
+    - {{ controller_mgr_custom_flags }}
+{% else % }
+{%   for flag in controller_mgr_custom_flags %}
     - {{ flag }}
-{% endfor %}
+{%   endfor %}
+{% endif %}
     livenessProbe:
       httpGet:
         host: 127.0.0.1
diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
index a549d5296..d21db5470 100644
--- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
@@ -27,9 +27,13 @@ spec:
     - --leader-elect=true
     - --master={{ kube_apiserver_endpoint }}
     - --v={{ kube_log_level }}
-{% for flag in scheduler_custom_flags %}
+{% if scheduler_custom_flags is string %}
+    - {{ scheduler_custom_flags }}
+{% else % }
+{%   for flag in scheduler_custom_flags %}
     - {{ flag }}
-{% endfor %}
+{%   endfor %}
+{% endif %}
     livenessProbe:
       httpGet:
         host: 127.0.0.1
diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2
index d2ca95ad4..df207a545 100644
--- a/roles/kubernetes/node/templates/kubelet.j2
+++ b/roles/kubernetes/node/templates/kubelet.j2
@@ -44,7 +44,7 @@ KUBELET_HOSTNAME="--hostname-override={{ ansible_hostname }}"
 {%   set node_labels %}--node-labels=node-role.kubernetes.io/node=true{% endset %}
 {% endif %}
 
-KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}"
+KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% if kubelet_custom_flags is string %}{{kubelet_custom_flags}}{% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}"
 {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave", "canal"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %}

From 04a769bb372b408c790725e23f5dbd0cf1599d7f Mon Sep 17 00:00:00 2001
From: Spencer Smith <robertspencersmith@gmail.com>
Date: Mon, 17 Apr 2017 11:11:10 -0400
Subject: [PATCH 3/5] ensure spacing on string of flags

---
 roles/kubernetes/node/templates/kubelet.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2
index df207a545..8de1e63e9 100644
--- a/roles/kubernetes/node/templates/kubelet.j2
+++ b/roles/kubernetes/node/templates/kubelet.j2
@@ -44,7 +44,7 @@ KUBELET_HOSTNAME="--hostname-override={{ ansible_hostname }}"
 {%   set node_labels %}--node-labels=node-role.kubernetes.io/node=true{% endset %}
 {% endif %}
 
-KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% if kubelet_custom_flags is string %}{{kubelet_custom_flags}}{% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}"
+KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}"
 {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave", "canal"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
 {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %}

From 3f302c8d471702eac316a5f418af00fab48859d1 Mon Sep 17 00:00:00 2001
From: Spencer Smith <robertspencersmith@gmail.com>
Date: Mon, 17 Apr 2017 12:13:39 -0400
Subject: [PATCH 4/5] ensure spacing on string of flags

---
 .../master/templates/manifests/kube-apiserver.manifest.j2       | 2 +-
 .../templates/manifests/kube-controller-manager.manifest.j2     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
index c0ddf329b..a3b8a6f0a 100644
--- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
@@ -83,7 +83,7 @@ spec:
 {% endif %}
 {% if apiserver_custom_flags is string %}
     - {{ apiserver_custom_flags }}
-{% else % }
+{% else %}
 {%   for flag in apiserver_custom_flags %}
     - {{ flag }}
 {%   endfor %}
diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
index 1bdcc4324..b483047db 100644
--- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
@@ -48,7 +48,7 @@ spec:
 {% endif %}
 {% if controller_mgr_custom_flags is string %}
     - {{ controller_mgr_custom_flags }}
-{% else % }
+{% else %}
 {%   for flag in controller_mgr_custom_flags %}
     - {{ flag }}
 {%   endfor %}

From 72d5db92a8b4bc957312a2240e81b2164c80db94 Mon Sep 17 00:00:00 2001
From: Spencer Smith <robertspencersmith@gmail.com>
Date: Mon, 17 Apr 2017 12:24:24 -0400
Subject: [PATCH 5/5] remove stray spaces in templating

---
 .../master/templates/manifests/kube-scheduler.manifest.j2       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
index d21db5470..694450ce7 100644
--- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2
@@ -29,7 +29,7 @@ spec:
     - --v={{ kube_log_level }}
 {% if scheduler_custom_flags is string %}
     - {{ scheduler_custom_flags }}
-{% else % }
+{% else %}
 {%   for flag in scheduler_custom_flags %}
     - {{ flag }}
 {%   endfor %}