Merge pull request #1093 from mattymo/scaledns

Add autoscalers for dnsmasq and kubedns

roles/dnsmasq/defaults/main.yml

@@ -25,4 +25,8 @@ dnsmasq_image_tag: "{{ dnsmasq_version }}"
 dns_cpu_limit: 100m
 dns_memory_limit: 170Mi
 dns_cpu_requests: 70m
-dns_memory_requests: 70Mi
+dns_memory_requests: 50Mi
+
+# Autoscaler parameters
+dnsmasq_nodes_per_replica: 10
+dnsmasq_min_replicas: 1

roles/dnsmasq/tasks/main.yml

@@ -1,4 +1,6 @@
 ---
+- include: pre_upgrade.yml
+
 - name: ensure dnsmasq.d directory exists
   file:
     path: /etc/dnsmasq.d
@@ -54,26 +56,15 @@
     src: "{{item.file}}"
     dest: "{{kube_config_dir}}/{{item.file}}"
   with_items:
-    - {file: dnsmasq-ds.yml, type: ds}
+    - {name: dnsmasq, file: dnsmasq-deploy.yml, type: deployment}
-    - {file: dnsmasq-svc.yml, type: svc}
+    - {name: dnsmasq, file: dnsmasq-svc.yml, type: svc}
+    - {name: dnsmasq-autoscaler, file: dnsmasq-autoscaler.yml, type: deployment}
   register: manifests
   when: inventory_hostname == groups['kube-master'][0]
 
-#FIXME: remove manifests.changed condition if kubernetes/features#124 is implemented
-- name: Delete existing dnsmasq daemonset
-  kube:
-    name: dnsmasq
-    namespace: "{{system_namespace}}"
-    kubectl: "{{bin_dir}}/kubectl"
-    resource: "ds"
-    filename: "{{kube_config_dir}}/{{item.item.file}}"
-    state: absent
-  with_items: "{{ manifests.results }}"
-  when: inventory_hostname == groups['kube-master'][0] and item.item.type == "ds" and (manifests.changed or dnsmasq_config.changed)
-
 - name: Start Resources
   kube:
-    name: dnsmasq
+    name: "{{item.item.name}}"
     namespace: "{{system_namespace}}"
     kubectl: "{{bin_dir}}/kubectl"
     resource: "{{item.item.type}}"
@@ -88,3 +79,4 @@
     port: 53
     delay: 5
   when: inventory_hostname == groups['kube-node'][0]
+

roles/dnsmasq/tasks/pre_upgrade.yml

@@ -0,0 +1,9 @@
+---
+- name: Delete legacy dnsmasq daemonset
+  kube:
+    name: dnsmasq
+    namespace: "{{system_namespace}}"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "ds"
+    state: absent
+  when: inventory_hostname == groups['kube-master'][0]

roles/dnsmasq/templates/dnsmasq-autoscaler.yml

@@ -0,0 +1,50 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: dnsmasq-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-app: dnsmasq-autoscaler
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: dnsmasq-autoscaler
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
+    spec:
+      containers:
+      - name: autoscaler
+        image: gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.1.1
+        resources:
+            requests:
+                cpu: "20m"
+                memory: "10Mi"
+        command:
+          - /cluster-proportional-autoscaler
+          - --namespace=kube-system
+          - --configmap=dnsmasq-autoscaler
+          - --target=ReplicationController/dnsmasq
+          # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate.
+          # If using small nodes, "nodesPerReplica" should dominate.
+          - --default-params={"linear":{"nodesPerReplica":{{ dnsmasq_nodes_per_replica }},"preventSinglePointFailure":true}}
+          - --logtostderr=true
+          - --v={{ kube_log_level }}
+

roles/dnsmasq/templates/dnsmasq-deploy.yml

@@ -1,16 +1,22 @@
 ---
 apiVersion: extensions/v1beta1
-kind: DaemonSet
+kind: Deployment
 metadata:
   name: dnsmasq
   namespace: "{{system_namespace}}"
   labels:
     k8s-app: dnsmasq
+    kubernetes.io/cluster-service: "true"
 spec:
+  replicas: {{ dnsmasq_min_replicas }}
+  selector:
+    matchLabels:
+      k8s-app: dnsmasq
   template:
     metadata:
       labels:
         k8s-app: dnsmasq
+        kubernetes.io/cluster-service: "true"
     spec:
       containers:
         - name: dnsmasq
@@ -55,3 +61,4 @@ spec:
           hostPath:
             path: /etc/dnsmasq.d-available
       dnsPolicy: Default  # Don't use cluster DNS.
+

roles/kubernetes-apps/ansible/defaults/main.yml

@@ -7,8 +7,9 @@ exechealthz_version: 1.1
 dns_cpu_limit: 100m
 dns_memory_limit: 170Mi
 dns_cpu_requests: 70m
-dns_memory_requests: 70Mi
+dns_memory_requests: 50Mi
-dns_replicas: 1
+kubedns_min_replicas: 1
+kubedns_nodes_per_replica: 10
 
 # Images
 kubedns_image_repo: "gcr.io/google_containers/kubedns-amd64"

roles/kubernetes-apps/ansible/tasks/main.yml

@@ -13,15 +13,16 @@
     src: "{{item.file}}"
     dest: "{{kube_config_dir}}/{{item.file}}"
   with_items:
-    - {file: kubedns-rc.yml, type: rc}
+    - {name: kubedns, file: kubedns-deploy.yml, type: deployment}
-    - {file: kubedns-svc.yml, type: svc}
+    - {name: kubedns, file: kubedns-svc.yml, type: svc}
+    - {name: kubedns-autoscaler, file: kubedns-autoscaler.yml, type: deployment}
   register: manifests
   when: dns_mode != 'none' and inventory_hostname == groups['kube-master'][0]
   tags: dnsmasq
 
 - name: Kubernetes Apps | Start Resources
   kube:
-    name: kubedns
+    name: "{{item.item.name}}"
     namespace: "{{ system_namespace }}"
     kubectl: "{{bin_dir}}/kubectl"
     resource: "{{item.item.type}}"

roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml

@@ -0,0 +1,49 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: kubedns-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-app: kubedns-autoscaler
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: kubedns-autoscaler
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
+    spec:
+      containers:
+      - name: autoscaler
+        image: gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.1.1
+        resources:
+            requests:
+                cpu: "20m"
+                memory: "10Mi"
+        command:
+          - /cluster-proportional-autoscaler
+          - --namespace=kube-system
+          - --configmap=kubedns-autoscaler
+          # Should keep target in sync with cluster/addons/dns/kubedns-controller.yaml.base
+          - --target=replicationcontroller/kubedns
+          - --default-params={"linear":{"nodesPerReplica":{{ kubedns_nodes_per_replica }},"min":{{ kubedns_min_replicas }}}}
+          - --logtostderr=true
+          - --v=2
+

roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml

@@ -1,5 +1,5 @@
-apiVersion: v1
+apiVersion: extensions/v1beta1
-kind: ReplicationController
+kind: Deployment
 metadata:
   name: kubedns
   namespace: {{ system_namespace }}
@@ -8,10 +8,11 @@ metadata:
     version: v19
     kubernetes.io/cluster-service: "true"
 spec:
-  replicas: {{ dns_replicas }}
+  replicas: {{ kubedns_min_replicas }}
   selector:
-    k8s-app: kubedns
+    matchLabels:
-    version: v19
+      k8s-app: kubedns
+      version: v19
   template:
     metadata:
       labels: