diff --git a/inventory/sample/group_vars/k8s-cluster/addons.yml b/inventory/sample/group_vars/k8s-cluster/addons.yml index 01699138b..ca801d3cb 100644 --- a/inventory/sample/group_vars/k8s-cluster/addons.yml +++ b/inventory/sample/group_vars/k8s-cluster/addons.yml @@ -11,6 +11,12 @@ registry_enabled: false # registry_storage_class: "" # registry_disk_size: "10Gi" +# Metrics Server deployment +metrics_server_enabled: false +# metrics_server_kubelet_insecure_tls: true +# metrics_server_metric_resolution: 60s +# metrics_server_kubelet_preferred_address_types: "InternalIP" + # Local volume provisioner deployment local_volume_provisioner_enabled: false # local_volume_provisioner_namespace: kube-system diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 35e0d17de..6420a94de 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -198,6 +198,9 @@ registry_image_repo: "registry" registry_image_tag: "2.6" registry_proxy_image_repo: "gcr.io/google_containers/kube-registry-proxy" registry_proxy_image_tag: "0.4" +metrics_server_version: "v0.3.1" +metrics_server_image_repo: "k8s.gcr.io/metrics-server-amd64" +metrics_server_image_tag: "{{ metrics_server_version }}" local_volume_provisioner_image_repo: "quay.io/external_storage/local-volume-provisioner" local_volume_provisioner_image_tag: "v2.1.0" cephfs_provisioner_image_repo: "quay.io/external_storage/cephfs-provisioner" @@ -209,6 +212,9 @@ ingress_nginx_default_backend_image_tag: "1.5" cert_manager_version: "v0.5.2" cert_manager_controller_image_repo: "quay.io/jetstack/cert-manager-controller" cert_manager_controller_image_tag: "{{ cert_manager_version }}" +addon_resizer_version: "1.8.3" +addon_resizer_image_repo: "k8s.gcr.io/addon-resizer" +addon_resizer_image_tag: "{{ addon_resizer_version }}" downloads: netcheck_server: @@ -548,6 +554,25 @@ downloads: groups: - kube-node + metrics_server: + enabled: "{{ metrics_server_enabled }}" + container: true + repo: "{{ metrics_server_image_repo }}" + tag: "{{ metrics_server_image_tag }}" + sha256: "{{ metrics_server_digest_checksum|default(None) }}" + groups: + - kube-master + + addon_resizer: + # Currently addon_resizer is only used by metrics server + enabled: "{{ metrics_server_enabled }}" + container: true + repo: "{{ addon_resizer_image_repo }}" + tag: "{{ addon_resizer_image_tag }}" + sha256: "{{ addon_resizer_digest_checksum|default(None) }}" + groups: + - kube-master + local_volume_provisioner: enabled: "{{ local_volume_provisioner_enabled }}" container: true diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index 52ddee1bc..13cf0af89 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -21,6 +21,14 @@ dependencies: - apps - registry + - role: kubernetes-apps/metrics_server + when: + - metrics_server_enabled + - inventory_hostname == groups['kube-master'][0] + tags: + - apps + - metrics_server + - role: kubernetes-apps/persistent_volumes when: - persistent_volumes_enabled diff --git a/roles/kubernetes-apps/metrics_server/defaults/main.yml b/roles/kubernetes-apps/metrics_server/defaults/main.yml new file mode 100644 index 000000000..94dcfc0d9 --- /dev/null +++ b/roles/kubernetes-apps/metrics_server/defaults/main.yml @@ -0,0 +1,12 @@ +--- +metrics_server_kubelet_insecure_tls: true +metrics_server_kubelet_preferred_address_types: "InternalIP" +metrics_server_metric_resolution: 60s +metrics_server_cpu: 40m +metrics_server_memory: 35Mi +metrics_server_memory_per_node: 4Mi +metrics_server_min_cluster_size: 5 +addon_resizer_limits_cpu: 100m +addon_resizer_limits_memory: 300Mi +addon_resizer_requests_cpu: 5m +addon_resizer_requests_memory: 50Mi diff --git a/roles/kubernetes-apps/metrics_server/tasks/main.yml b/roles/kubernetes-apps/metrics_server/tasks/main.yml new file mode 100644 index 000000000..d7dc45443 --- /dev/null +++ b/roles/kubernetes-apps/metrics_server/tasks/main.yml @@ -0,0 +1,57 @@ +--- +# If all masters have node role, there are no tainted master and toleration should not be specified. +- name: Check all masters are node or not + set_fact: + masters_are_not_tainted: "{{ groups['kube-node'] | intersect(groups['kube-master']) == groups['kube-master'] }}" + +- name: Metrics Server | Delete addon dir + file: + path: "{{ kube_config_dir }}/addons/metrics_server" + state: absent + when: + - inventory_hostname == groups['kube-master'][0] + tags: + - upgrade + +- name: Metrics Server | Create addon dir + file: + path: "{{ kube_config_dir }}/addons/metrics_server" + state: directory + owner: root + group: root + mode: 0755 + when: + - inventory_hostname == groups['kube-master'][0] + +- name: Metrics Server | Templates list + set_fact: + metrics_server_templates: + - { name: auth-delegator, file: auth-delegator.yaml, type: clusterrolebinding } + - { name: auth-reader, file: auth-reader.yaml, type: rolebinding } + - { name: metrics-server-cm, file: metrics-server-cm.yaml, type: cm } + - { name: metrics-server-sa, file: metrics-server-sa.yaml, type: sa } + - { name: metrics-server-deployment, file: metrics-server-deployment.yaml, type: deploy } + - { name: metrics-server-service, file: metrics-server-service.yaml, type: service } + - { name: metrics-apiservice, file: metrics-apiservice.yaml, type: service } + - { name: resource-reader-clusterrolebinding, file: resource-reader-clusterrolebinding.yaml, type: clusterrolebinding } + - { name: resource-reader, file: resource-reader.yaml, type: clusterrole } + +- name: Metrics Server | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/metrics_server/{{ item.file }}" + with_items: "{{ metrics_server_templates }}" + register: metrics_server_manifests + when: + - inventory_hostname == groups['kube-master'][0] + +- name: Metrics Server | Apply manifests + kube: + name: "{{ item.item.name }}" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/metrics_server/{{ item.item.file }}" + state: "latest" + with_items: "{{ metrics_server_manifests.results }}" + when: + - inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/metrics_server/templates/auth-delegator.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/auth-delegator.yaml.j2 new file mode 100644 index 000000000..cbaa62506 --- /dev/null +++ b/roles/kubernetes-apps/metrics_server/templates/auth-delegator.yaml.j2 @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-server:system:auth-delegator + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system diff --git a/roles/kubernetes-apps/metrics_server/templates/auth-reader.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/auth-reader.yaml.j2 new file mode 100644 index 000000000..60da052c1 --- /dev/null +++ b/roles/kubernetes-apps/metrics_server/templates/auth-reader.yaml.j2 @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metrics-server-auth-reader + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system diff --git a/roles/kubernetes-apps/metrics_server/templates/metrics-apiservice.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/metrics-apiservice.yaml.j2 new file mode 100644 index 000000000..51770da68 --- /dev/null +++ b/roles/kubernetes-apps/metrics_server/templates/metrics-apiservice.yaml.j2 @@ -0,0 +1,16 @@ +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + name: v1beta1.metrics.k8s.io + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + service: + name: metrics-server + namespace: kube-system + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ metrics_server_kubelet_insecure_tls }} + groupPriorityMinimum: 100 + versionPriority: 100 diff --git a/roles/kubernetes-apps/metrics_server/templates/metrics-server-cm.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/metrics-server-cm.yaml.j2 new file mode 100644 index 000000000..f969ff759 --- /dev/null +++ b/roles/kubernetes-apps/metrics_server/templates/metrics-server-cm.yaml.j2 @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metrics-server-config + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: EnsureExists +data: + NannyConfiguration: |- + apiVersion: nannyconfig/v1alpha1 + kind: NannyConfiguration diff --git a/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 new file mode 100644 index 000000000..6cb51d025 --- /dev/null +++ b/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 @@ -0,0 +1,134 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: metrics-server + namespace: kube-system + labels: + app.kubernetes.io/name: metrics-server + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + version: {{ metrics_server_version }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: metrics-server + version: {{ metrics_server_version }} + template: + metadata: + name: metrics-server + labels: + app.kubernetes.io/name: metrics-server + version: {{ metrics_server_version }} + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + seccomp.security.alpha.kubernetes.io/pod: 'docker/default' + spec: +{% if kube_version is version('v1.11.1', '>=') %} + priorityClassName: system-cluster-critical +{% endif %} + serviceAccountName: metrics-server + containers: + - name: metrics-server + image: {{ metrics_server_image_repo }}:{{ metrics_server_image_tag }} + command: + - /metrics-server +{% if metrics_server_kubelet_preferred_address_types %} + - --kubelet-preferred-address-types={{ metrics_server_kubelet_preferred_address_types }} +{% endif %} +{% if metrics_server_kubelet_insecure_tls %} + - --kubelet-insecure-tls +{% endif %} + - --metric-resolution={{ metrics_server_metric_resolution }} + ports: + - containerPort: 443 + name: https + protocol: TCP + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 10 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 443 + scheme: HTTPS + initialDelaySeconds: 30 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 10 + securityContext: + # Currently non root is not supported: + # https://github.com/kubernetes-incubator/metrics-server/issues/37 + # + # runAsNonRoot: true + # runAsUser: 65534 + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + - name: metrics-server-nanny + image: {{ addon_resizer_image_repo }}:{{ addon_resizer_image_tag }} + resources: + limits: + cpu: {{ addon_resizer_limits_cpu }} + memory: {{ addon_resizer_limits_memory }} + requests: + cpu: {{ addon_resizer_requests_cpu }} + memory: {{ addon_resizer_requests_memory }} + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: metrics-server-config-volume + mountPath: /etc/config + command: + - /pod_nanny + - --config-dir=/etc/config + - --cpu={{ metrics_server_cpu }} + - --extra-cpu=0.5m + - --memory={{ metrics_server_memory }} + - --extra-memory={{ metrics_server_memory_per_node }} + - --threshold=5 + - --deployment=metrics-server-{{ metrics_server_version }} + - --container=metrics-server + - --poll-period=300000 + - --estimator=exponential + # Specifies the smallest cluster (defined in number of nodes) + # resources will be scaled to. + - --minClusterSize={{ metrics_server_min_cluster_size }} + volumes: + - name: metrics-server-config-volume + configMap: + name: metrics-server-config +{% if not masters_are_not_tainted %} + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: "CriticalAddonsOnly" + operator: "Exists" +{% endif %} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" diff --git a/roles/kubernetes-apps/metrics_server/templates/metrics-server-sa.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/metrics-server-sa.yaml.j2 new file mode 100644 index 000000000..fa79edca4 --- /dev/null +++ b/roles/kubernetes-apps/metrics_server/templates/metrics-server-sa.yaml.j2 @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metrics-server + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile diff --git a/roles/kubernetes-apps/metrics_server/templates/metrics-server-service.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/metrics-server-service.yaml.j2 new file mode 100644 index 000000000..aa01cd645 --- /dev/null +++ b/roles/kubernetes-apps/metrics_server/templates/metrics-server-service.yaml.j2 @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: metrics-server + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/cluster-service: "true" + app.kubernetes.io/name: "metrics-server" +spec: + selector: + app.kubernetes.io/name: metrics-server + ports: + - port: 443 + protocol: TCP + targetPort: https diff --git a/roles/kubernetes-apps/metrics_server/templates/resource-reader-clusterrolebinding.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/resource-reader-clusterrolebinding.yaml.j2 new file mode 100644 index 000000000..0e59d5ed0 --- /dev/null +++ b/roles/kubernetes-apps/metrics_server/templates/resource-reader-clusterrolebinding.yaml.j2 @@ -0,0 +1,16 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:metrics-server + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system diff --git a/roles/kubernetes-apps/metrics_server/templates/resource-reader.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/resource-reader.yaml.j2 new file mode 100644 index 000000000..07db929c5 --- /dev/null +++ b/roles/kubernetes-apps/metrics_server/templates/resource-reader.yaml.j2 @@ -0,0 +1,27 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:metrics-server + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "extensions" + resources: + - deployments + verbs: + - get + - list + - update + - watch diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index c9c8c5575..9039d4c6c 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -252,6 +252,7 @@ dashboard_enabled: true # Addons which can be enabled helm_enabled: false registry_enabled: false +metrics_server_enabled: false enable_network_policy: true local_volume_provisioner_enabled: "{{ local_volumes_enabled | default('false') }}" persistent_volumes_enabled: false diff --git a/tests/files/gce_centos7-flannel-addons.yml b/tests/files/gce_centos7-flannel-addons.yml index 0a8712a91..3847fbc91 100644 --- a/tests/files/gce_centos7-flannel-addons.yml +++ b/tests/files/gce_centos7-flannel-addons.yml @@ -17,3 +17,4 @@ cloud_provider: gce kube_encrypt_secret_data: true ingress_nginx_enabled: true cert_manager_enabled: true +metrics_server_enabled: true