Adding support for Prometheus

roles/download/defaults/main.yml

@@ -91,6 +91,20 @@ fluentd_image_tag: "{{ fluentd_version }}"
 kibana_version: "v4.6.1"
 kibana_image_repo: "gcr.io/google_containers/kibana"
 kibana_image_tag: "{{ kibana_version }}"
+prometheus_operator_image_repo: "quay.io/coreos/prometheus-operator"
+prometheus_operator_image_tag: "v0.3.0"
+node_exporter_image_repo: "quay.io/prometheus/node-exporter"
+node_exporter_image_tag: "v0.13.0"
+kube_state_metrics_image_repo: "gcr.io/google_containers/kube-state-metrics"
+kube_state_metrics_image_tag: "v0.3.0"
+alertmanager_image_repo: "quay.io/prometheus/alertmanager"
+alertmanager_image_tag: "v0.5.1"
+grafana_image_repo: "grafana/grafana"
+grafana_image_tag: "3.1.1"
+grafana_watcher_image_repo: "quay.io/coreos/grafana-watcher"
+grafana_watcher_image_tag: "latest"
+prometheus_image_repo: "quay.io/prometheus/prometheus"
+prometheus_image_tag: "v1.5.1"
 
 downloads:
   netcheck_server:
@@ -221,6 +235,42 @@ downloads:
     repo: "{{ kibana_image_repo }}"
     tag: "{{ kibana_image_tag }}"
     sha256: "{{ kibana_digest_checksum|default(None) }}"
+  prometheus_operator:
+    container: true
+    repo: "{{ prometheus_operator_image_repo }}"
+    tag: "{{ prometheus_operator_image_tag }}"
+    sha256: "{{ prometheus_operator_digest_checksum|default(None) }}"
+  node_exporter:
+    container: true
+    repo: "{{ node_exporter_image_repo }}"
+    tag: "{{ node_exporter_image_tag }}"
+    sha256: "{{ node_exporter_digest_checksum|default(None) }}"
+  kube_state_metrics:
+    container: true
+    repo: "{{ kube_state_metrics_image_repo }}"
+    tag: "{{ kube_state_metrics_image_tag }}"
+    sha256: "{{ kube_state_metrics_digest_checksum|default(None) }}"
+  alertmanager:
+    container: true
+    repo: "{{ alertmanager_image_repo }}"
+    tag: "{{ alertmanager_image_tag }}"
+    sha256: "{{ alertmanager_digest_checksum|default(None) }}"
+  grafana:
+    container: true
+    repo: "{{ grafana_image_repo }}"
+    tag: "{{ grafana_image_tag }}"
+    sha256: "{{ grafana_digest_checksum|default(None) }}"
+  grafana_watcher:
+    container: true
+    repo: "{{ grafana_watcher_image_repo }}"
+    tag: "{{ grafana_watcher_image_tag }}"
+    sha256: "{{ grafana_watcher_digest_checksum|default(None) }}"
+  prometheus:
+    container: true
+    repo: "{{ prometheus_image_repo }}"
+    tag: "{{ prometheus_image_tag }}"
+    sha256: "{{ prometheus_digest_checksum|default(None) }}"
+  
 
 download:
   container: "{{ file.container|default('false') }}"

roles/kubernetes-apps/meta/main.yml

@@ -17,3 +17,6 @@ dependencies:
   - role: kubernetes-apps/efk
     when: efk_enabled
     tags: [ apps, efk ]
+  - role: kubernetes-apps/prometheus
+    when: prometheus_enabled
+    tags: [ apps, prometheus ]

roles/kubernetes-apps/prometheus/defaults/main.yml

@@ -0,0 +1,79 @@
+---
+prometheus_namespace: monitoring
+
+prometheus_resources:
+  - name: prometheus-operator
+    type: deployment
+  - name: kube-state-metrics
+    type: deployment
+  - name: kube-state-metrics
+    type: service
+  - name: node-exporter
+    type: daemonset
+  - name: node-exporter
+    type: service
+  - name: grafana-dashboards
+    type: configmap
+  - name: grafana
+    type: deployment
+  - name: grafana
+    type: service
+  - name: prometheus-k8s-rules
+    type: configmap
+  - name: prometheus-k8s
+    type: service
+  - name: alertmanager-main
+    type: configmap
+  - name: alertmanager-main
+    type: service
+  - name: k8s-apps-https
+    type: servicemonitor
+  - name: k8s-apps-http
+    type: servicemonitor
+  - name: k8s
+    type: prometheus
+  - name: main
+    type: alertmanager
+
+prometheus_operator_cpu_limit: 200m 
+prometheus_operator_mem_limit: 300M
+prometheus_operator_cpu_requests: 100m
+prometheus_operator_mem_requests: 50M
+
+node_exporter_ds_cpu_limit: 200m
+node_exporter_ds_mem_limit: 50Mi
+node_exporter_ds_cpu_requests: 100m
+node_exporter_ds_mem_requests: 30Mi
+
+kube_state_metrics_cpu_limit: 200m
+kube_state_metrics_mem_limit: 50Mi
+kube_state_metrics_cpu_requests: 100m
+kube_state_metrics_mem_requests: 30Mi
+
+alertmanager_config_receivers:
+  - name: webhook
+    webhook_configs:
+      - url: "http://alertmanagerwh:30500/"
+alertmanager_config_global:
+  resolve_timeout: 5m
+alertmanager_config_route:
+  group_by:
+    - job
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 12h
+  receiver: webhook 
+
+alertmanager_replicas: 3
+
+grafana_cpu_limit: 300m
+grafana_mem_limit: 300Mi
+grafana_cpu_requests: 100m
+grafana_mem_requests: 100Mi
+
+grafana_watcher_cpu_limit: 100m
+grafana_watcher_mem_limit: 32Mi
+grafana_watcher_cpu_requests: 50m
+grafana_watcher_mem_requests: 16Mi
+
+prometheus_mem_requests: 400Mi

roles/kubernetes-apps/prometheus/meta/main.yml

@@ -0,0 +1,16 @@
+---
+dependencies:
+  - role: download
+    file: "{{ downloads.prometheus_operator }}"
+  - role: download
+    file: "{{ downloads.prometheus }}"
+  - role: download
+    file: "{{ downloads.node_exporter }}"
+  - role: download
+    file: "{{ downloads.kube_state_metrics }}"
+  - role: download
+    file: "{{ downloads.alertmanager }}"
+  - role: download
+    file: "{{ downloads.grafana }}"
+  - role: download
+    file: "{{ downloads.grafana_watcher }}"

roles/kubernetes-apps/prometheus/tasks/main.yml

@@ -0,0 +1,61 @@
+---
+- name: "Prometheus | Check is Prometheus namespace exists"
+  command: "{{bin_dir}}/kubectl get ns {{ prometheus_namespace }}"
+  register: prom_ns_check
+  changed_when: False
+  failed_when: False
+  run_once: true
+
+- name: "Prometheus | Create Prometheus namespace"
+  command: "{{bin_dir}}/kubectl create namespace {{ prometheus_namespace }}"
+  changed_when: False
+  when: prom_ns_check|failed
+  run_once: true
+
+- name: "Prometheus | Create apps directory"
+  file:
+    path: "{{ kube_config_dir }}/apps"
+    state: directory
+
+- name: "Prometheus | Write prometheus manifests"
+  template:
+    src: "{{ item.name }}-{{ item.type }}.yml.j2"
+    dest: "{{ kube_config_dir }}/apps/{{ item.name }}-{{ item.type }}.yml"
+  register: prometheus_templates
+  with_items:
+    - "{{ prometheus_resources }}"
+
+- name: "Prometheus | Create Prometheus Operator deployment"
+  kube:
+    filename: "{{ kube_config_dir }}/apps/{{ item.item.name }}-{{ item.item.type }}.yml"
+    kubectl: "{{bin_dir}}/kubectl"
+    name: "{{ item.item.name }}"
+    namespace: "{{ prometheus_namespace }}"
+    resource: "{{ item.item.type }}"
+    state: "{{ item.changed | ternary('latest','present') }}"
+  when: item.item.name == 'prometheus-operator'
+  with_items: "{{ prometheus_templates.results }}"
+  run_once: true
+
+- name: "Prometheus | Wait for TPRs to become available"
+  command: "{{ bin_dir }}/kubectl get {{ item }} --namespace={{ prometheus_namespace }}"
+  register: tpr_available
+  until: tpr_available.rc == 0
+  retries: 10
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - servicemonitor
+    - prometheus
+    - alertmanager
+
+- name: "Prometheus | Create prometheus resources"
+  kube:
+    filename: "{{ kube_config_dir }}/apps/{{ item.item.name }}-{{ item.item.type }}.yml"
+    kubectl: "{{bin_dir}}/kubectl"
+    name: "{{ item.item.name }}"
+    namespace: "{{ prometheus_namespace }}"
+    resource: "{{ item.item.type }}"
+    state: "{{ item.changed | ternary('latest','present') }}"
+  when: item.item.name != 'prometheus-operator'
+  with_items: "{{ prometheus_templates.results }}"
+  run_once: true

roles/kubernetes-apps/prometheus/templates/alertmanager-main-configmap.yml.j2

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alertmanager-main
+  creationTimestamp: null
+data:
+  alertmanager.yaml: |-
+    global: 
+      {{ alertmanager_config_global | to_nice_yaml | indent(6, False) }}
+    route: 
+      {{ alertmanager_config_route | to_nice_yaml | indent(6, False)}}
+    receivers:
+      {{ alertmanager_config_receivers | to_nice_yaml | indent(6, False) }}

roles/kubernetes-apps/prometheus/templates/alertmanager-main-service.yml.j2

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: alertmanager-main
+spec:
+  type: NodePort
+  ports:
+  - name: web
+    nodePort: 30903
+    port: 9093
+    protocol: TCP
+    targetPort: web
+  selector:
+    alertmanager: main

roles/kubernetes-apps/prometheus/templates/grafana-deployment.yml.j2

@@ -0,0 +1,58 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: grafana
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: grafana
+    spec:
+      containers:
+      - name: grafana
+        image: {{ grafana_image_repo }}:{{ grafana_image_tag }}
+        env:
+        - name: GF_AUTH_BASIC_ENABLED
+          value: "true"
+        - name: GF_AUTH_ANONYMOUS_ENABLED
+          value: "true"
+        volumeMounts:
+        - name: grafana-storage
+          mountPath: /var/grafana-storage
+        ports:
+        - name: web
+          containerPort: 3000
+        resources:
+          requests:
+            memory: {{ grafana_mem_requests }}
+            cpu: {{ grafana_cpu_requests }}
+          limits:
+            memory: {{ grafana_mem_limit }}
+            cpu: {{ grafana_cpu_limit }}
+      - name: grafana-watcher
+        image: {{ grafana_watcher_image_repo }}:{{ grafana_watcher_image_tag }}
+        args:
+          - '--watch-dir=/var/grafana-dashboards'
+          - '--grafana-url=http://admin:admin@localhost:3000'
+        volumeMounts:
+        - name: grafana-dashboards
+          mountPath: /var/grafana-dashboards
+        resources:
+          requests:
+            memory: {{ grafana_watcher_mem_requests }}
+            cpu: {{ grafana_watcher_cpu_requests }}
+          limits:
+            memory: {{ grafana_watcher_mem_limit }}
+            cpu: {{ grafana_watcher_cpu_limit }}
+        volumeMounts:
+        - name: grafana-dashboards
+          mountPath: /var/grafana-dashboards
+      volumes:
+      - name: grafana-storage
+        emptyDir: {}
+      - name: grafana-dashboards
+        configMap:
+          name: grafana-dashboards
+
+

roles/kubernetes-apps/prometheus/templates/grafana-service.yml.j2

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana
+  labels:
+    app: grafana
+spec:
+  type: NodePort
+  ports:
+  - name: web
+    port: 3000
+    protocol: TCP
+    nodePort: 30902
+  selector:
+    app: grafana

roles/kubernetes-apps/prometheus/templates/k8s-apps-http-servicemonitor.yml.j2

@@ -0,0 +1,23 @@
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ServiceMonitor
+metadata:
+  name: k8s-apps-http
+  labels:
+    k8s-apps: http
+spec:
+  jobLabel: k8s-app
+  selector:
+    matchExpressions:
+    - {key: k8s-app, operator: Exists}
+  namespaceSelector:
+    matchNames:
+    - kube-system
+    - monitoring
+  endpoints:
+  - port: http-metrics
+    interval: 15s
+  - port: http-metrics-dnsmasq
+    interval: 15s
+  - port: http-metrics-skydns
+    interval: 15s

roles/kubernetes-apps/prometheus/templates/k8s-apps-https-servicemonitor.yml.j2

@@ -0,0 +1,22 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: ServiceMonitor
+metadata:
+  name: k8s-apps-https
+  labels:
+    k8s-apps: https
+spec:
+  jobLabel: k8s-app
+  selector:
+    matchExpressions:
+    - {key: k8s-app, operator: Exists}
+  namespaceSelector:
+    matchNames:
+    - kube-system
+  endpoints:
+  - port: https-metrics
+    interval: 15s
+    scheme: https
+    tlsConfig:
+      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+      insecureSkipVerify: true
+      bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token

roles/kubernetes-apps/prometheus/templates/k8s-prometheus.yml.j2

@@ -0,0 +1,25 @@
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: Prometheus
+metadata:
+  name: k8s
+  labels:
+    prometheus: k8s
+spec:
+  version: "{{ prometheus_image_tag }}"
+  baseImage: "{{ prometheus_image_repo }}"
+  serviceMonitorSelector:
+    matchExpression:
+    - {key: k8s-apps, operator: Exists}
+  resources:
+    requests:
+      # 2Gi is default, but won't schedule if you don't have a node with >2Gi
+      # memory. Modify based on your target and time-series count for
+      # production use. This value is mainly meant for demonstration/testing
+      # purposes.
+      memory: {{ prometheus_mem_requests }}
+  alerting:
+    alertmanagers:
+    - namespace: monitoring
+      name: alertmanager-main
+      port: web
+

roles/kubernetes-apps/prometheus/templates/kube-state-metrics-deployment.yml.j2

@@ -0,0 +1,24 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: kube-state-metrics
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: kube-state-metrics
+    spec:
+      containers:
+      - name: kube-state-metrics
+        image: "{{ kube_state_metrics_image_repo }}:{{ kube_state_metrics_image_tag }}"
+        ports:
+        - name: metrics
+          containerPort: 8080
+        resources:
+          requests:
+            memory: {{ kube_state_metrics_mem_requests }}
+            cpu: {{ kube_state_metrics_cpu_requests }}
+          limits:
+            memory: {{ kube_state_metrics_mem_limit }}
+            cpu: {{ kube_state_metrics_cpu_limit }}

roles/kubernetes-apps/prometheus/templates/kube-state-metrics-service.yml.j2

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: kube-state-metrics
+    k8s-app: kube-state-metrics
+  name: kube-state-metrics
+spec:
+  ports:
+  - name: http-metrics
+    port: 8080
+    targetPort: metrics
+    protocol: TCP
+  selector:
+    app: kube-state-metrics

roles/kubernetes-apps/prometheus/templates/main-alertmanager.yml.j2

@@ -0,0 +1,10 @@
+apiVersion: "monitoring.coreos.com/v1alpha1"
+kind: "Alertmanager"
+metadata:
+  name: "main"
+  labels:
+    alertmanager: "main"
+spec:
+  replicas: {{ alertmanager_replicas }}
+  version: {{ alertmanager_image_tag }}
+  baseImage: {{ alertmanager_image_repo }}

roles/kubernetes-apps/prometheus/templates/node-exporter-daemonset.yml.j2

@@ -0,0 +1,44 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: node-exporter
+spec:
+  template:
+    metadata:
+      labels:
+        app: node-exporter
+      name: node-exporter
+    spec:
+      hostNetwork: true
+      hostPID: true
+      containers:
+      - image: "{{ node_exporter_image_repo }}:{{ node_exporter_image_tag }}"
+        args:
+        - "-collector.procfs=/host/proc"
+        - "-collector.sysfs=/host/sys"
+        name: node-exporter
+        ports:
+        - containerPort: 9100
+          hostPort: 9100
+          name: scrape
+        resources:
+          requests:
+            memory: {{ node_exporter_ds_mem_requests }}
+            cpu: {{ node_exporter_ds_cpu_requests }}
+          limits:
+            memory: {{ node_exporter_ds_mem_limit }}
+            cpu:  {{ node_exporter_ds_cpu_limit }}
+        volumeMounts:
+        - name: proc
+          readOnly:  true
+          mountPath: /host/proc
+        - name: sys
+          readOnly: true
+          mountPath: /host/sys
+      volumes:
+      - name: proc
+        hostPath:
+          path: /proc
+      - name: sys
+        hostPath:
+          path: /sys

roles/kubernetes-apps/prometheus/templates/node-exporter-service.yml.j2

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: node-exporter
+    k8s-app: node-exporter
+  name: node-exporter
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+  - name: http-metrics
+    port: 9100
+    protocol: TCP
+  selector:
+    app: node-exporter

roles/kubernetes-apps/prometheus/templates/prometheus-k8s-rules-configmap.yml.j2

@@ -0,0 +1,449 @@
+#jinja2:variable_start_string:'[[' , variable_end_string:']]', block_start_string:'[%' , block_end_string:'%]'
+---
+apiVersion: v1
+data:
+  etcd2.rules: "### General cluster availability ###\n\n# alert if another failed
+    peer will result in an unavailable cluster\nALERT InsufficientPeers\n  IF count(up{job=\"etcd-k8s\"}
+    == 0) > (count(up{job=\"etcd-k8s\"}) / 2 - 1)\n  FOR 3m\n  LABELS {\n    severity
+    = \"critical\"\n  }\n  ANNOTATIONS {\n    summary = \"Etcd cluster small\",\n
+    \   description = \"If one more etcd peer goes down the cluster will be unavailable\",\n
+    \ }\n\n### HTTP requests alerts ###\n\n# alert if more than 1% of requests to
+    an HTTP endpoint have failed with a non 4xx response\nALERT HighNumberOfFailedHTTPRequests\n
+    \ IF sum by(method) (rate(etcd_http_failed_total{job=\"etcd-k8s\", code!~\"4[0-9]{2}\"}[5m]))\n
+    \   / sum by(method) (rate(etcd_http_received_total{job=\"etcd-k8s\"}[5m])) >
+    0.01\n  FOR 10m\n  LABELS {\n    severity = \"warning\"\n  }\n  ANNOTATIONS {\n
+    \   summary = \"a high number of HTTP requests are failing\",\n    description
+    = \"{{ $value }}% of requests for {{ $labels.method }} failed on etcd instance
+    {{ $labels.instance }}\",\n  }\n\n# alert if more than 5% of requests to an HTTP
+    endpoint have failed with a non 4xx response\nALERT HighNumberOfFailedHTTPRequests\n
+    \ IF sum by(method) (rate(etcd_http_failed_total{job=\"etcd-k8s\", code!~\"4[0-9]{2}\"}[5m]))
+    \n    / sum by(method) (rate(etcd_http_received_total{job=\"etcd-k8s\"}[5m]))
+    > 0.05\n  FOR 5m\n  LABELS {\n    severity = \"critical\"\n  }\n  ANNOTATIONS
+    {\n    summary = \"a high number of HTTP requests are failing\",\n    description
+    = \"{{ $value }}% of requests for {{ $labels.method }} failed on etcd instance
+    {{ $labels.instance }}\",\n  }\n\n# alert if 50% of requests get a 4xx response\nALERT
+    HighNumberOfFailedHTTPRequests\n  IF sum by(method) (rate(etcd_http_failed_total{job=\"etcd-k8s\",
+    code=~\"4[0-9]{2}\"}[5m]))\n    / sum by(method) (rate(etcd_http_received_total{job=\"etcd-k8s\"}[5m]))
+    > 0.5\n  FOR 10m\n  LABELS {\n    severity = \"critical\"\n  }\n  ANNOTATIONS
+    {\n    summary = \"a high number of HTTP requests are failing\",\n    description
+    = \"{{ $value }}% of requests for {{ $labels.method }} failed with 4xx responses
+    on etcd instance {{ $labels.instance }}\",\n  }\n\n# alert if the 99th percentile
+    of HTTP requests take more than 150ms\nALERT HTTPRequestsSlow\n  IF histogram_quantile(0.99,
+    rate(etcd_http_successful_duration_second_bucket[5m])) > 0.15\n  FOR 10m\n  LABELS
+    {\n    severity = \"warning\"\n  }\n  ANNOTATIONS {\n    summary = \"slow HTTP
+    requests\",\n    description = \"on ectd instance {{ $labels.instance }} HTTP
+    requests to {{ $label.method }} are slow\",\n  }\n\n### File descriptor alerts
+    ###\n\ninstance:fd_utilization = process_open_fds / process_max_fds\n\n# alert
+    if file descriptors are likely to exhaust within the next 4 hours\nALERT FdExhaustionClose\n
+    \ IF predict_linear(instance:fd_utilization[1h], 3600 * 4) > 1\n  FOR 10m\n  LABELS
+    {\n    severity = \"warning\"\n  }\n  ANNOTATIONS {\n    summary = \"file descriptors
+    soon exhausted\",\n    description = \"{{ $labels.job }} instance {{ $labels.instance
+    }} will exhaust in file descriptors soon\",\n  }\n\n# alert if file descriptors
+    are likely to exhaust within the next hour\nALERT FdExhaustionClose\n  IF predict_linear(instance:fd_utilization[10m],
+    3600) > 1\n  FOR 10m\n  LABELS {\n    severity = \"critical\"\n  }\n  ANNOTATIONS
+    {\n    summary = \"file descriptors soon exhausted\",\n    description = \"{{
+    $labels.job }} instance {{ $labels.instance }} will exhaust in file descriptors
+    soon\",\n  }\n\n### etcd proposal alerts ###\n\n# alert if there are several failed
+    proposals within an hour\nALERT HighNumberOfFailedProposals\n  IF increase(etcd_server_proposal_failed_total{job=\"etcd\"}[1h])
+    > 5\n  LABELS {\n    severity = \"warning\"\n  }\n  ANNOTATIONS {\n    summary
+    = \"a high number of failed proposals within the etcd cluster are happening\",\n
+    \   description = \"etcd instance {{ $labels.instance }} has seen {{ $value }}
+    proposal failures within the last hour\",\n  }\n\n### etcd disk io latency alerts
+    ###\n\n# alert if 99th percentile of fsync durations is higher than 500ms\nALERT
+    HighFsyncDurations\n  IF histogram_quantile(0.99, rate(etcd_wal_fsync_durations_seconds_bucket[5m]))
+    > 0.5\n  FOR 10m\n  LABELS {\n    severity = \"warning\"\n  }\n  ANNOTATIONS {\n
+    \   summary = \"high fsync durations\",\n    description = \"ectd instance {{
+    $labels.instance }} fync durations are high\",\n  }\n"
+  kubernetes.rules: |+
+    # NOTE: These rules were kindly contributed by the SoundCloud engineering team.
+
+    ### Container resources ###
+
+    cluster_namespace_controller_pod_container:spec_memory_limit_bytes =
+      sum by (cluster,namespace,controller,pod_name,container_name) (
+        label_replace(
+          container_spec_memory_limit_bytes{container_name!=""},
+          "controller", "$1",
+          "pod_name", "^(.*)-[a-z0-9]+"
+        )
+      )
+
+    cluster_namespace_controller_pod_container:spec_cpu_shares =
+      sum by (cluster,namespace,controller,pod_name,container_name) (
+        label_replace(
+          container_spec_cpu_shares{container_name!=""},
+          "controller", "$1",
+          "pod_name", "^(.*)-[a-z0-9]+"
+        )
+      )
+
+    cluster_namespace_controller_pod_container:cpu_usage:rate =
+      sum by (cluster,namespace,controller,pod_name,container_name) (
+        label_replace(
+          irate(
+            container_cpu_usage_seconds_total{container_name!=""}[5m]
+          ),
+          "controller", "$1",
+          "pod_name", "^(.*)-[a-z0-9]+"
+        )
+      )
+
+    cluster_namespace_controller_pod_container:memory_usage:bytes =
+      sum by (cluster,namespace,controller,pod_name,container_name) (
+        label_replace(
+          container_memory_usage_bytes{container_name!=""},
+          "controller", "$1",
+          "pod_name", "^(.*)-[a-z0-9]+"
+        )
+      )
+
+    cluster_namespace_controller_pod_container:memory_working_set:bytes =
+      sum by (cluster,namespace,controller,pod_name,container_name) (
+        label_replace(
+          container_memory_working_set_bytes{container_name!=""},
+          "controller", "$1",
+          "pod_name", "^(.*)-[a-z0-9]+"
+        )
+      )
+
+    cluster_namespace_controller_pod_container:memory_rss:bytes =
+      sum by (cluster,namespace,controller,pod_name,container_name) (
+        label_replace(
+          container_memory_rss{container_name!=""},
+          "controller", "$1",
+          "pod_name", "^(.*)-[a-z0-9]+"
+        )
+      )
+
+    cluster_namespace_controller_pod_container:memory_cache:bytes =
+      sum by (cluster,namespace,controller,pod_name,container_name) (
+        label_replace(
+          container_memory_cache{container_name!=""},
+          "controller", "$1",
+          "pod_name", "^(.*)-[a-z0-9]+"
+        )
+      )
+
+    cluster_namespace_controller_pod_container:disk_usage:bytes =
+      sum by (cluster,namespace,controller,pod_name,container_name) (
+        label_replace(
+          container_disk_usage_bytes{container_name!=""},
+          "controller", "$1",
+          "pod_name", "^(.*)-[a-z0-9]+"
+        )
+      )
+
+    cluster_namespace_controller_pod_container:memory_pagefaults:rate =
+      sum by (cluster,namespace,controller,pod_name,container_name,scope,type) (
+        label_replace(
+          irate(
+            container_memory_failures_total{container_name!=""}[5m]
+          ),
+          "controller", "$1",
+          "pod_name", "^(.*)-[a-z0-9]+"
+        )
+      )
+
+    cluster_namespace_controller_pod_container:memory_oom:rate =
+      sum by (cluster,namespace,controller,pod_name,container_name,scope,type) (
+        label_replace(
+          irate(
+            container_memory_failcnt{container_name!=""}[5m]
+          ),
+          "controller", "$1",
+          "pod_name", "^(.*)-[a-z0-9]+"
+        )
+      )
+
+    ### Cluster resources ###
+
+    cluster:memory_allocation:percent =
+      100 * sum by (cluster) (
+        container_spec_memory_limit_bytes{pod_name!=""}
+      ) / sum by (cluster) (
+        machine_memory_bytes
+      )
+
+    cluster:memory_used:percent =
+      100 * sum by (cluster) (
+        container_memory_usage_bytes{pod_name!=""}
+      ) / sum by (cluster) (
+        machine_memory_bytes
+      )
+
+    cluster:cpu_allocation:percent =
+      100 * sum by (cluster) (
+        container_spec_cpu_shares{pod_name!=""}
+      ) / sum by (cluster) (
+        container_spec_cpu_shares{id="/"} * on(cluster,instance) machine_cpu_cores
+      )
+
+    cluster:node_cpu_use:percent =
+      100 * sum by (cluster) (
+        rate(node_cpu{mode!="idle"}[5m])
+      ) / sum by (cluster) (
+        machine_cpu_cores
+      )
+
+    ### API latency ###
+
+    # Raw metrics are in microseconds. Convert to seconds.
+    cluster_resource_verb:apiserver_latency:quantile_seconds{quantile="0.99"} =
+      histogram_quantile(
+        0.99,
+        sum by(le,cluster,job,resource,verb) (apiserver_request_latencies_bucket)
+      ) / 1e6
+    cluster_resource_verb:apiserver_latency:quantile_seconds{quantile="0.9"} =
+      histogram_quantile(
+        0.9,
+        sum by(le,cluster,job,resource,verb) (apiserver_request_latencies_bucket)
+      ) / 1e6
+    cluster_resource_verb:apiserver_latency:quantile_seconds{quantile="0.5"} =
+      histogram_quantile(
+        0.5,
+        sum by(le,cluster,job,resource,verb) (apiserver_request_latencies_bucket)
+      ) / 1e6
+
+    ### Scheduling latency ###
+
+    cluster:scheduler_e2e_scheduling_latency:quantile_seconds{quantile="0.99"} =
+      histogram_quantile(0.99,sum by (le,cluster) (scheduler_e2e_scheduling_latency_microseconds_bucket)) / 1e6
+    cluster:scheduler_e2e_scheduling_latency:quantile_seconds{quantile="0.9"} =
+      histogram_quantile(0.9,sum by (le,cluster) (scheduler_e2e_scheduling_latency_microseconds_bucket)) / 1e6
+    cluster:scheduler_e2e_scheduling_latency:quantile_seconds{quantile="0.5"} =
+      histogram_quantile(0.5,sum by (le,cluster) (scheduler_e2e_scheduling_latency_microseconds_bucket)) / 1e6
+
+    cluster:scheduler_scheduling_algorithm_latency:quantile_seconds{quantile="0.99"} =
+      histogram_quantile(0.99,sum by (le,cluster) (scheduler_scheduling_algorithm_latency_microseconds_bucket)) / 1e6
+    cluster:scheduler_scheduling_algorithm_latency:quantile_seconds{quantile="0.9"} =
+      histogram_quantile(0.9,sum by (le,cluster) (scheduler_scheduling_algorithm_latency_microseconds_bucket)) / 1e6
+    cluster:scheduler_scheduling_algorithm_latency:quantile_seconds{quantile="0.5"} =
+      histogram_quantile(0.5,sum by (le,cluster) (scheduler_scheduling_algorithm_latency_microseconds_bucket)) / 1e6
+
+    cluster:scheduler_binding_latency:quantile_seconds{quantile="0.99"} =
+      histogram_quantile(0.99,sum by (le,cluster) (scheduler_binding_latency_microseconds_bucket)) / 1e6
+    cluster:scheduler_binding_latency:quantile_seconds{quantile="0.9"} =
+      histogram_quantile(0.9,sum by (le,cluster) (scheduler_binding_latency_microseconds_bucket)) / 1e6
+    cluster:scheduler_binding_latency:quantile_seconds{quantile="0.5"} =
+      histogram_quantile(0.5,sum by (le,cluster) (scheduler_binding_latency_microseconds_bucket)) / 1e6
+
+    ALERT K8SNodeDown
+      IF up{job="kubelet"} == 0
+      FOR 1h
+      LABELS {
+        service = "k8s",
+        severity = "warning"
+      }
+      ANNOTATIONS {
+        summary = "Kubelet cannot be scraped",
+        description = "Prometheus could not scrape a {{ $labels.job }} for more than one hour",
+      }
+
+    ALERT K8SNodeNotReady
+      IF kube_node_status_ready{condition="true"} == 0
+      FOR 1h
+      LABELS {
+        service = "k8s",
+        severity = "warning",
+      }
+      ANNOTATIONS {
+        summary = "Node status is NotReady",
+        description = "The Kubelet on {{ $labels.node }} has not checked in with the API, or has set itself to NotReady, for more than an hour",
+      }
+
+    ALERT K8SManyNodesNotReady
+      IF
+        count by (cluster) (kube_node_status_ready{condition="true"} == 0) > 1
+        AND
+          (
+            count by (cluster) (kube_node_status_ready{condition="true"} == 0)
+          /
+            count by (cluster) (kube_node_status_ready{condition="true"})
+          ) > 0.2
+      FOR 1m
+      LABELS {
+        service = "k8s",
+        severity = "critical",
+      }
+      ANNOTATIONS {
+        summary = "Many K8s nodes are Not Ready",
+        description = "{{ $value }} K8s nodes (more than 10% of cluster {{ $labels.cluster }}) are in the NotReady state.",
+      }
+
+    ALERT K8SKubeletNodeExporterDown
+      IF up{job="node-exporter"} == 0
+      FOR 15m
+      LABELS {
+        service = "k8s",
+        severity = "warning"
+      }
+      ANNOTATIONS {
+        summary = "Kubelet node_exporter cannot be scraped",
+        description = "Prometheus could not scrape a {{ $labels.job }} for more than one hour.",
+      }
+
+    ALERT K8SKubeletDown
+      IF absent(up{job="kubelet"}) or count by (cluster) (up{job="kubelet"} == 0) / count by (cluster) (up{job="kubelet"}) > 0.1
+      FOR 1h
+      LABELS {
+        service = "k8s",
+        severity = "critical"
+      }
+      ANNOTATIONS {
+        summary = "Many Kubelets cannot be scraped",
+        description = "Prometheus failed to scrape more than 10% of kubelets, or all Kubelets have disappeared from service discovery.",
+      }
+
+    ALERT K8SApiserverDown
+      IF up{job="kubernetes"} == 0
+      FOR 15m
+      LABELS {
+        service = "k8s",
+        severity = "warning"
+      }
+      ANNOTATIONS {
+        summary = "API server unreachable",
+        description = "An API server could not be scraped.",
+      }
+
+    # Disable for non HA kubernetes setups.
+    ALERT K8SApiserverDown
+      IF absent({job="kubernetes"}) or (count by(cluster) (up{job="kubernetes"} == 1) < count by(cluster) (up{job="kubernetes"}))
+      FOR 5m
+      LABELS {
+        service = "k8s",
+        severity = "critical"
+      }
+      ANNOTATIONS {
+        summary = "API server unreachable",
+        description = "Prometheus failed to scrape multiple API servers, or all API servers have disappeared from service discovery.",
+      }
+
+    ALERT K8SSchedulerDown
+      IF absent(up{job="kube-scheduler"}) or (count by(cluster) (up{job="kube-scheduler"} == 1) == 0)
+      FOR 5m
+      LABELS {
+        service = "k8s",
+        severity = "critical",
+      }
+      ANNOTATIONS {
+        summary = "Scheduler is down",
+        description = "There is no running K8S scheduler. New pods are not being assigned to nodes.",
+      }
+
+    ALERT K8SControllerManagerDown
+      IF absent(up{job="kube-controller-manager"}) or (count by(cluster) (up{job="kube-controller-manager"} == 1) == 0)
+      FOR 5m
+      LABELS {
+        service = "k8s",
+        severity = "critical",
+      }
+      ANNOTATIONS {
+        summary = "Controller manager is down",
+        description = "There is no running K8S controller manager. Deployments and replication controllers are not making progress.",
+      }
+
+    ALERT K8SConntrackTableFull
+      IF 100*node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 50
+      FOR 10m
+      LABELS {
+        service = "k8s",
+        severity = "warning"
+      }
+      ANNOTATIONS {
+        summary = "Number of tracked connections is near the limit",
+        description = "The nf_conntrack table is {{ $value }}% full.",
+      }
+
+    ALERT K8SConntrackTableFull
+      IF 100*node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 90
+      LABELS {
+        service = "k8s",
+        severity = "critical"
+      }
+      ANNOTATIONS {
+        summary = "Number of tracked connections is near the limit",
+        description = "The nf_conntrack table is {{ $value }}% full.",
+      }
+
+    # To catch the conntrack sysctl de-tuning when it happens
+    ALERT K8SConntrackTuningMissing
+      IF node_nf_conntrack_udp_timeout > 10
+      FOR 10m
+      LABELS {
+        service = "k8s",
+        severity = "warning",
+      }
+      ANNOTATIONS {
+        summary = "Node does not have the correct conntrack tunings",
+        description = "Nodes keep un-setting the correct tunings, investigate when it happens.",
+      }
+
+    ALERT K8STooManyOpenFiles
+      IF 100*process_open_fds{job=~"kubelet|kubernetes"} / process_max_fds > 50
+      FOR 10m
+      LABELS {
+        service = "k8s",
+        severity = "warning"
+      }
+      ANNOTATIONS {
+        summary = "{{ $labels.job }} has too many open file descriptors",
+        description = "{{ $labels.node }} is using {{ $value }}% of the available file/socket descriptors.",
+      }
+
+    ALERT K8STooManyOpenFiles
+      IF 100*process_open_fds{job=~"kubelet|kubernetes"} / process_max_fds > 80
+      FOR 10m
+      LABELS {
+        service = "k8s",
+        severity = "critical"
+      }
+      ANNOTATIONS {
+        summary = "{{ $labels.job }} has too many open file descriptors",
+        description = "{{ $labels.node }} is using {{ $value }}% of the available file/socket descriptors.",
+      }
+
+    # Some verbs excluded because they are expected to be long-lasting:
+    # WATCHLIST is long-poll, CONNECT is `kubectl exec`.
+    ALERT K8SApiServerLatency
+      IF histogram_quantile(
+          0.99,
+          sum without (instance,node,resource) (apiserver_request_latencies_bucket{verb!~"CONNECT|WATCHLIST|WATCH"})
+        ) / 1e6 > 1.0
+      FOR 10m
+      LABELS {
+        service = "k8s",
+        severity = "warning"
+      }
+      ANNOTATIONS {
+        summary = "Kubernetes apiserver latency is high",
+        description = "99th percentile Latency for {{ $labels.verb }} requests to the kube-apiserver is higher than 1s.",
+      }
+
+    ALERT K8SApiServerEtcdAccessLatency
+      IF etcd_request_latencies_summary{quantile="0.99"} / 1e6 > 1.0
+      FOR 15m
+      LABELS {
+        service = "k8s",
+        severity = "warning"
+      }
+      ANNOTATIONS {
+        summary = "Access to etcd is slow",
+        description = "99th percentile latency for apiserver to access etcd is higher than 1s.",
+      }
+
+    ALERT K8SKubeletTooManyPods
+      IF kubelet_running_pod_count > 100
+      LABELS {
+        service = "k8s",
+        severity = "warning",
+      }
+      ANNOTATIONS {
+        summary = "Kubelet is close to pod limit",
+        description = "Kubelet {{$labels.instance}} is running {{$value}} pods, close to the limit of 110",
+      }
+
+kind: ConfigMap
+metadata:
+  creationTimestamp: null
+  name: prometheus-k8s-rules

roles/kubernetes-apps/prometheus/templates/prometheus-k8s-service.yml.j2

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus-k8s
+spec:
+  type: NodePort
+  ports:
+  - name: web
+    nodePort: 30900
+    port: 9090
+    protocol: TCP
+    targetPort: web
+  selector:
+    prometheus: k8s

roles/kubernetes-apps/prometheus/templates/prometheus-operator-deployment.yml.j2

@@ -0,0 +1,23 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: prometheus-operator
+  labels:
+    operator: prometheus
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        operator: prometheus
+    spec:
+      containers:
+       - name: prometheus-operator
+         image: "{{ prometheus_operator_image_repo }}:{{ prometheus_operator_image_tag }}"
+         resources:
+           requests:
+             cpu: {{ prometheus_operator_cpu_requests }}
+             memory: {{ prometheus_operator_mem_requests }}
+           limits:
+             cpu: {{ prometheus_operator_cpu_limit }}
+             memory: {{ prometheus_operator_mem_limit }}