From 8218b9970ff73dc14f260724a38cf5f8e0d5ef3b Mon Sep 17 00:00:00 2001 From: Brad Beam Date: Thu, 26 Jan 2017 17:33:01 -0600 Subject: [PATCH] Adding EFK logging stack --- cluster.yml | 1 + inventory/group_vars/all.yml | 2 + roles/download/defaults/main.yml | 24 +++++++++ .../efk/elasticsearch/defaults/main.yml | 6 +++ .../efk/elasticsearch/meta/main.yml | 7 +++ .../efk/elasticsearch/tasks/main.yml | 35 +++++++++++++ .../templates/elasticsearch-deployment.yml.j2 | 51 +++++++++++++++++++ .../templates/elasticsearch-service.yml.j2 | 18 +++++++ .../efk/fluentd/defaults/main.yml | 5 ++ .../kubernetes-apps/efk/fluentd/meta/main.yml | 3 ++ .../efk/fluentd/tasks/main.yml | 17 +++++++ .../efk/fluentd/templates/fluentd-ds.yml.j2 | 49 ++++++++++++++++++ .../efk/kibana/defaults/main.yml | 7 +++ .../kubernetes-apps/efk/kibana/meta/main.yml | 3 ++ .../kubernetes-apps/efk/kibana/tasks/main.yml | 34 +++++++++++++ .../kibana/templates/kibana-deployment.yml.j2 | 46 +++++++++++++++++ .../kibana/templates/kibana-service.yml.j2 | 18 +++++++ roles/kubernetes-apps/efk/meta/main.yml | 4 ++ roles/kubernetes-apps/meta/main.yaml | 3 ++ 19 files changed, 333 insertions(+) create mode 100644 roles/kubernetes-apps/efk/elasticsearch/defaults/main.yml create mode 100644 roles/kubernetes-apps/efk/elasticsearch/meta/main.yml create mode 100644 roles/kubernetes-apps/efk/elasticsearch/tasks/main.yml create mode 100644 roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-deployment.yml.j2 create mode 100644 roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-service.yml.j2 create mode 100644 roles/kubernetes-apps/efk/fluentd/defaults/main.yml create mode 100644 roles/kubernetes-apps/efk/fluentd/meta/main.yml create mode 100644 roles/kubernetes-apps/efk/fluentd/tasks/main.yml create mode 100644 roles/kubernetes-apps/efk/fluentd/templates/fluentd-ds.yml.j2 create mode 100644 roles/kubernetes-apps/efk/kibana/defaults/main.yml create mode 100644 roles/kubernetes-apps/efk/kibana/meta/main.yml create mode 100644 roles/kubernetes-apps/efk/kibana/tasks/main.yml create mode 100644 roles/kubernetes-apps/efk/kibana/templates/kibana-deployment.yml.j2 create mode 100644 roles/kubernetes-apps/efk/kibana/templates/kibana-service.yml.j2 create mode 100644 roles/kubernetes-apps/efk/meta/main.yml diff --git a/cluster.yml b/cluster.yml index becad3b6d..1a08283b5 100644 --- a/cluster.yml +++ b/cluster.yml @@ -63,3 +63,4 @@ any_errors_fatal: true roles: - { role: kubernetes-apps, tags: apps } + - { role: kubernetes-apps/efk, tags: [ apps, efk ] } diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index e5047ca47..8242b5fd9 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -204,3 +204,5 @@ kpm_packages: [] rkt_version: 1.21.0 etcd_deployment_type: docker kubelet_deployment_type: docker + +efk_enabled: false diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 7e06ac574..dad46f58d 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -82,6 +82,15 @@ kubedns_image_repo: "gcr.io/google_containers/kubedns-amd64" kubedns_image_tag: "{{ kubedns_version }}" test_image_repo: busybox test_image_tag: latest +elasticsearch_version: "v2.4.1" +elasticsearch_image_repo: "gcr.io/google_containers/elasticsearch" +elasticsearch_image_tag: "{{ elasticsearch_version }}" +fluentd_version: "1.22" +fluentd_image_repo: "gcr.io/google_containers/fluentd-elasticsearch" +fluentd_image_tag: "{{ fluentd_version }}" +kibana_version: "v4.6.1" +kibana_image_repo: "gcr.io/google_containers/kibana" +kibana_image_tag: "{{ kibana_version }}" downloads: netcheck_server: @@ -203,6 +212,21 @@ downloads: repo: "{{ exechealthz_image_repo }}" tag: "{{ exechealthz_image_tag }}" sha256: "{{ exechealthz_digest_checksum|default(None) }}" + elasticsearch: + container: true + repo: "{{ elasticsearch_image_repo }}" + tag: "{{ elasticsearch_image_tag }}" + sha256: "{{ elasticsearch_digest_checksum|default(None) }}" + fluentd: + container: true + repo: "{{ fluentd_image_repo }}" + tag: "{{ fluentd_image_tag }}" + sha256: "{{ fluentd_digest_checksum|default(None) }}" + kibana: + container: true + repo: "{{ kibana_image_repo }}" + tag: "{{ kibana_image_tag }}" + sha256: "{{ kibana_digest_checksum|default(None) }}" download: container: "{{ file.container|default('false') }}" diff --git a/roles/kubernetes-apps/efk/elasticsearch/defaults/main.yml b/roles/kubernetes-apps/efk/elasticsearch/defaults/main.yml new file mode 100644 index 000000000..e5af87425 --- /dev/null +++ b/roles/kubernetes-apps/efk/elasticsearch/defaults/main.yml @@ -0,0 +1,6 @@ +--- +elasticsearch_cpu_limit: 1000m +elasticsearch_mem_limit: 0M +elasticsearch_cpu_requests: 100m +elasticsearch_mem_requests: 0M +elasticsearch_service_port: 9200 diff --git a/roles/kubernetes-apps/efk/elasticsearch/meta/main.yml b/roles/kubernetes-apps/efk/elasticsearch/meta/main.yml new file mode 100644 index 000000000..cd0a80606 --- /dev/null +++ b/roles/kubernetes-apps/efk/elasticsearch/meta/main.yml @@ -0,0 +1,7 @@ +dependencies: + - role: download + file: "{{ downloads.elasticsearch }}" +# TODO: bradbeam add in curator +# https://github.com/Skillshare/kubernetes-efk/blob/master/configs/elasticsearch.yml#L94 +# - role: download +# file: "{{ downloads.curator }}" diff --git a/roles/kubernetes-apps/efk/elasticsearch/tasks/main.yml b/roles/kubernetes-apps/efk/elasticsearch/tasks/main.yml new file mode 100644 index 000000000..848f7226b --- /dev/null +++ b/roles/kubernetes-apps/efk/elasticsearch/tasks/main.yml @@ -0,0 +1,35 @@ +--- +- name: "ElasticSearch | Write ES deployment" + template: + src: elasticsearch-deployment.yml.j2 + dest: "{{ kube_config_dir }}/elasticsearch-deployment.yaml" + register: es_deployment_manifest + +- name: "ElasticSearch | Create ES deployment" + kube: + filename: "{{kube_config_dir}}/elasticsearch-deployment.yaml" + kubectl: "{{bin_dir}}/kubectl" + name: "elasticsearch-logging-v1" + namespace: "{{system_namespace}}" + resource: "rc" + state: "{{ item | ternary('latest','present') }}" + #state: "latest" + with_items: "{{ es_deployment_manifest.changed }}" + run_once: true + +- name: "ElasticSearch | Write ES service " + template: + src: elasticsearch-service.yml.j2 + dest: "{{ kube_config_dir }}/elasticsearch-service.yaml" + register: es_service_manifest + +- name: "ElasticSearch | Create ES service" + kube: + filename: "{{kube_config_dir}}/elasticsearch-service.yaml" + kubectl: "{{bin_dir}}/kubectl" + name: "elasticsearch-logging" + namespace: "{{system_namespace}}" + resource: "svc" + state: "{{ item | ternary('latest','present') }}" + with_items: "{{ es_service_manifest.changed }}" + run_once: true diff --git a/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-deployment.yml.j2 b/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-deployment.yml.j2 new file mode 100644 index 000000000..98d26675d --- /dev/null +++ b/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-deployment.yml.j2 @@ -0,0 +1,51 @@ +--- +# https://raw.githubusercontent.com/kubernetes/kubernetes/v1.5.2/cluster/addons/fluentd-elasticsearch/es-controller.yaml +apiVersion: v1 +kind: ReplicationController +metadata: + name: elasticsearch-logging-v1 + namespace: "{{ system_namespace }}" + labels: + k8s-app: elasticsearch-logging + version: "{{ elasticsearch_image_tag }}" + kubernetes.io/cluster-service: "true" +spec: + replicas: 2 + selector: + k8s-app: elasticsearch-logging + version: "{{ elasticsearch_image_tag }}" + template: + metadata: + labels: + k8s-app: elasticsearch-logging + version: "{{ elasticsearch_image_tag }}" + kubernetes.io/cluster-service: "true" + spec: + containers: + - image: "{{ elasticsearch_image_repo }}:{{ elasticsearch_image_tag }}" + name: elasticsearch-logging + resources: + # need more cpu upon initialization, therefore burstable class + limits: + cpu: {{ elasticsearch_cpu_limit }} +{% if elasticsearch_mem_limit is defined and elasticsearch_mem_limit != "0M" %} + mem: {{ elasticsearch_mem_limit }} +{% endif %} + requests: + cpu: {{ elasticsearch_cpu_requests }} +{% if elasticsearch_mem_requests is defined and elasticsearch_mem_requests != "0M" %} + mem: {{ elasticsearch_mem_requests }} +{% endif %} + ports: + - containerPort: 9200 + name: db + protocol: TCP + - containerPort: 9300 + name: transport + protocol: TCP + volumeMounts: + - name: es-persistent-storage + mountPath: /data + volumes: + - name: es-persistent-storage + emptyDir: {} diff --git a/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-service.yml.j2 b/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-service.yml.j2 new file mode 100644 index 000000000..b7558f9d9 --- /dev/null +++ b/roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-service.yml.j2 @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch-logging + namespace: "{{ system_namespace }}" + labels: + k8s-app: elasticsearch-logging + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "Elasticsearch" +spec: + ports: + - port: {{ elasticsearch_service_port }} + protocol: TCP + targetPort: db + selector: + k8s-app: elasticsearch-logging + diff --git a/roles/kubernetes-apps/efk/fluentd/defaults/main.yml b/roles/kubernetes-apps/efk/fluentd/defaults/main.yml new file mode 100644 index 000000000..dfb2c98a2 --- /dev/null +++ b/roles/kubernetes-apps/efk/fluentd/defaults/main.yml @@ -0,0 +1,5 @@ +--- +fluentd_cpu_limit: 0m +fluentd_mem_limit: 200Mi +fluentd_cpu_requests: 100m +fluentd_mem_requests: 200Mi diff --git a/roles/kubernetes-apps/efk/fluentd/meta/main.yml b/roles/kubernetes-apps/efk/fluentd/meta/main.yml new file mode 100644 index 000000000..1ba777c76 --- /dev/null +++ b/roles/kubernetes-apps/efk/fluentd/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - role: download + file: "{{ downloads.fluentd }}" diff --git a/roles/kubernetes-apps/efk/fluentd/tasks/main.yml b/roles/kubernetes-apps/efk/fluentd/tasks/main.yml new file mode 100644 index 000000000..eebdaed73 --- /dev/null +++ b/roles/kubernetes-apps/efk/fluentd/tasks/main.yml @@ -0,0 +1,17 @@ +--- +- name: "Fluentd | Write fluentd daemonset" + template: + src: fluentd-ds.yml.j2 + dest: "{{ kube_config_dir }}/fluentd-ds.yaml" + register: fluentd_ds_manifest + +- name: "Fluentd | Create fluentd daemonset" + kube: + filename: "{{kube_config_dir}}/fluentd-ds.yaml" + kubectl: "{{bin_dir}}/kubectl" + name: "fluentd-es-v{{ fluentd_version }}" + namespace: "{{system_namespace}}" + resource: "ds" + state: "{{ item | ternary('latest','present') }}" + with_items: "{{ fluentd_ds_manifest.changed }}" + run_once: true diff --git a/roles/kubernetes-apps/efk/fluentd/templates/fluentd-ds.yml.j2 b/roles/kubernetes-apps/efk/fluentd/templates/fluentd-ds.yml.j2 new file mode 100644 index 000000000..d15b10b43 --- /dev/null +++ b/roles/kubernetes-apps/efk/fluentd/templates/fluentd-ds.yml.j2 @@ -0,0 +1,49 @@ +--- +# https://raw.githubusercontent.com/kubernetes/kubernetes/v1.5.2/cluster/addons/fluentd-elasticsearch/es-controller.yaml +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: "fluentd-es-v{{ fluentd_version }}" + namespace: "{{ system_namespace }}" + labels: + k8s-app: fluentd-es + kubernetes.io/cluster-service: "true" + version: "v{{ fluentd_version }}" +spec: + template: + metadata: + labels: + k8s-app: fluentd-es + kubernetes.io/cluster-service: "true" + version: "v{{ fluentd_version }}" + spec: + containers: + - name: fluentd-es + image: "{{ fluentd_image_repo }}:{{ fluentd_image_tag }}" + command: + - '/bin/sh' + - '-c' + - '/usr/sbin/td-agent 2>&1 >> /var/log/fluentd.log' + resources: + limits: +{% if fluentd_cpu_limit is defined and fluentd_cpu_limit != "0m" %} + mem: {{ fluentd_cpu_limit }} +{% endif %} + memory: {{ fluentd_mem_limit }} + requests: + cpu: {{ fluentd_cpu_requests }} + memory: {{ fluentd_mem_requests }} + volumeMounts: + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + terminationGracePeriodSeconds: 30 + volumes: + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers diff --git a/roles/kubernetes-apps/efk/kibana/defaults/main.yml b/roles/kubernetes-apps/efk/kibana/defaults/main.yml new file mode 100644 index 000000000..3878cddab --- /dev/null +++ b/roles/kubernetes-apps/efk/kibana/defaults/main.yml @@ -0,0 +1,7 @@ +--- +kibana_cpu_limit: 100m +kibana_mem_limit: 0M +kibana_cpu_requests: 100m +kibana_mem_requests: 0M +kibana_service_port: 5601 +kibaba_base_url: "/api/v1/proxy/namespaces/kube-system/services/kibana-logging" diff --git a/roles/kubernetes-apps/efk/kibana/meta/main.yml b/roles/kubernetes-apps/efk/kibana/meta/main.yml new file mode 100644 index 000000000..34d0ab21a --- /dev/null +++ b/roles/kubernetes-apps/efk/kibana/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - role: download + file: "{{ downloads.kibana }}" diff --git a/roles/kubernetes-apps/efk/kibana/tasks/main.yml b/roles/kubernetes-apps/efk/kibana/tasks/main.yml new file mode 100644 index 000000000..beb499d5a --- /dev/null +++ b/roles/kubernetes-apps/efk/kibana/tasks/main.yml @@ -0,0 +1,34 @@ +--- +- name: "Kibana | Write Kibana deployment" + template: + src: kibana-deployment.yml.j2 + dest: "{{ kube_config_dir }}/kibana-deployment.yaml" + register: kibana_deployment_manifest + +- name: "Kibana | Create Kibana deployment" + kube: + filename: "{{kube_config_dir}}/kibana-deployment.yaml" + kubectl: "{{bin_dir}}/kubectl" + name: "kibana-logging" + namespace: "{{system_namespace}}" + resource: "rc" + state: "{{ item | ternary('latest','present') }}" + with_items: "{{ kibana_deployment_manifest.changed }}" + run_once: true + +- name: "Kibana | Write Kibana service " + template: + src: kibana-service.yml.j2 + dest: "{{ kube_config_dir }}/kibana-service.yaml" + register: kibana_service_manifest + +- name: "Kibana | Create Kibana service" + kube: + filename: "{{kube_config_dir}}/kibana-service.yaml" + kubectl: "{{bin_dir}}/kubectl" + name: "kibana-logging" + namespace: "{{system_namespace}}" + resource: "svc" + state: "{{ item | ternary('latest','present') }}" + with_items: "{{ kibana_service_manifest.changed }}" + run_once: true diff --git a/roles/kubernetes-apps/efk/kibana/templates/kibana-deployment.yml.j2 b/roles/kubernetes-apps/efk/kibana/templates/kibana-deployment.yml.j2 new file mode 100644 index 000000000..d949a9790 --- /dev/null +++ b/roles/kubernetes-apps/efk/kibana/templates/kibana-deployment.yml.j2 @@ -0,0 +1,46 @@ +--- +# https://raw.githubusercontent.com/kubernetes/kubernetes/v1.5.2/cluster/addons/fluentd-kibana/kibana-controller.yaml +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: kibana-logging + namespace: "{{ system_namespace }}" + labels: + k8s-app: kibana-logging + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: kibana-logging + template: + metadata: + labels: + k8s-app: kibana-logging + spec: + containers: + - name: kibana-logging + image: "{{ kibana_image_repo }}:{{ kibana_image_tag }}" + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: {{ kibana_cpu_limit }} +{% if kibana_mem_limit is defined and kibana_mem_limit != "0M" %} + mem: {{ kibana_mem_limit }} +{% endif %} + requests: + cpu: {{ kibana_cpu_requests }} +{% if kibana_mem_requests is defined and kibana_mem_requests != "0M" %} + mem: {{ kibana_mem_requests }} +{% endif %} + env: + - name: "ELASTICSEARCH_URL" + value: "http://elasticsearch-logging:{{ elasticsearch_service_port }}" +{% if kibana_base_url is defined and kibana_base_url != "" %} + - name: "KIBANA_BASE_URL" + value: "{{ kibana_base_url }}" +{% endif %} + ports: + - containerPort: 5601 + name: ui + protocol: TCP diff --git a/roles/kubernetes-apps/efk/kibana/templates/kibana-service.yml.j2 b/roles/kubernetes-apps/efk/kibana/templates/kibana-service.yml.j2 new file mode 100644 index 000000000..241b896f0 --- /dev/null +++ b/roles/kubernetes-apps/efk/kibana/templates/kibana-service.yml.j2 @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: kibana-logging + namespace: "{{ system_namespace }}" + labels: + k8s-app: kibana-logging + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "Kibana" +spec: + ports: + - port: {{ kibana_service_port }} + protocol: TCP + targetPort: ui + selector: + k8s-app: kibana-logging + diff --git a/roles/kubernetes-apps/efk/meta/main.yml b/roles/kubernetes-apps/efk/meta/main.yml new file mode 100644 index 000000000..e11bbae29 --- /dev/null +++ b/roles/kubernetes-apps/efk/meta/main.yml @@ -0,0 +1,4 @@ +dependencies: + - role: kubernetes-apps/efk/elasticsearch + - role: kubernetes-apps/efk/fluentd + - role: kubernetes-apps/efk/kibana diff --git a/roles/kubernetes-apps/meta/main.yaml b/roles/kubernetes-apps/meta/main.yaml index 2ddb435de..b04e051d9 100644 --- a/roles/kubernetes-apps/meta/main.yaml +++ b/roles/kubernetes-apps/meta/main.yaml @@ -18,3 +18,6 @@ dependencies: tags: [download, netchecker] - {role: kubernetes-apps/ansible, tags: apps} - {role: kubernetes-apps/kpm, tags: [apps, kpm]} + - role: kubernetes-apps/efk + when: efk_enabled + tags: [ apps, efk ]