From 2556eb2733bf964a52ccc099bbcb58c0870068a5 Mon Sep 17 00:00:00 2001 From: bac-w <57893632+bac-w@users.noreply.github.com> Date: Fri, 30 Apr 2021 18:09:59 +0300 Subject: [PATCH] Upgrade cilium role (#7521) * Upgrade cilium roles * Del old test result * Add hubble ui examples * Refactor hubble metrics * Markdown fix pipeline errors * yamllint check and fix * refactor install from https://github.com/kubernetes-sigs/kubespray/pull/7520 * Docs syntax change (fix) * Cilium set default 1.8.9 * Update cilium version in Readme --- README.md | 2 +- docs/cilium.md | 50 ++++++ roles/download/defaults/main.yml | 12 +- .../network_plugin/meta/main.yml | 5 - roles/network_plugin/cilium/defaults/main.yml | 27 ++- .../cilium/tasks/apply.yml} | 15 +- roles/network_plugin/cilium/tasks/install.yml | 34 +++- roles/network_plugin/cilium/tasks/main.yml | 2 + roles/network_plugin/cilium/tasks/reset.yml | 2 +- .../cilium/templates/cilium-config.yml.j2 | 17 +- .../cilium/templates/cilium-cr.yml.j2 | 20 +++ .../cilium/templates/cilium-crb.yml.j2 | 2 + .../cilium/templates/cilium-ds.yml.j2 | 28 +++ .../cilium/templates/hubble-config.yml.j2 | 86 ++++++++++ .../cilium/templates/hubble-cr.yml.j2 | 106 ++++++++++++ .../cilium/templates/hubble-crb.yml.j2 | 44 +++++ .../cilium/templates/hubble-cronjob.yml.j2 | 49 ++++++ .../cilium/templates/hubble-deploy.yml.j2 | 161 ++++++++++++++++++ .../cilium/templates/hubble-job.yml.j2 | 45 +++++ .../cilium/templates/hubble-sa.yml.j2 | 23 +++ .../cilium/templates/hubble-service.yml.j2 | 58 +++++++ 21 files changed, 772 insertions(+), 16 deletions(-) rename roles/{kubernetes-apps/network_plugin/cilium/tasks/main.yml => network_plugin/cilium/tasks/apply.yml} (59%) create mode 100644 roles/network_plugin/cilium/templates/hubble-config.yml.j2 create mode 100644 roles/network_plugin/cilium/templates/hubble-cr.yml.j2 create mode 100644 roles/network_plugin/cilium/templates/hubble-crb.yml.j2 create mode 100644 roles/network_plugin/cilium/templates/hubble-cronjob.yml.j2 create mode 100644 roles/network_plugin/cilium/templates/hubble-deploy.yml.j2 create mode 100644 roles/network_plugin/cilium/templates/hubble-job.yml.j2 create mode 100644 roles/network_plugin/cilium/templates/hubble-sa.yml.j2 create mode 100644 roles/network_plugin/cilium/templates/hubble-service.yml.j2 diff --git a/README.md b/README.md index 9ff07bbc5..d89e19044 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ Note: Upstart/SysV init based OS types are not supported. - [cni-plugins](https://github.com/containernetworking/plugins) v0.9.1 - [calico](https://github.com/projectcalico/calico) v3.17.4 - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) - - [cilium](https://github.com/cilium/cilium) v1.8.8 + - [cilium](https://github.com/cilium/cilium) v1.8.9 - [flanneld](https://github.com/coreos/flannel) v0.13.0 - [kube-ovn](https://github.com/alauda/kube-ovn) v1.6.2 - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.2.2 diff --git a/docs/cilium.md b/docs/cilium.md index ea4294a94..9fdfbfae7 100644 --- a/docs/cilium.md +++ b/docs/cilium.md @@ -11,3 +11,53 @@ Hence, in this configuration in Kubespray, Cilium will always contact the external loadbalancer (even from a node in the control plane) and if there is no external load balancer It will ignore any local load balancer deployed by Kubespray and **only contacts the first master**. + +## Choose Cilium version + +```yml +cilium_version: v1.8.9 ## or v1.9.6 +``` + +## Add variable to config + +Use following variables: + +Example: + +```yml +cilium_config_extra_vars: + enable-endpoint-routes: true +``` + +## Install Cilium Hubble + +k8s-net-cilium.yml: + +```yml +cilium_enable_hubble: true ## enable support hubble in cilium +cilium_hubble_install: true ## install hubble-relay, hubble-ui +cilium_hubble_tls_generate: true ## install hubble-certgen and generate certificates +``` + +To validate that Hubble UI is properly configured, set up a port forwarding for hubble-ui service: + +```shell script +kubectl port-forward -n kube-system svc/hubble-ui 12000:80 +``` + +and then open [http://localhost:12000/](http://localhost:12000/). + +## Hubble metrics + +```yml +cilium_enable_hubble_metrics: true +cilium_hubble_metrics: + - dns + - drop + - tcp + - flow + - icmp + - http +``` + +[More](https://docs.cilium.io/en/v1.9/operations/metrics/#hubble-exported-metrics) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index b55732422..61005a2fb 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -77,7 +77,7 @@ flannel_version: "v0.13.0" cni_version: "v0.9.1" weave_version: 2.8.1 pod_infra_version: "3.3" -cilium_version: "v1.8.8" +cilium_version: "v1.8.9" kube_ovn_version: "v1.6.2" kube_router_version: "v1.2.2" multus_version: "v3.7" @@ -429,6 +429,16 @@ cilium_init_image_repo: "{{ quay_image_repo }}/cilium/cilium-init" cilium_init_image_tag: "2019-04-05" cilium_operator_image_repo: "{{ quay_image_repo }}/cilium/operator" cilium_operator_image_tag: "{{ cilium_version }}" +cilium_hubble_relay_image_repo: "{{ quay_image_repo }}/cilium/hubble-relay" +cilium_hubble_relay_image_tag: "{{ cilium_version }}" +cilium_hubble_certgen_image_repo: "{{ quay_image_repo }}/cilium/certgen" +cilium_hubble_certgen_image_tag: "v0.1.3" +cilium_hubble_ui_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui" +cilium_hubble_ui_image_tag: "v0.7.3" +cilium_hubble_ui_backend_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui-backend" +cilium_hubble_ui_backend_image_tag: "v0.7.3" +cilium_hubble_envoy_image_repo: "{{ docker_image_repo }}/envoyproxy/envoy" +cilium_hubble_envoy_image_tag: "v1.14.5" kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn" kube_ovn_container_image_tag: "{{ kube_ovn_version }}" kube_router_image_repo: "{{ docker_image_repo }}/cloudnativelabs/kube-router" diff --git a/roles/kubernetes-apps/network_plugin/meta/main.yml b/roles/kubernetes-apps/network_plugin/meta/main.yml index 1b929ba41..00aae5095 100644 --- a/roles/kubernetes-apps/network_plugin/meta/main.yml +++ b/roles/kubernetes-apps/network_plugin/meta/main.yml @@ -1,10 +1,5 @@ --- dependencies: - - role: kubernetes-apps/network_plugin/cilium - when: kube_network_plugin == 'cilium' or cilium_deploy_additionally | default(false) | bool - tags: - - cilium - - role: kubernetes-apps/network_plugin/calico when: kube_network_plugin == 'calico' tags: diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml index 4ca451621..04c43a71d 100644 --- a/roles/network_plugin/cilium/defaults/main.yml +++ b/roles/network_plugin/cilium/defaults/main.yml @@ -22,9 +22,6 @@ cilium_cpu_requests: 100m cilium_tunnel_mode: vxlan # Optional features cilium_enable_prometheus: false -cilium_enable_hubble_metrics: false -cilium_enable_hubble: false -cilium_hubble_metrics: "" # Enable if you want to make use of hostPort mappings cilium_enable_portmap: false # Monitor aggregation level (none/low/medium/maximum) @@ -55,6 +52,30 @@ cilium_native_routing_cidr: "" # IPsec based transparent encryption between nodes cilium_ipsec_enabled: false +# Hubble +### Enable Hubble without install +cilium_enable_hubble: false +### Enable Hubble Metrics +cilium_enable_hubble_metrics: false +### if cilium_enable_hubble_metrics: true +cilium_hubble_metrics: {} +# - dns +# - drop +# - tcp +# - flow +# - icmp +# - http +### Enable Hubble install +cilium_hubble_install: false +### Enable auto generate certs if cilium_hubble_install: true +cilium_hubble_tls_generate: false + # IP address management mode for v1.9+. # https://docs.cilium.io/en/v1.9/concepts/networking/ipam/ cilium_ipam_mode: kubernetes + +## A dictionary of extra config variables to add to cilium-config, formatted like: +## cilium_config_extra_vars: +## var1: "value1" +## var2: "value2" +cilium_config_extra_vars: {} diff --git a/roles/kubernetes-apps/network_plugin/cilium/tasks/main.yml b/roles/network_plugin/cilium/tasks/apply.yml similarity index 59% rename from roles/kubernetes-apps/network_plugin/cilium/tasks/main.yml rename to roles/network_plugin/cilium/tasks/apply.yml index d3d6ceec5..9824337e2 100644 --- a/roles/kubernetes-apps/network_plugin/cilium/tasks/main.yml +++ b/roles/network_plugin/cilium/tasks/apply.yml @@ -7,7 +7,7 @@ resource: "{{ item.item.type }}" filename: "{{ kube_config_dir }}/{{ item.item.file }}" state: "latest" - with_items: "{{ cilium_node_manifests.results }}" + loop: "{{ cilium_node_manifests.results }}" when: inventory_hostname == groups['kube_control_plane'][0] and not item is skipped - name: Cilium | Wait for pods to run @@ -18,3 +18,16 @@ delay: 10 ignore_errors: yes when: inventory_hostname == groups['kube_control_plane'][0] + +- name: Cilium | Hubble install + kube: + name: "{{ item.item.name }}" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/hubble/{{ item.item.file }}" + state: "latest" + loop: "{{ cilium_hubble_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] and not item is skipped + - cilium_enable_hubble and cilium_hubble_install diff --git a/roles/network_plugin/cilium/tasks/install.yml b/roles/network_plugin/cilium/tasks/install.yml index c6e59f47d..6cd5ff946 100644 --- a/roles/network_plugin/cilium/tasks/install.yml +++ b/roles/network_plugin/cilium/tasks/install.yml @@ -20,16 +20,27 @@ dest: "{{ cilium_cert_dir }}/{{ item.d }}" state: hard force: yes - with_items: + loop: - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"} - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} +- name: Cilium | Create hubble dir + file: + path: "{{ kube_config_dir }}/addons/hubble" + state: directory + owner: root + group: root + mode: 0755 + when: + - inventory_hostname == groups['kube_control_plane'][0] + - cilium_hubble_install + - name: Cilium | Create Cilium node manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/{{ item.file }}" - with_items: + loop: - {name: cilium, file: cilium-config.yml, type: cm} - {name: cilium, file: cilium-crb.yml, type: clusterrolebinding} - {name: cilium, file: cilium-cr.yml, type: clusterrole} @@ -42,6 +53,25 @@ - inventory_hostname in groups['kube_control_plane'] - item.when | default(True) | bool +- name: Cilium | Create Cilium Hubble manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/addons/hubble/{{ item.file }}" + loop: + - {name: hubble, file: hubble-config.yml, type: cm} + - {name: hubble, file: hubble-crb.yml, type: clusterrolebinding} + - {name: hubble, file: hubble-cr.yml, type: clusterrole} + - {name: hubble, file: hubble-cronjob.yml, type: cronjob, when: "{{ cilium_hubble_tls_generate }}"} + - {name: hubble, file: hubble-deploy.yml, type: deploy} + - {name: hubble, file: hubble-job.yml, type: job, when: "{{ cilium_hubble_tls_generate }}"} + - {name: hubble, file: hubble-sa.yml, type: sa} + - {name: hubble, file: hubble-service.yml, type: service} + register: cilium_hubble_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + - cilium_enable_hubble and cilium_hubble_install + - item.when | default(True) | bool + - name: Cilium | Enable portmap addon template: src: 000-cilium-portmap.conflist.j2 diff --git a/roles/network_plugin/cilium/tasks/main.yml b/roles/network_plugin/cilium/tasks/main.yml index c3bee558e..63c99dc06 100644 --- a/roles/network_plugin/cilium/tasks/main.yml +++ b/roles/network_plugin/cilium/tasks/main.yml @@ -2,3 +2,5 @@ - import_tasks: check.yml - include_tasks: install.yml + +- include_tasks: apply.yml diff --git a/roles/network_plugin/cilium/tasks/reset.yml b/roles/network_plugin/cilium/tasks/reset.yml index 65b6e9a78..432df8a5c 100644 --- a/roles/network_plugin/cilium/tasks/reset.yml +++ b/roles/network_plugin/cilium/tasks/reset.yml @@ -3,7 +3,7 @@ include_tasks: reset_iface.yml vars: iface: "{{ item }}" - with_items: + loop: - cilium_host - cilium_net - cilium_vxlan diff --git a/roles/network_plugin/cilium/templates/cilium-config.yml.j2 b/roles/network_plugin/cilium/templates/cilium-config.yml.j2 index 0c8a9b480..94cd21664 100644 --- a/roles/network_plugin/cilium/templates/cilium-config.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-config.yml.j2 @@ -154,13 +154,26 @@ data: # Hubble settings {% if cilium_enable_hubble %} enable-hubble: "true" - hubble-metrics: "{{ cilium_hubble_metrics }}" - hubble-listen-address: ":4244" {% if cilium_enable_hubble_metrics %} hubble-metrics-server: ":9091" + hubble-metrics: +{% for hubble_metrics_cycle in cilium_hubble_metrics %} + {{ hubble_metrics_cycle }} +{% endfor %} +{% endif %} + hubble-listen-address: ":4244" +{% if cilium_enable_hubble and cilium_hubble_install %} + hubble-disable-tls: "false" + hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt + hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key + hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt {% endif %} {% endif %} +{% for key, value in cilium_config_extra_vars.items() %} + {{ key }}: "{{ value }}" +{% endfor %} + # IPsec based transparent encryption between nodes {% if cilium_ipsec_enabled %} enable-ipsec: "true" diff --git a/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 index e4939e19a..4fbbf173a 100644 --- a/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 @@ -58,6 +58,16 @@ rules: - ciliumnodes/status - ciliumidentities - ciliumidentities/status +{% endif %} +{% if cilium_version | regex_replace('v') is version('1.9', '>=') %} + - ciliumnetworkpolicies/finalizers + - ciliumclusterwidenetworkpolicies/finalizers + - ciliumendpoints/finalizers + - ciliumnodes/finalizers + - ciliumidentities/finalizers + - ciliumlocalredirectpolicies + - ciliumlocalredirectpolicies/status + - ciliumlocalredirectpolicies/finalizers {% endif %} verbs: - '*' @@ -183,6 +193,16 @@ rules: - ciliumnodes/status - ciliumidentities - ciliumidentities/status +{% endif %} +{% if cilium_version | regex_replace('v') is version('1.9', '>=') %} + - ciliumnetworkpolicies/finalizers + - ciliumclusterwidenetworkpolicies/finalizers + - ciliumendpoints/finalizers + - ciliumnodes/finalizers + - ciliumidentities/finalizers + - ciliumlocalredirectpolicies + - ciliumlocalredirectpolicies/status + - ciliumlocalredirectpolicies/finalizers {% endif %} verbs: - '*' diff --git a/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 b/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 index 8857d31a6..5bc7b2779 100644 --- a/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 @@ -24,6 +24,8 @@ subjects: - kind: ServiceAccount name: cilium namespace: kube-system +{% if cilium_version | regex_replace('v') is version('1.9', '<') %} - apiGroup: rbac.authorization.k8s.io kind: Group name: system:nodes +{% endif %} diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 index 1c79cc140..08aa56379 100644 --- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 @@ -170,6 +170,11 @@ spec: - mountPath: /etc/ipsec name: cilium-ipsec-secrets readOnly: true +{% endif %} +{% if cilium_hubble_install %} + - mountPath: /var/lib/cilium/tls/hubble + name: hubble-tls + readOnly: true {% endif %} dnsPolicy: ClusterFirstWithHostNet hostNetwork: true @@ -196,7 +201,11 @@ spec: key: wait-bpf-mount name: cilium-config optional: true +{% if cilium_version | regex_replace('v') is version('1.9', '<') %} image: "{{cilium_init_image_repo}}:{{cilium_init_image_tag}}" +{% else %} + image: "{{cilium_image_repo}}:{{cilium_image_tag}}" +{% endif %} imagePullPolicy: {{ k8s_image_pull_policy }} name: clean-cilium-state securityContext: @@ -289,6 +298,25 @@ spec: - name: cilium-ipsec-secrets secret: secretName: cilium-ipsec-keys +{% endif %} +{% if cilium_hubble_install %} + - name: hubble-tls + projected: + sources: + - secret: + name: hubble-server-certs + items: + - key: tls.crt + path: server.crt + - key: tls.key + path: server.key + optional: true + - configMap: + name: hubble-ca-cert + items: + - key: ca.crt + path: client-ca.crt + optional: true {% endif %} updateStrategy: rollingUpdate: diff --git a/roles/network_plugin/cilium/templates/hubble-config.yml.j2 b/roles/network_plugin/cilium/templates/hubble-config.yml.j2 new file mode 100644 index 000000000..d9723db03 --- /dev/null +++ b/roles/network_plugin/cilium/templates/hubble-config.yml.j2 @@ -0,0 +1,86 @@ +--- +# Source: cilium/templates/hubble-relay-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-relay-config + namespace: kube-system +data: + config.yaml: | + peer-service: unix:///var/run/cilium/hubble.sock + listen-address: :4245 + dial-timeout: + retry-timeout: + sort-buffer-len-max: + sort-buffer-drain-timeout: + tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt + tls-client-key-file: /var/lib/hubble-relay/tls/client.key + tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt + disable-server-tls: true +--- +# Source: cilium/templates/hubble-ui-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: hubble-ui-envoy + namespace: kube-system +data: + envoy.yaml: | + static_resources: + listeners: + - name: listener_hubble_ui + address: + socket_address: + address: 0.0.0.0 + port_value: 8081 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + config: + codec_type: auto + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ['*'] + routes: + - match: + prefix: '/api/' + route: + cluster: backend + max_grpc_timeout: 0s + prefix_rewrite: '/' + - match: + prefix: '/' + route: + cluster: frontend + cors: + allow_origin_string_match: + - prefix: '*' + allow_methods: GET, PUT, DELETE, POST, OPTIONS + allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout + max_age: '1728000' + expose_headers: grpc-status,grpc-message + http_filters: + - name: envoy.filters.http.grpc_web + - name: envoy.filters.http.cors + - name: envoy.filters.http.router + clusters: + - name: frontend + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + hosts: + - socket_address: + address: 127.0.0.1 + port_value: 8080 + - name: backend + connect_timeout: 0.25s + type: logical_dns + lb_policy: round_robin + http2_protocol_options: {} + hosts: + - socket_address: + address: 127.0.0.1 + port_value: 8090 diff --git a/roles/network_plugin/cilium/templates/hubble-cr.yml.j2 b/roles/network_plugin/cilium/templates/hubble-cr.yml.j2 new file mode 100644 index 000000000..4a95565d2 --- /dev/null +++ b/roles/network_plugin/cilium/templates/hubble-cr.yml.j2 @@ -0,0 +1,106 @@ +{% if cilium_hubble_tls_generate %} +--- +# Source: cilium/templates/hubble-generate-certs-clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hubble-generate-certs +rules: + - apiGroups: + - "" + resources: + - secrets + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - hubble-server-certs + - hubble-relay-client-certs + - hubble-relay-server-certs + verbs: + - update + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - hubble-ca-cert + verbs: + - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - hubble-ca-secret + verbs: + - get +{% endif %} +--- +# Source: cilium/templates/hubble-relay-clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-relay +rules: + - apiGroups: + - "" + resources: + - componentstatuses + - endpoints + - namespaces + - nodes + - pods + - services + verbs: + - get + - list + - watch +--- +# Source: cilium/templates/hubble-ui-clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui +rules: + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - componentstatuses + - endpoints + - namespaces + - nodes + - pods + - services + verbs: + - get + - list + - watch + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - apiGroups: + - cilium.io + resources: + - "*" + verbs: + - get + - list + - watch diff --git a/roles/network_plugin/cilium/templates/hubble-crb.yml.j2 b/roles/network_plugin/cilium/templates/hubble-crb.yml.j2 new file mode 100644 index 000000000..f033429ce --- /dev/null +++ b/roles/network_plugin/cilium/templates/hubble-crb.yml.j2 @@ -0,0 +1,44 @@ +{% if cilium_hubble_tls_generate %} +--- +# Source: cilium/templates/hubble-generate-certs-clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hubble-generate-certs +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-generate-certs +subjects: +- kind: ServiceAccount + name: hubble-generate-certs + namespace: kube-system +{% endif %} +--- +# Source: cilium/templates/hubble-relay-clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-relay +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-relay +subjects: +- kind: ServiceAccount + namespace: kube-system + name: hubble-relay +--- +# Source: cilium/templates/hubble-ui-clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: hubble-ui +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hubble-ui +subjects: +- kind: ServiceAccount + namespace: kube-system + name: hubble-ui diff --git a/roles/network_plugin/cilium/templates/hubble-cronjob.yml.j2 b/roles/network_plugin/cilium/templates/hubble-cronjob.yml.j2 new file mode 100644 index 000000000..6486cfd93 --- /dev/null +++ b/roles/network_plugin/cilium/templates/hubble-cronjob.yml.j2 @@ -0,0 +1,49 @@ +--- +# Source: cilium/templates/hubble-generate-certs-cronjob.yaml +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: hubble-generate-certs + namespace: kube-system + labels: + k8s-app: hubble-generate-certs +spec: + schedule: "0 0 1 */4 *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + metadata: + labels: + k8s-app: hubble-generate-certs + spec: + serviceAccount: hubble-generate-certs + serviceAccountName: hubble-generate-certs + containers: + - name: certgen + image: "{{ cilium_hubble_certgen_image_repo }}:{{ cilium_hubble_certgen_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - "/usr/bin/cilium-certgen" + # Because this is executed as a job, we pass the values as command + # line args instead of via config map. This allows users to inspect + # the values used in past runs by inspecting the completed pod. + args: + - "--cilium-namespace=kube-system" + - "--hubble-ca-reuse-secret=true" + - "--hubble-ca-secret-name=hubble-ca-secret" + - "--hubble-ca-generate=true" + - "--hubble-ca-validity-duration=94608000s" + - "--hubble-ca-config-map-create=true" + - "--hubble-ca-config-map-name=hubble-ca-cert" + - "--hubble-server-cert-generate=true" + - "--hubble-server-cert-common-name=*.default.hubble-grpc.cilium.io" + - "--hubble-server-cert-validity-duration=94608000s" + - "--hubble-server-cert-secret-name=hubble-server-certs" + - "--hubble-relay-client-cert-generate=true" + - "--hubble-relay-client-cert-validity-duration=94608000s" + - "--hubble-relay-client-cert-secret-name=hubble-relay-client-certs" + - "--hubble-relay-server-cert-generate=false" + hostNetwork: true + restartPolicy: OnFailure + ttlSecondsAfterFinished: 1800 diff --git a/roles/network_plugin/cilium/templates/hubble-deploy.yml.j2 b/roles/network_plugin/cilium/templates/hubble-deploy.yml.j2 new file mode 100644 index 000000000..43dd02bae --- /dev/null +++ b/roles/network_plugin/cilium/templates/hubble-deploy.yml.j2 @@ -0,0 +1,161 @@ +--- +# Source: cilium/templates/hubble-relay-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hubble-relay + labels: + k8s-app: hubble-relay + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: hubble-relay + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + labels: + k8s-app: hubble-relay + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "k8s-app" + operator: In + values: + - cilium + topologyKey: "kubernetes.io/hostname" + containers: + - name: hubble-relay + image: "{{ cilium_hubble_relay_image_repo }}:{{ cilium_hubble_relay_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - hubble-relay + args: + - serve + ports: + - name: grpc + containerPort: 4245 + readinessProbe: + tcpSocket: + port: grpc + livenessProbe: + tcpSocket: + port: grpc + volumeMounts: + - mountPath: /var/run/cilium + name: hubble-sock-dir + readOnly: true + - mountPath: /etc/hubble-relay + name: config + readOnly: true + - mountPath: /var/lib/hubble-relay/tls + name: tls + readOnly: true + restartPolicy: Always + serviceAccount: hubble-relay + serviceAccountName: hubble-relay + terminationGracePeriodSeconds: 0 + volumes: + - configMap: + name: hubble-relay-config + items: + - key: config.yaml + path: config.yaml + name: config + - hostPath: + path: /var/run/cilium + type: Directory + name: hubble-sock-dir + - projected: + sources: + - secret: + name: hubble-relay-client-certs + items: + - key: tls.crt + path: client.crt + - key: tls.key + path: client.key + - configMap: + name: hubble-ca-cert + items: + - key: ca.crt + path: hubble-server-ca.crt + name: tls +--- +# Source: cilium/templates/hubble-ui-deployment.yaml +kind: Deployment +apiVersion: apps/v1 +metadata: + namespace: kube-system + labels: + k8s-app: hubble-ui + name: hubble-ui +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: hubble-ui + template: + metadata: + annotations: + labels: + k8s-app: hubble-ui + spec: + securityContext: + runAsUser: 1001 + serviceAccount: hubble-ui + serviceAccountName: hubble-ui + containers: + - name: frontend + image: "{{ cilium_hubble_ui_image_repo }}:{{ cilium_hubble_ui_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + ports: + - containerPort: 8080 + name: http + resources: + {} + - name: backend + image: "{{ cilium_hubble_ui_backend_image_repo }}:{{ cilium_hubble_ui_backend_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: EVENTS_SERVER_PORT + value: "8090" + - name: FLOWS_API_ADDR + value: "hubble-relay:80" + ports: + - containerPort: 8090 + name: grpc + resources: + {} + - name: proxy + image: "{{ cilium_hubble_envoy_image_repo }}:{{ cilium_hubble_envoy_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + ports: + - containerPort: 8081 + name: http + resources: + {} + command: ["envoy"] + args: + [ + "-c", + "/etc/envoy.yaml", + "-l", + "info" + ] + volumeMounts: + - name: hubble-ui-envoy-yaml + mountPath: /etc/envoy.yaml + subPath: envoy.yaml + volumes: + - name: hubble-ui-envoy-yaml + configMap: + name: hubble-ui-envoy diff --git a/roles/network_plugin/cilium/templates/hubble-job.yml.j2 b/roles/network_plugin/cilium/templates/hubble-job.yml.j2 new file mode 100644 index 000000000..d4213bd39 --- /dev/null +++ b/roles/network_plugin/cilium/templates/hubble-job.yml.j2 @@ -0,0 +1,45 @@ +--- +# Source: cilium/templates/hubble-generate-certs-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: hubble-generate-certs + namespace: kube-system + labels: + k8s-app: hubble-generate-certs +spec: + template: + metadata: + labels: + k8s-app: hubble-generate-certs + spec: + serviceAccount: hubble-generate-certs + serviceAccountName: hubble-generate-certs + containers: + - name: certgen + image: "{{ cilium_hubble_certgen_image_repo }}:{{ cilium_hubble_certgen_image_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + command: + - "/usr/bin/cilium-certgen" + # Because this is executed as a job, we pass the values as command + # line args instead of via config map. This allows users to inspect + # the values used in past runs by inspecting the completed pod. + args: + - "--cilium-namespace=kube-system" + - "--hubble-ca-reuse-secret=true" + - "--hubble-ca-secret-name=hubble-ca-secret" + - "--hubble-ca-generate=true" + - "--hubble-ca-validity-duration=94608000s" + - "--hubble-ca-config-map-create=true" + - "--hubble-ca-config-map-name=hubble-ca-cert" + - "--hubble-server-cert-generate=true" + - "--hubble-server-cert-common-name=*.default.hubble-grpc.cilium.io" + - "--hubble-server-cert-validity-duration=94608000s" + - "--hubble-server-cert-secret-name=hubble-server-certs" + - "--hubble-relay-client-cert-generate=true" + - "--hubble-relay-client-cert-validity-duration=94608000s" + - "--hubble-relay-client-cert-secret-name=hubble-relay-client-certs" + - "--hubble-relay-server-cert-generate=false" + hostNetwork: true + restartPolicy: OnFailure + ttlSecondsAfterFinished: 1800 diff --git a/roles/network_plugin/cilium/templates/hubble-sa.yml.j2 b/roles/network_plugin/cilium/templates/hubble-sa.yml.j2 new file mode 100644 index 000000000..9b3203dbd --- /dev/null +++ b/roles/network_plugin/cilium/templates/hubble-sa.yml.j2 @@ -0,0 +1,23 @@ +{% if cilium_hubble_tls_generate %} +--- +# Source: cilium/templates/hubble-generate-certs-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hubble-generate-certs + namespace: kube-system +{% endif %} +--- +# Source: cilium/templates/hubble-relay-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hubble-relay + namespace: kube-system +--- +# Source: cilium/templates/hubble-ui-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hubble-ui + namespace: kube-system diff --git a/roles/network_plugin/cilium/templates/hubble-service.yml.j2 b/roles/network_plugin/cilium/templates/hubble-service.yml.j2 new file mode 100644 index 000000000..56dba7631 --- /dev/null +++ b/roles/network_plugin/cilium/templates/hubble-service.yml.j2 @@ -0,0 +1,58 @@ +{% if cilium_enable_prometheus or cilium_enable_hubble_metrics %} +--- +# Source: cilium/templates/cilium-agent-service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-metrics + namespace: kube-system + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: "9091" + labels: + k8s-app: hubble +spec: + clusterIP: None + type: ClusterIP + ports: + - name: hubble-metrics + port: 9091 + protocol: TCP + targetPort: hubble-metrics + selector: + k8s-app: cilium +{% endif %} +--- +# Source: cilium/templates/hubble-relay-service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-relay + namespace: kube-system + labels: + k8s-app: hubble-relay +spec: + type: ClusterIP + selector: + k8s-app: hubble-relay + ports: + - protocol: TCP + port: 80 + targetPort: 4245 +--- +# Source: cilium/templates/hubble-ui-service.yaml +kind: Service +apiVersion: v1 +metadata: + name: hubble-ui + labels: + k8s-app: hubble-ui + namespace: kube-system +spec: + selector: + k8s-app: hubble-ui + ports: + - name: http + port: 80 + targetPort: 8081 + type: ClusterIP