From caff539ccd91d22bd26c6cdba59da0225ee3bb0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Necatican=20Y=C4=B1ld=C4=B1r=C4=B1m?= Date: Sun, 16 Jan 2022 20:29:28 +0300 Subject: [PATCH] Add identity_allocation_mode support for Cilium (#8430) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Emin Aktaş Co-authored-by: Yasin Taha Erol Signed-off-by: necatican Co-authored-by: Emin Aktaş Co-authored-by: Yasin Taha Erol --- docs/cilium.md | 12 +++++++++++- .../sample/group_vars/k8s_cluster/k8s-net-cilium.yml | 1 + roles/network_plugin/cilium/defaults/main.yml | 8 ++++++++ roles/network_plugin/cilium/tasks/check.yml | 7 ++++++- roles/network_plugin/cilium/tasks/install.yml | 4 ++++ .../cilium/templates/cilium-config.yml.j2 | 4 ++++ .../cilium/templates/cilium-deploy.yml.j2 | 4 ++++ .../network_plugin/cilium/templates/cilium-ds.yml.j2 | 4 ++++ 8 files changed, 42 insertions(+), 2 deletions(-) diff --git a/docs/cilium.md b/docs/cilium.md index 548f87b78..4ce441e0b 100644 --- a/docs/cilium.md +++ b/docs/cilium.md @@ -15,7 +15,7 @@ balancer deployed by Kubespray and **only contacts the first master**. ## Choose Cilium version ```yml -cilium_version: v1.9.9 +cilium_version: v1.11.0 ``` ## Add variable to config @@ -29,6 +29,16 @@ cilium_config_extra_vars: enable-endpoint-routes: true ``` +## Change Identity Allocation Mode + +Cilium assigns an identity for each endpoint. This identity is used to enforce basic connectivity between endpoints. + +Cilium currently supports two different identity allocation modes: + +- "crd" stores identities in kubernetes as CRDs (custom resource definition). + - These can be queried with `kubectl get ciliumid` +- "kvstore" stores identities in an etcd kvstore. + ## Install Cilium Hubble k8s-net-cilium.yml: diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml b/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml index dcd841665..505aeb242 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml @@ -1,3 +1,4 @@ # see roles/network_plugin/cilium/defaults/main.yml # cilium_version: "v1.11.0" +# identity_allocation_mode: kvstore # kvstore or crd diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml index ce823f027..3f83aea99 100644 --- a/roles/network_plugin/cilium/defaults/main.yml +++ b/roles/network_plugin/cilium/defaults/main.yml @@ -6,6 +6,14 @@ cilium_mtu: "" cilium_enable_ipv4: true cilium_enable_ipv6: false +# Identity allocation mode selects how identities are shared between cilium +# nodes by setting how they are stored. The options are "crd" or "kvstore". +# - "crd" stores identities in kubernetes as CRDs (custom resource definition). +# These can be queried with: +# `kubectl get ciliumid` +# - "kvstore" stores identities in an etcd kvstore. +identity_allocation_mode: kvstore + # Etcd SSL dirs cilium_cert_dir: /etc/cilium/certs kube_etcd_cacert_file: ca.pem diff --git a/roles/network_plugin/cilium/tasks/check.yml b/roles/network_plugin/cilium/tasks/check.yml index 88ebfe958..2a9ddc7c1 100644 --- a/roles/network_plugin/cilium/tasks/check.yml +++ b/roles/network_plugin/cilium/tasks/check.yml @@ -6,4 +6,9 @@ msg: "cilium_ipsec_key should be defined to use cilium_ipsec_enabled" when: - cilium_ipsec_enabled - - cilium_tunnel_mode in ['vxlan'] \ No newline at end of file + - cilium_tunnel_mode in ['vxlan'] + +- name: Stop if bad Cilium identity allocation mode + assert: + that: identity_allocation_mode in ['crd', 'kvstore'] + msg: "identity_allocation_mode must be either 'crd' or 'kvstore'" diff --git a/roles/network_plugin/cilium/tasks/install.yml b/roles/network_plugin/cilium/tasks/install.yml index 8acd63390..6c402a936 100644 --- a/roles/network_plugin/cilium/tasks/install.yml +++ b/roles/network_plugin/cilium/tasks/install.yml @@ -13,6 +13,8 @@ mode: 0750 owner: root group: root + when: + - identity_allocation_mode == "kvstore" - name: Cilium | Link etcd certificates for cilium file: @@ -25,6 +27,8 @@ - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"} - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} + when: + - identity_allocation_mode == "kvstore" - name: Cilium | Create hubble dir file: diff --git a/roles/network_plugin/cilium/templates/cilium-config.yml.j2 b/roles/network_plugin/cilium/templates/cilium-config.yml.j2 index c9f4d26a6..7d6adfe59 100644 --- a/roles/network_plugin/cilium/templates/cilium-config.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-config.yml.j2 @@ -5,6 +5,9 @@ metadata: name: cilium-config namespace: kube-system data: + identity-allocation-mode: {{ identity_allocation_mode }} + +{% if identity_allocation_mode == "kvstore" %} # This etcd-config contains the etcd endpoints of your cluster. If you use # TLS please make sure you follow the tutorial in https://cilium.link/etcd-config etcd-config: |- @@ -29,6 +32,7 @@ data: # https://docs.cilium.io/en/latest/cmdref/kvstore/ kvstore: etcd kvstore-opt: '{"etcd.config": "/var/lib/etcd-config/etcd.config"}' +{% endif %} # If you want metrics enabled in all of your Cilium agents, set the port for # which the Cilium agents will have their metrics exposed. diff --git a/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2 b/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2 index 38d72b390..854d96022 100644 --- a/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2 @@ -135,12 +135,14 @@ spec: periodSeconds: 10 timeoutSeconds: 3 volumeMounts: +{% if identity_allocation_mode == "kvstore" %} - mountPath: /var/lib/etcd-config name: etcd-config-path readOnly: true - mountPath: "{{cilium_cert_dir}}" name: etcd-secrets readOnly: true +{% endif %} - mountPath: /tmp/cilium/config-map name: cilium-config-path readOnly: true @@ -153,6 +155,7 @@ spec: tolerations: - operator: Exists volumes: +{% if identity_allocation_mode == "kvstore" %} # To read the etcd config stored in config maps - configMap: defaultMode: 420 @@ -165,6 +168,7 @@ spec: - name: etcd-secrets hostPath: path: "{{cilium_cert_dir}}" +{% endif %} - configMap: name: cilium-config name: cilium-config-path diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 index e70fbb9f0..7dd601e31 100644 --- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 @@ -146,12 +146,14 @@ spec: mountPath: {{ cri_socket }} readOnly: true {% endif %} +{% if identity_allocation_mode == "kvstore" %} - mountPath: /var/lib/etcd-config name: etcd-config-path readOnly: true - mountPath: "{{cilium_cert_dir}}" name: etcd-secrets readOnly: true +{% endif %} - mountPath: /var/lib/cilium/clustermesh name: clustermesh-secrets readOnly: true @@ -270,6 +272,7 @@ spec: path: /run/xtables.lock type: FileOrCreate name: xtables-lock +{% if identity_allocation_mode == "kvstore" %} # To read the etcd config stored in config maps - configMap: defaultMode: 420 @@ -282,6 +285,7 @@ spec: - name: etcd-secrets hostPath: path: "{{cilium_cert_dir}}" +{% endif %} # To read the clustermesh configuration - name: clustermesh-secrets secret: