feat: upgrade cilium and add default variables (#9065)

Signed-off-by: eminaktas <eminaktas34@gmail.com>
Signed-off-by: Emin Aktas <emin.aktas@trendyol.com>

README.md

@@ -143,7 +143,7 @@ Note: Upstart/SysV init based OS types are not supported.
   - [cni-plugins](https://github.com/containernetworking/plugins) v1.1.1
   - [calico](https://github.com/projectcalico/calico) v3.23.1
   - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
-  - [cilium](https://github.com/cilium/cilium) v1.11.3
+  - [cilium](https://github.com/cilium/cilium) v1.11.6
   - [flannel](https://github.com/flannel-io/flannel) v0.17.0
   - [kube-ovn](https://github.com/alauda/kube-ovn) v1.9.2
   - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.5.0

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -1,19 +1,86 @@
-# see roles/network_plugin/cilium/defaults/main.yml
+---
+# cilium_version: "v1.11.6"
 
-# cilium_version: "v1.11.3"
+# Log-level
-# cilium_identity_allocation_mode: kvstore # kvstore or crd
+# cilium_debug: false
 
-# For adding and mounting extra volumes to the cilium operator
+# cilium_mtu: ""
-# cilium_operator_extra_volumes: []
+# cilium_enable_ipv4: true
-# cilium_operator_extra_volume_mounts: []
+# cilium_enable_ipv6: false
 
-# Name of the cluster. Only relevant when building a mesh of clusters.
+# Cilium agent health port
-# cilium_cluster_name: default
+# cilium_agent_health_port: "9879"
+
+# Identity allocation mode selects how identities are shared between cilium
+# nodes by setting how they are stored. The options are "crd" or "kvstore".
+# - "crd" stores identities in kubernetes as CRDs (custom resource definition).
+#   These can be queried with:
+#     `kubectl get ciliumid`
+# - "kvstore" stores identities in an etcd kvstore.
+# - In order to support External Workloads, "crd" is required
+#   - Ref: https://docs.cilium.io/en/stable/gettingstarted/external-workloads/#setting-up-support-for-external-workloads-beta
+# - KVStore operations are only required when cilium-operator is running with any of the below options:
+#   - --synchronize-k8s-services
+#   - --synchronize-k8s-nodes
+#   - --identity-allocation-mode=kvstore
+#   - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations
+# cilium_identity_allocation_mode: kvstore
+
+# Etcd SSL dirs
+# cilium_cert_dir: /etc/cilium/certs
+# kube_etcd_cacert_file: ca.pem
+# kube_etcd_cert_file: cert.pem
+# kube_etcd_key_file: cert-key.pem
+
+# Limits for apps
+# cilium_memory_limit: 500M
+# cilium_cpu_limit: 500m
+# cilium_memory_requests: 64M
+# cilium_cpu_requests: 100m
+
+# Overlay Network Mode
+# cilium_tunnel_mode: vxlan
+# Optional features
+# cilium_enable_prometheus: false
+# Enable if you want to make use of hostPort mappings
+# cilium_enable_portmap: false
+# Monitor aggregation level (none/low/medium/maximum)
+# cilium_monitor_aggregation: medium
+# The monitor aggregation flags determine which TCP flags which, upon the
+# first observation, cause monitor notifications to be generated.
+#
+# Only effective when monitor aggregation is set to "medium" or higher.
+# cilium_monitor_aggregation_flags: "all"
+# Kube Proxy Replacement mode (strict/probe/partial)
+# cilium_kube_proxy_replacement: probe
+
+# If upgrading from Cilium < 1.5, you may want to override some of these options
+# to prevent service disruptions. See also:
+# http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
+# cilium_preallocate_bpf_maps: false
+
+# `cilium_tofqdns_enable_poller` is deprecated in 1.8, removed in 1.9
+# cilium_tofqdns_enable_poller: false
+
+# `cilium_enable_legacy_services` is deprecated in 1.6, removed in 1.9
+# cilium_enable_legacy_services: false
 
 # Unique ID of the cluster. Must be unique across all conneted clusters and
 # in the range of 1 and 255. Only relevant when building a mesh of clusters.
 # This value is not defined by default
-# cluster-id:
+# cilium_cluster_id:
+
+# Deploy cilium even if kube_network_plugin is not cilium.
+# This enables to deploy cilium alongside another CNI to replace kube-proxy.
+# cilium_deploy_additionally: false
+
+# Auto direct nodes routes can be used to advertise pods routes in your cluster
+# without any tunelling (with `cilium_tunnel_mode` sets to `disabled`).
+# This works only if you have a L2 connectivity between all your nodes.
+# You wil also have to specify the variable `cilium_native_routing_cidr` to
+# make this work. Please refer to the cilium documentation for more
+# information about this kind of setups.
+# cilium_auto_direct_node_routes: false
 
 # Allows to explicitly specify the IPv4 CIDR for native routing.
 # When specified, Cilium assumes networking for this CIDR is preconfigured and
@@ -30,7 +97,6 @@
 # Allows to explicitly specify the IPv6 CIDR for native routing.
 # cilium_native_routing_cidr_ipv6: ""
 
-# Encryption
 # Enable transparent network encryption.
 # cilium_encryption_enabled: false
 
@@ -40,8 +106,139 @@
 
 # Enable encryption for pure node to node traffic.
 # This option is only effective when `cilium_encryption_type` is set to `ipsec`.
-# cilium_ipsec_node_encryption: "false"
+# cilium_ipsec_node_encryption: false
 
-# Enables the fallback to the user-space implementation.
+# If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation.
+# When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard,
+# it will fallback on the wireguard-go user-space implementation of WireGuard.
 # This option is only effective when `cilium_encryption_type` is set to `wireguard`.
-# cilium_wireguard_userspace_fallback: "false"
+# cilium_wireguard_userspace_fallback: false
+
+# IP Masquerade Agent
+# https://docs.cilium.io/en/stable/concepts/networking/masquerading/
+# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded
+# cilium_ip_masq_agent_enable: false
+### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded
+# cilium_non_masquerade_cidrs:
+#   - 10.0.0.0/8
+#   - 172.16.0.0/12
+#   - 192.168.0.0/16
+#   - 100.64.0.0/10
+#   - 192.0.0.0/24
+#   - 192.0.2.0/24
+#   - 192.88.99.0/24
+#   - 198.18.0.0/15
+#   - 198.51.100.0/24
+#   - 203.0.113.0/24
+#   - 240.0.0.0/4
+### Indicates whether to masquerade traffic to the link local prefix.
+### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list.
+# cilium_masq_link_local: false
+### A time interval at which the agent attempts to reload config from disk
+# cilium_ip_masq_resync_interval: 60s
+
+# Hubble
+### Enable Hubble without install
+# cilium_enable_hubble: false
+### Enable Hubble Metrics
+# cilium_enable_hubble_metrics: false
+### if cilium_enable_hubble_metrics: true
+# cilium_hubble_metrics: {}
+# - dns
+# - drop
+# - tcp
+# - flow
+# - icmp
+# - http
+### Enable Hubble install
+# cilium_hubble_install: false
+### Enable auto generate certs if cilium_hubble_install: true
+# cilium_hubble_tls_generate: false
+
+# IP address management mode for v1.9+.
+# https://docs.cilium.io/en/v1.9/concepts/networking/ipam/
+# cilium_ipam_mode: kubernetes
+
+# Extra arguments for the Cilium agent
+# cilium_agent_custom_args: []
+
+# For adding and mounting extra volumes to the cilium agent
+# cilium_agent_extra_volumes: []
+# cilium_agent_extra_volume_mounts: []
+
+# cilium_agent_extra_env_vars: []
+
+# cilium_operator_replicas: 2
+
+# The address at which the cillium operator bind health check api
+# cilium_operator_api_serve_addr: "127.0.0.1:9234"
+
+## A dictionary of extra config variables to add to cilium-config, formatted like:
+##  cilium_config_extra_vars:
+##    var1: "value1"
+##    var2: "value2"
+# cilium_config_extra_vars: {}
+
+# For adding and mounting extra volumes to the cilium operator
+# cilium_operator_extra_volumes: []
+# cilium_operator_extra_volume_mounts: []
+
+# Extra arguments for the Cilium Operator
+# cilium_operator_custom_args: []
+
+# Name of the cluster. Only relevant when building a mesh of clusters.
+# cilium_cluster_name: default
+
+# Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`.
+# This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime.
+# Available for Cilium v1.10 and up.
+# cilium_cni_exclusive: true
+
+# Configure the log file for CNI logging with retention policy of 7 days.
+# Disable CNI file logging by setting this field to empty explicitly.
+# Available for Cilium v1.12 and up.
+# cilium_cni_log_file: "/var/run/cilium/cilium-cni.log"
+
+# -- Configure cgroup related configuration
+# -- Enable auto mount of cgroup2 filesystem.
+# When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at
+# `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod.
+# If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted
+# cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the
+# volume will be mounted inside the cilium agent pod at the same path.
+# Available for Cilium v1.11 and up
+# cilium_cgroup_auto_mount: true
+# -- Configure cgroup root where cgroup2 filesystem is mounted on the host
+# cilium_cgroup_host_root: "/run/cilium/cgroupv2"
+
+# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
+# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
+# cilium_bpf_map_dynamic_size_ratio: "0.0"
+
+# -- Enables masquerading of IPv4 traffic leaving the node from endpoints.
+# Available for Cilium v1.10 and up
+# cilium_enable_ipv4_masquerade: true
+# -- Enables masquerading of IPv6 traffic leaving the node from endpoints.
+# Available for Cilium v1.10 and up
+# cilium_enable_ipv6_masquerade: true
+
+# -- Enable native IP masquerade support in eBPF
+# cilium_enable_bpf_masquerade: false
+
+# -- Configure whether direct routing mode should route traffic via
+# host stack (true) or directly and more efficiently out of BPF (false) if
+# the kernel supports it. The latter has the implication that it will also
+# bypass netfilter in the host namespace.
+# cilium_enable_host_legacy_routing: true
+
+# -- Enable use of the remote node identity.
+# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity
+# cilium_enable_remote_node_identity: true
+
+# -- Enable the use of well-known identities.
+# cilium_enable_well_known_identities: false
+
+# cilium_enable_bpf_clock_probe: true
+
+# -- Whether to enable CNP status updates.
+# cilium_disable_cnp_status_updates: true

roles/download/defaults/main.yml

@@ -110,7 +110,7 @@ flannel_cni_version: "v1.0.1"
 cni_version: "v1.1.1"
 weave_version: 2.8.1
 pod_infra_version: "3.6"
-cilium_version: "v1.11.3"
+cilium_version: "v1.11.6"
 kube_ovn_version: "v1.9.2"
 kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}"
 kube_router_version: "v1.5.0"

roles/network_plugin/cilium/defaults/main.yml

@@ -6,6 +6,9 @@ cilium_mtu: ""
 cilium_enable_ipv4: true
 cilium_enable_ipv6: false
 
+# Cilium agent health port
+cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879{%- else -%}9876{%- endif -%}"
+
 # Identity allocation mode selects how identities are shared between cilium
 # nodes by setting how they are stored. The options are "crd" or "kvstore".
 # - "crd" stores identities in kubernetes as CRDs (custom resource definition).
@@ -91,13 +94,13 @@ cilium_encryption_type: "ipsec"
 
 # Enable encryption for pure node to node traffic.
 # This option is only effective when `cilium_encryption_type` is set to `ipsec`.
-cilium_ipsec_node_encryption: "false"
+cilium_ipsec_node_encryption: false
 
 # If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation.
 # When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard,
 # it will fallback on the wireguard-go user-space implementation of WireGuard.
 # This option is only effective when `cilium_encryption_type` is set to `wireguard`.
-cilium_wireguard_userspace_fallback: "false"
+cilium_wireguard_userspace_fallback: false
 
 # IP Masquerade Agent
 # https://docs.cilium.io/en/stable/concepts/networking/masquerading/
@@ -153,6 +156,8 @@ cilium_agent_extra_volume_mounts: []
 
 cilium_agent_extra_env_vars: []
 
+cilium_operator_replicas: 2
+
 # The address at which the cillium operator bind health check api
 cilium_operator_api_serve_addr: "127.0.0.1:9234"
 
@@ -175,9 +180,59 @@ cilium_cluster_name: default
 # Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`.
 # This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime.
 # Available for Cilium v1.10 and up.
-cilium_cni_exclusive: "true"
+cilium_cni_exclusive: true
 
 # Configure the log file for CNI logging with retention policy of 7 days.
 # Disable CNI file logging by setting this field to empty explicitly.
 # Available for Cilium v1.12 and up.
 cilium_cni_log_file: "/var/run/cilium/cilium-cni.log"
+
+# -- Configure cgroup related configuration
+# -- Enable auto mount of cgroup2 filesystem.
+# When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at
+# `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod.
+# If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted
+# cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the
+# volume will be mounted inside the cilium agent pod at the same path.
+# Available for Cilium v1.11 and up
+cilium_cgroup_auto_mount: true
+# -- Configure cgroup root where cgroup2 filesystem is mounted on the host
+cilium_cgroup_host_root: "/run/cilium/cgroupv2"
+
+# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
+# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
+cilium_bpf_map_dynamic_size_ratio: "{%- if cilium_version | regex_replace('v') is version('1.8', '>=') -%}0.0025{%- else -%}0.0{%- endif -%}"
+
+# -- Enables masquerading of IPv4 traffic leaving the node from endpoints.
+# Available for Cilium v1.10 and up
+cilium_enable_ipv4_masquerade: true
+# -- Enables masquerading of IPv6 traffic leaving the node from endpoints.
+# Available for Cilium v1.10 and up
+cilium_enable_ipv6_masquerade: true
+
+# -- Enable native IP masquerade support in eBPF
+cilium_enable_bpf_masquerade: false
+
+# -- Configure whether direct routing mode should route traffic via
+# host stack (true) or directly and more efficiently out of BPF (false) if
+# the kernel supports it. The latter has the implication that it will also
+# bypass netfilter in the host namespace.
+cilium_enable_host_legacy_routing: true
+
+# -- Enable use of the remote node identity.
+# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity
+cilium_enable_remote_node_identity: true
+
+# -- Enable the use of well-known identities.
+cilium_enable_well_known_identities: false
+
+# The monitor aggregation flags determine which TCP flags which, upon the
+# first observation, cause monitor notifications to be generated.
+#
+# Only effective when monitor aggregation is set to "medium" or higher.
+cilium_monitor_aggregation_flags: "all"
+
+cilium_enable_bpf_clock_probe: true
+
+# -- Whether to enable CNP status updates.
+cilium_disable_cnp_status_updates: true

roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2

@@ -8,7 +8,7 @@ metadata:
   name: cilium-operator
   namespace: kube-system
 spec:
-  replicas: 1
+  replicas: {{ cilium_operator_replicas }}
   selector:
     matchLabels:
       io.cilium/app: operator

roles/network_plugin/cilium/templates/cilium/config.yml.j2

@@ -232,6 +232,33 @@ data:
   ipam: "{{ cilium_ipam_mode }}"
 {% endif %}
 
+  agent-health-port: "{{ cilium_agent_health_port }}"
+
+{% if cilium_version | regex_replace('v') is version('1.11', '>=') and cilium_cgroup_host_root != '' %}
+  cgroup-root: "{{ cilium_cgroup_host_root }}"
+{% endif %}
+
+  bpf-map-dynamic-size-ratio: "{{ cilium_bpf_map_dynamic_size_ratio }}"
+
+{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
+  enable-ipv4-masquerade: "{{ cilium_enable_ipv4_masquerade }}"
+  enable-ipv6-masquerade: "{{ cilium_enable_ipv6_masquerade }}"
+{% endif %}
+
+  enable-bpf-masquerade: "{{ cilium_enable_bpf_masquerade }}"
+
+  enable-host-legacy-routing: "{{ cilium_enable_host_legacy_routing }}"
+
+  enable-remote-node-identity: "{{ cilium_enable_remote_node_identity }}"
+
+  enable-well-known-identities: "{{ cilium_enable_well_known_identities }}"
+
+  monitor-aggregation-flags: "{{ cilium_monitor_aggregation_flags }}"
+
+  enable-bpf-clock-probe: "{{ cilium_enable_bpf_clock_probe }}"
+
+  disable-cnp-status-updates: "{{ cilium_disable_cnp_status_updates }}"
+
 {% if cilium_ip_masq_agent_enable %}
 ---
 apiVersion: v1

roles/network_plugin/cilium/templates/cilium/ds.yml.j2

@@ -96,11 +96,7 @@ spec:
           httpGet:
             host: '127.0.0.1'
             path: /healthz
-{% if cilium_version | regex_replace('v') is version('1.11.6', '>=') %}
+            port: {{ cilium_agent_health_port }}
-            port: 9879
-{% else %}
-            port: 9876
-{% endif %}
             scheme: HTTP
             httpHeaders:
             - name: "brief"
@@ -112,11 +108,7 @@ spec:
           httpGet:
             host: '127.0.0.1'
             path: /healthz
-{% if cilium_version | regex_replace('v') is version('1.11.6', '>=') %}
+            port: {{ cilium_agent_health_port }}
-            port: 9879
-{% else %}
-            port: 9876
-{% endif %}
             scheme: HTTP
             httpHeaders:
             - name: "brief"
@@ -129,11 +121,7 @@ spec:
           httpGet:
             host: 127.0.0.1
             path: /healthz
-{% if cilium_version | regex_replace('v') is version('1.11.6', '>=') %}
+            port: {{ cilium_agent_health_port }}
-            port: 9879
-{% else %}
-            port: 9876
-{% endif %}
             scheme: HTTP
             httpHeaders:
             - name: "brief"
@@ -228,6 +216,35 @@ spec:
 {% endif %}
       hostNetwork: true
       initContainers:
+{% if cilium_version | regex_replace('v') is version('1.11', '>=') and cilium_cgroup_auto_mount %}
+      - name: mount-cgroup
+        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        env:
+        - name: CGROUP_ROOT
+          value: {{ cilium_cgroup_host_root }}
+        - name: BIN_PATH
+          value: /opt/cni/bin
+        command:
+        - sh
+        - -ec
+        # The statically linked Go program binary is invoked to avoid any
+        # dependency on utilities like sh and mount that can be missing on certain
+        # distros installed on the underlying host. Copy the binary to the
+        # same directory where we install cilium cni plugin so that exec permissions
+        # are available.
+        - |
+          cp /usr/bin/cilium-mount /hostbin/cilium-mount;
+          nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT;
+          rm /hostbin/cilium-mount
+        volumeMounts:
+        - name: hostproc
+          mountPath: /hostproc
+        - name: cni-path
+          mountPath: /hostbin
+        securityContext:
+          privileged: true
+{% endif %}
       - command:
         - /init-container.sh
         env:
@@ -276,7 +293,7 @@ spec:
 {% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
           # Required to mount cgroup filesystem from the host to cilium agent pod
         - name: cilium-cgroup
-          mountPath: /run/cilium/cgroupv2
+          mountPath: {{ cilium_cgroup_host_root }}
           mountPropagation: HostToContainer
 {% endif %}
         - mountPath: /var/run/cilium
@@ -334,7 +351,7 @@ spec:
       # To keep state between restarts / upgrades for cgroup2 filesystem
       - name: cilium-cgroup
         hostPath:
-          path: /run/cilium/cgroupv2
+          path: {{ cilium_cgroup_host_root }}
           type: DirectoryOrCreate
 {% endif %}
         # To install cilium cni plugin in the host