diff --git a/README.md b/README.md index 2c5d1ed25..b55c20579 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ Note: Upstart/SysV init based OS types are not supported. - [cni-plugins](https://github.com/containernetworking/plugins) v1.1.1 - [calico](https://github.com/projectcalico/calico) v3.23.1 - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) - - [cilium](https://github.com/cilium/cilium) v1.11.3 + - [cilium](https://github.com/cilium/cilium) v1.11.6 - [flannel](https://github.com/flannel-io/flannel) v0.17.0 - [kube-ovn](https://github.com/alauda/kube-ovn) v1.9.2 - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.5.0 diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml b/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml index 922f42881..235f4647f 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml @@ -1,19 +1,86 @@ -# see roles/network_plugin/cilium/defaults/main.yml +--- +# cilium_version: "v1.11.6" -# cilium_version: "v1.11.3" -# cilium_identity_allocation_mode: kvstore # kvstore or crd +# Log-level +# cilium_debug: false -# For adding and mounting extra volumes to the cilium operator -# cilium_operator_extra_volumes: [] -# cilium_operator_extra_volume_mounts: [] +# cilium_mtu: "" +# cilium_enable_ipv4: true +# cilium_enable_ipv6: false -# Name of the cluster. Only relevant when building a mesh of clusters. -# cilium_cluster_name: default +# Cilium agent health port +# cilium_agent_health_port: "9879" + +# Identity allocation mode selects how identities are shared between cilium +# nodes by setting how they are stored. The options are "crd" or "kvstore". +# - "crd" stores identities in kubernetes as CRDs (custom resource definition). +# These can be queried with: +# `kubectl get ciliumid` +# - "kvstore" stores identities in an etcd kvstore. +# - In order to support External Workloads, "crd" is required +# - Ref: https://docs.cilium.io/en/stable/gettingstarted/external-workloads/#setting-up-support-for-external-workloads-beta +# - KVStore operations are only required when cilium-operator is running with any of the below options: +# - --synchronize-k8s-services +# - --synchronize-k8s-nodes +# - --identity-allocation-mode=kvstore +# - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations +# cilium_identity_allocation_mode: kvstore + +# Etcd SSL dirs +# cilium_cert_dir: /etc/cilium/certs +# kube_etcd_cacert_file: ca.pem +# kube_etcd_cert_file: cert.pem +# kube_etcd_key_file: cert-key.pem + +# Limits for apps +# cilium_memory_limit: 500M +# cilium_cpu_limit: 500m +# cilium_memory_requests: 64M +# cilium_cpu_requests: 100m + +# Overlay Network Mode +# cilium_tunnel_mode: vxlan +# Optional features +# cilium_enable_prometheus: false +# Enable if you want to make use of hostPort mappings +# cilium_enable_portmap: false +# Monitor aggregation level (none/low/medium/maximum) +# cilium_monitor_aggregation: medium +# The monitor aggregation flags determine which TCP flags which, upon the +# first observation, cause monitor notifications to be generated. +# +# Only effective when monitor aggregation is set to "medium" or higher. +# cilium_monitor_aggregation_flags: "all" +# Kube Proxy Replacement mode (strict/probe/partial) +# cilium_kube_proxy_replacement: probe + +# If upgrading from Cilium < 1.5, you may want to override some of these options +# to prevent service disruptions. See also: +# http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action +# cilium_preallocate_bpf_maps: false + +# `cilium_tofqdns_enable_poller` is deprecated in 1.8, removed in 1.9 +# cilium_tofqdns_enable_poller: false + +# `cilium_enable_legacy_services` is deprecated in 1.6, removed in 1.9 +# cilium_enable_legacy_services: false # Unique ID of the cluster. Must be unique across all conneted clusters and # in the range of 1 and 255. Only relevant when building a mesh of clusters. # This value is not defined by default -# cluster-id: +# cilium_cluster_id: + +# Deploy cilium even if kube_network_plugin is not cilium. +# This enables to deploy cilium alongside another CNI to replace kube-proxy. +# cilium_deploy_additionally: false + +# Auto direct nodes routes can be used to advertise pods routes in your cluster +# without any tunelling (with `cilium_tunnel_mode` sets to `disabled`). +# This works only if you have a L2 connectivity between all your nodes. +# You wil also have to specify the variable `cilium_native_routing_cidr` to +# make this work. Please refer to the cilium documentation for more +# information about this kind of setups. +# cilium_auto_direct_node_routes: false # Allows to explicitly specify the IPv4 CIDR for native routing. # When specified, Cilium assumes networking for this CIDR is preconfigured and @@ -30,7 +97,6 @@ # Allows to explicitly specify the IPv6 CIDR for native routing. # cilium_native_routing_cidr_ipv6: "" -# Encryption # Enable transparent network encryption. # cilium_encryption_enabled: false @@ -40,8 +106,139 @@ # Enable encryption for pure node to node traffic. # This option is only effective when `cilium_encryption_type` is set to `ipsec`. -# cilium_ipsec_node_encryption: "false" +# cilium_ipsec_node_encryption: false -# Enables the fallback to the user-space implementation. +# If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation. +# When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard, +# it will fallback on the wireguard-go user-space implementation of WireGuard. # This option is only effective when `cilium_encryption_type` is set to `wireguard`. -# cilium_wireguard_userspace_fallback: "false" +# cilium_wireguard_userspace_fallback: false + +# IP Masquerade Agent +# https://docs.cilium.io/en/stable/concepts/networking/masquerading/ +# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded +# cilium_ip_masq_agent_enable: false +### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded +# cilium_non_masquerade_cidrs: +# - 10.0.0.0/8 +# - 172.16.0.0/12 +# - 192.168.0.0/16 +# - 100.64.0.0/10 +# - 192.0.0.0/24 +# - 192.0.2.0/24 +# - 192.88.99.0/24 +# - 198.18.0.0/15 +# - 198.51.100.0/24 +# - 203.0.113.0/24 +# - 240.0.0.0/4 +### Indicates whether to masquerade traffic to the link local prefix. +### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list. +# cilium_masq_link_local: false +### A time interval at which the agent attempts to reload config from disk +# cilium_ip_masq_resync_interval: 60s + +# Hubble +### Enable Hubble without install +# cilium_enable_hubble: false +### Enable Hubble Metrics +# cilium_enable_hubble_metrics: false +### if cilium_enable_hubble_metrics: true +# cilium_hubble_metrics: {} +# - dns +# - drop +# - tcp +# - flow +# - icmp +# - http +### Enable Hubble install +# cilium_hubble_install: false +### Enable auto generate certs if cilium_hubble_install: true +# cilium_hubble_tls_generate: false + +# IP address management mode for v1.9+. +# https://docs.cilium.io/en/v1.9/concepts/networking/ipam/ +# cilium_ipam_mode: kubernetes + +# Extra arguments for the Cilium agent +# cilium_agent_custom_args: [] + +# For adding and mounting extra volumes to the cilium agent +# cilium_agent_extra_volumes: [] +# cilium_agent_extra_volume_mounts: [] + +# cilium_agent_extra_env_vars: [] + +# cilium_operator_replicas: 2 + +# The address at which the cillium operator bind health check api +# cilium_operator_api_serve_addr: "127.0.0.1:9234" + +## A dictionary of extra config variables to add to cilium-config, formatted like: +## cilium_config_extra_vars: +## var1: "value1" +## var2: "value2" +# cilium_config_extra_vars: {} + +# For adding and mounting extra volumes to the cilium operator +# cilium_operator_extra_volumes: [] +# cilium_operator_extra_volume_mounts: [] + +# Extra arguments for the Cilium Operator +# cilium_operator_custom_args: [] + +# Name of the cluster. Only relevant when building a mesh of clusters. +# cilium_cluster_name: default + +# Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. +# This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime. +# Available for Cilium v1.10 and up. +# cilium_cni_exclusive: true + +# Configure the log file for CNI logging with retention policy of 7 days. +# Disable CNI file logging by setting this field to empty explicitly. +# Available for Cilium v1.12 and up. +# cilium_cni_log_file: "/var/run/cilium/cilium-cni.log" + +# -- Configure cgroup related configuration +# -- Enable auto mount of cgroup2 filesystem. +# When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at +# `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod. +# If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted +# cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the +# volume will be mounted inside the cilium agent pod at the same path. +# Available for Cilium v1.11 and up +# cilium_cgroup_auto_mount: true +# -- Configure cgroup root where cgroup2 filesystem is mounted on the host +# cilium_cgroup_host_root: "/run/cilium/cgroupv2" + +# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic +# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. +# cilium_bpf_map_dynamic_size_ratio: "0.0" + +# -- Enables masquerading of IPv4 traffic leaving the node from endpoints. +# Available for Cilium v1.10 and up +# cilium_enable_ipv4_masquerade: true +# -- Enables masquerading of IPv6 traffic leaving the node from endpoints. +# Available for Cilium v1.10 and up +# cilium_enable_ipv6_masquerade: true + +# -- Enable native IP masquerade support in eBPF +# cilium_enable_bpf_masquerade: false + +# -- Configure whether direct routing mode should route traffic via +# host stack (true) or directly and more efficiently out of BPF (false) if +# the kernel supports it. The latter has the implication that it will also +# bypass netfilter in the host namespace. +# cilium_enable_host_legacy_routing: true + +# -- Enable use of the remote node identity. +# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity +# cilium_enable_remote_node_identity: true + +# -- Enable the use of well-known identities. +# cilium_enable_well_known_identities: false + +# cilium_enable_bpf_clock_probe: true + +# -- Whether to enable CNP status updates. +# cilium_disable_cnp_status_updates: true diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 7d35e47e6..d6080634b 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -110,7 +110,7 @@ flannel_cni_version: "v1.0.1" cni_version: "v1.1.1" weave_version: 2.8.1 pod_infra_version: "3.6" -cilium_version: "v1.11.3" +cilium_version: "v1.11.6" kube_ovn_version: "v1.9.2" kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}" kube_router_version: "v1.5.0" diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml index 294b0b0ea..e244735d9 100644 --- a/roles/network_plugin/cilium/defaults/main.yml +++ b/roles/network_plugin/cilium/defaults/main.yml @@ -6,6 +6,9 @@ cilium_mtu: "" cilium_enable_ipv4: true cilium_enable_ipv6: false +# Cilium agent health port +cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879{%- else -%}9876{%- endif -%}" + # Identity allocation mode selects how identities are shared between cilium # nodes by setting how they are stored. The options are "crd" or "kvstore". # - "crd" stores identities in kubernetes as CRDs (custom resource definition). @@ -91,13 +94,13 @@ cilium_encryption_type: "ipsec" # Enable encryption for pure node to node traffic. # This option is only effective when `cilium_encryption_type` is set to `ipsec`. -cilium_ipsec_node_encryption: "false" +cilium_ipsec_node_encryption: false # If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation. # When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard, # it will fallback on the wireguard-go user-space implementation of WireGuard. # This option is only effective when `cilium_encryption_type` is set to `wireguard`. -cilium_wireguard_userspace_fallback: "false" +cilium_wireguard_userspace_fallback: false # IP Masquerade Agent # https://docs.cilium.io/en/stable/concepts/networking/masquerading/ @@ -153,6 +156,8 @@ cilium_agent_extra_volume_mounts: [] cilium_agent_extra_env_vars: [] +cilium_operator_replicas: 2 + # The address at which the cillium operator bind health check api cilium_operator_api_serve_addr: "127.0.0.1:9234" @@ -175,9 +180,59 @@ cilium_cluster_name: default # Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`. # This ensures no Pods can be scheduled using other CNI plugins during Cilium agent downtime. # Available for Cilium v1.10 and up. -cilium_cni_exclusive: "true" +cilium_cni_exclusive: true # Configure the log file for CNI logging with retention policy of 7 days. # Disable CNI file logging by setting this field to empty explicitly. # Available for Cilium v1.12 and up. cilium_cni_log_file: "/var/run/cilium/cilium-cni.log" + +# -- Configure cgroup related configuration +# -- Enable auto mount of cgroup2 filesystem. +# When `cilium_cgroup_auto_mount` is enabled, cgroup2 filesystem is mounted at +# `cilium_cgroup_host_root` path on the underlying host and inside the cilium agent pod. +# If users disable `cilium_cgroup_auto_mount`, it's expected that users have mounted +# cgroup2 filesystem at the specified `cilium_cgroup_auto_mount` volume, and then the +# volume will be mounted inside the cilium agent pod at the same path. +# Available for Cilium v1.11 and up +cilium_cgroup_auto_mount: true +# -- Configure cgroup root where cgroup2 filesystem is mounted on the host +cilium_cgroup_host_root: "/run/cilium/cgroupv2" + +# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic +# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. +cilium_bpf_map_dynamic_size_ratio: "{%- if cilium_version | regex_replace('v') is version('1.8', '>=') -%}0.0025{%- else -%}0.0{%- endif -%}" + +# -- Enables masquerading of IPv4 traffic leaving the node from endpoints. +# Available for Cilium v1.10 and up +cilium_enable_ipv4_masquerade: true +# -- Enables masquerading of IPv6 traffic leaving the node from endpoints. +# Available for Cilium v1.10 and up +cilium_enable_ipv6_masquerade: true + +# -- Enable native IP masquerade support in eBPF +cilium_enable_bpf_masquerade: false + +# -- Configure whether direct routing mode should route traffic via +# host stack (true) or directly and more efficiently out of BPF (false) if +# the kernel supports it. The latter has the implication that it will also +# bypass netfilter in the host namespace. +cilium_enable_host_legacy_routing: true + +# -- Enable use of the remote node identity. +# ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity +cilium_enable_remote_node_identity: true + +# -- Enable the use of well-known identities. +cilium_enable_well_known_identities: false + +# The monitor aggregation flags determine which TCP flags which, upon the +# first observation, cause monitor notifications to be generated. +# +# Only effective when monitor aggregation is set to "medium" or higher. +cilium_monitor_aggregation_flags: "all" + +cilium_enable_bpf_clock_probe: true + +# -- Whether to enable CNP status updates. +cilium_disable_cnp_status_updates: true diff --git a/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2 b/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2 index a5813d3d4..ab8a31926 100644 --- a/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2 @@ -8,7 +8,7 @@ metadata: name: cilium-operator namespace: kube-system spec: - replicas: 1 + replicas: {{ cilium_operator_replicas }} selector: matchLabels: io.cilium/app: operator diff --git a/roles/network_plugin/cilium/templates/cilium/config.yml.j2 b/roles/network_plugin/cilium/templates/cilium/config.yml.j2 index 8431d7e27..75232991f 100644 --- a/roles/network_plugin/cilium/templates/cilium/config.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium/config.yml.j2 @@ -232,6 +232,33 @@ data: ipam: "{{ cilium_ipam_mode }}" {% endif %} + agent-health-port: "{{ cilium_agent_health_port }}" + +{% if cilium_version | regex_replace('v') is version('1.11', '>=') and cilium_cgroup_host_root != '' %} + cgroup-root: "{{ cilium_cgroup_host_root }}" +{% endif %} + + bpf-map-dynamic-size-ratio: "{{ cilium_bpf_map_dynamic_size_ratio }}" + +{% if cilium_version | regex_replace('v') is version('1.10', '>=') %} + enable-ipv4-masquerade: "{{ cilium_enable_ipv4_masquerade }}" + enable-ipv6-masquerade: "{{ cilium_enable_ipv6_masquerade }}" +{% endif %} + + enable-bpf-masquerade: "{{ cilium_enable_bpf_masquerade }}" + + enable-host-legacy-routing: "{{ cilium_enable_host_legacy_routing }}" + + enable-remote-node-identity: "{{ cilium_enable_remote_node_identity }}" + + enable-well-known-identities: "{{ cilium_enable_well_known_identities }}" + + monitor-aggregation-flags: "{{ cilium_monitor_aggregation_flags }}" + + enable-bpf-clock-probe: "{{ cilium_enable_bpf_clock_probe }}" + + disable-cnp-status-updates: "{{ cilium_disable_cnp_status_updates }}" + {% if cilium_ip_masq_agent_enable %} --- apiVersion: v1 diff --git a/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 index cd603ff5c..d5ceaa676 100644 --- a/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 @@ -96,11 +96,7 @@ spec: httpGet: host: '127.0.0.1' path: /healthz -{% if cilium_version | regex_replace('v') is version('1.11.6', '>=') %} - port: 9879 -{% else %} - port: 9876 -{% endif %} + port: {{ cilium_agent_health_port }} scheme: HTTP httpHeaders: - name: "brief" @@ -112,11 +108,7 @@ spec: httpGet: host: '127.0.0.1' path: /healthz -{% if cilium_version | regex_replace('v') is version('1.11.6', '>=') %} - port: 9879 -{% else %} - port: 9876 -{% endif %} + port: {{ cilium_agent_health_port }} scheme: HTTP httpHeaders: - name: "brief" @@ -129,11 +121,7 @@ spec: httpGet: host: 127.0.0.1 path: /healthz -{% if cilium_version | regex_replace('v') is version('1.11.6', '>=') %} - port: 9879 -{% else %} - port: 9876 -{% endif %} + port: {{ cilium_agent_health_port }} scheme: HTTP httpHeaders: - name: "brief" @@ -228,6 +216,35 @@ spec: {% endif %} hostNetwork: true initContainers: +{% if cilium_version | regex_replace('v') is version('1.11', '>=') and cilium_cgroup_auto_mount %} + - name: mount-cgroup + image: "{{cilium_image_repo}}:{{cilium_image_tag}}" + imagePullPolicy: {{ k8s_image_pull_policy }} + env: + - name: CGROUP_ROOT + value: {{ cilium_cgroup_host_root }} + - name: BIN_PATH + value: /opt/cni/bin + command: + - sh + - -ec + # The statically linked Go program binary is invoked to avoid any + # dependency on utilities like sh and mount that can be missing on certain + # distros installed on the underlying host. Copy the binary to the + # same directory where we install cilium cni plugin so that exec permissions + # are available. + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + volumeMounts: + - name: hostproc + mountPath: /hostproc + - name: cni-path + mountPath: /hostbin + securityContext: + privileged: true +{% endif %} - command: - /init-container.sh env: @@ -276,7 +293,7 @@ spec: {% if cilium_version | regex_replace('v') is version('1.11', '>=') %} # Required to mount cgroup filesystem from the host to cilium agent pod - name: cilium-cgroup - mountPath: /run/cilium/cgroupv2 + mountPath: {{ cilium_cgroup_host_root }} mountPropagation: HostToContainer {% endif %} - mountPath: /var/run/cilium @@ -334,7 +351,7 @@ spec: # To keep state between restarts / upgrades for cgroup2 filesystem - name: cilium-cgroup hostPath: - path: /run/cilium/cgroupv2 + path: {{ cilium_cgroup_host_root }} type: DirectoryOrCreate {% endif %} # To install cilium cni plugin in the host