Cilium 1.12 Upgrade (#9225)

* Drop support for Cilium < 1.10

Signed-off-by: necatican <necaticanyildirim@gmail.com>

* Synchronize Cilium templates for 1.11.7

Signed-off-by: necatican <contact@necatican.com>

* Set Cilium v1.12.1 as the default version

Signed-off-by: necatican <contact@necatican.com>

Signed-off-by: necatican <necaticanyildirim@gmail.com>
Signed-off-by: necatican <contact@necatican.com>

README.md

@@ -144,7 +144,7 @@ Note: Upstart/SysV init based OS types are not supported.
   - [cni-plugins](https://github.com/containernetworking/plugins) v1.1.1
   - [calico](https://github.com/projectcalico/calico) v3.23.3
   - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
-  - [cilium](https://github.com/cilium/cilium) v1.11.7
+  - [cilium](https://github.com/cilium/cilium) v1.12.1
   - [flannel](https://github.com/flannel-io/flannel) v0.18.1
   - [kube-ovn](https://github.com/alauda/kube-ovn) v1.9.7
   - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.5.1

docs/cilium.md

@@ -56,7 +56,7 @@ cilium_operator_extra_volume_mounts:
 ## Choose Cilium version
 
 ```yml
-cilium_version: v1.11.3
+cilium_version: v1.12.1
 ```
 
 ## Add variable to config

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -1,5 +1,5 @@
 ---
-# cilium_version: "v1.11.7"
+# cilium_version: "v1.12.1"
 
 # Log-level
 # cilium_debug: false
@@ -118,6 +118,7 @@
 # https://docs.cilium.io/en/stable/concepts/networking/masquerading/
 # By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded
 # cilium_ip_masq_agent_enable: false
+
 ### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded
 # cilium_non_masquerade_cidrs:
 #   - 10.0.0.0/8

roles/download/defaults/main.yml

@@ -115,7 +115,7 @@ flannel_cni_version: "v1.1.0"
 cni_version: "v1.1.1"
 weave_version: 2.8.1
 pod_infra_version: "3.6"
-cilium_version: "v1.11.7"
+cilium_version: "v1.12.1"
 kube_ovn_version: "v1.9.7"
 kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}"
 kube_router_version: "v1.5.1"

roles/network_plugin/cilium/defaults/main.yml

@@ -1,4 +1,5 @@
 ---
+cilium_min_version_required: "1.10"
 # Log-level
 cilium_debug: false
 
@@ -7,7 +8,7 @@ cilium_enable_ipv4: true
 cilium_enable_ipv6: false
 
 # Cilium agent health port
-cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879{%- else -%}9876{%- endif -%}"
+cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879 {%- else -%} 9876 {%- endif -%}"
 
 # Identity allocation mode selects how identities are shared between cilium
 # nodes by setting how they are stored. The options are "crd" or "kvstore".
@@ -106,6 +107,7 @@ cilium_wireguard_userspace_fallback: false
 # https://docs.cilium.io/en/stable/concepts/networking/masquerading/
 # By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded
 cilium_ip_masq_agent_enable: false
+
 ### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded
 cilium_non_masquerade_cidrs:
   - 10.0.0.0/8
@@ -201,7 +203,7 @@ cilium_cgroup_host_root: "/run/cilium/cgroupv2"
 
 # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
 # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
-cilium_bpf_map_dynamic_size_ratio: "{%- if cilium_version | regex_replace('v') is version('1.8', '>=') -%}0.0025{%- else -%}0.0{%- endif -%}"
+cilium_bpf_map_dynamic_size_ratio: "0.0025"
 
 # -- Enables masquerading of IPv4 traffic leaving the node from endpoints.
 # Available for Cilium v1.10 and up
@@ -240,3 +242,8 @@ cilium_disable_cnp_status_updates: true
 # Configure how long to wait for the Cilium DaemonSet to be ready again
 cilium_rolling_restart_wait_retries_count: 30
 cilium_rolling_restart_wait_retries_delay_seconds: 10
+
+# Cilium changed the default metrics exporter ports in 1.12
+cilium_agent_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9962', '9090') }}"
+cilium_operator_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9963', '6942') }}"
+cilium_hubble_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9965', '9091') }}"

roles/network_plugin/cilium/tasks/check.yml

@@ -48,13 +48,10 @@
     msg: "cilium_encryption_type must be either 'ipsec' or 'wireguard'"
   when: cilium_encryption_enabled
 
-- name: Stop if `cilium_encryption_type` is set to "wireguard" and cilium_version is < v1.10.0
+- name: Stop if cilium_version is < v1.10.0
   assert:
-    that: cilium_version | regex_replace('v') is version('1.10', '>')
+    that: cilium_version | regex_replace('v') is version(cilium_min_version_required, '>=')
-    msg: "cilium_encryption_type is set to 'wireguard' but cilium_version is < v1.10.0"
+    msg: "cilium_version is too low. Minimum version {{ cilium_min_version_required }}"
-  when:
-    - cilium_encryption_enabled
-    - cilium_encryption_type == "wireguard"
 
 # TODO: Clean this task up when we drop backward compatibility support for `cilium_ipsec_enabled`
 - name: Set `cilium_encryption_type` to "ipsec" and  if `cilium_ipsec_enabled` is true

roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2

@@ -39,7 +39,14 @@ rules:
   - get
   - list
   - watch
-{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - ""
   resources:
@@ -47,22 +54,14 @@ rules:
   - services/status
   verbs:
   - update
-{% endif %}
 - apiGroups:
   - ""
   resources:
-  # to automatically read from k8s and import the node's pod CIDR to cilium's
-  # etcd so all nodes know how to reach another pod running in in a different
-  # node.
-  - nodes
   # to perform the translation of a CNP that contains `ToGroup` to its endpoints
   - services
   - endpoints
   # to check apiserver connectivity
   - namespaces
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
-  - componentstatuses
-{% endif %}
   verbs:
   - get
   - list
@@ -72,26 +71,22 @@ rules:
   resources:
   - ciliumnetworkpolicies
   - ciliumnetworkpolicies/status
+  - ciliumnetworkpolicies/finalizers
   - ciliumclusterwidenetworkpolicies
   - ciliumclusterwidenetworkpolicies/status
+  - ciliumclusterwidenetworkpolicies/finalizers
   - ciliumendpoints
   - ciliumendpoints/status
-{% if cilium_version | regex_replace('v') is version('1.6', '>=') %}
+  - ciliumendpoints/finalizers
   - ciliumnodes
   - ciliumnodes/status
+  - ciliumnodes/finalizers
   - ciliumidentities
   - ciliumidentities/status
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
-  - ciliumnetworkpolicies/finalizers
-  - ciliumclusterwidenetworkpolicies/finalizers
-  - ciliumendpoints/finalizers
-  - ciliumnodes/finalizers
   - ciliumidentities/finalizers
   - ciliumlocalredirectpolicies
   - ciliumlocalredirectpolicies/status
   - ciliumlocalredirectpolicies/finalizers
-{% endif %}
 {% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
   - ciliumendpointslices
 {% endif %}
@@ -101,12 +96,7 @@ rules:
   - ciliumenvoyconfigs
 {% endif %}
   verbs:
-  - get
+  - '*'
-  - list
-  - watch
-  - create
-  - update
-  - delete
 - apiGroups:
   - apiextensions.k8s.io
   resources:
@@ -117,16 +107,12 @@ rules:
   - list
   - update
   - watch
-{% if cilium_version | regex_replace('v') is version('1.8', '>=') %}
+# For cilium-operator running in HA mode.
-  # For cilium-operator running in HA mode.
+#
-  #
+# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
-  # Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
+# between multiple running instances.
-  # between mulitple running instances.
+# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
-  # The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
+# common and fewer objects in the cluster watch "all Leases".
-  # common and fewer objects in the cluster watch "all Leases".
-  # The support for leases was introduced in coordination.k8s.io/v1 during Kubernetes 1.14 release.
-  # In Cilium we currently don't support HA mode for K8s version < 1.14. This condition make sure
-  # that we only authorize access to leases resources in supported K8s versions.
 - apiGroups:
   - coordination.k8s.io
   resources:
@@ -135,4 +121,26 @@ rules:
   - create
   - get
   - update
+{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - update
+  resourceNames:
+  - ciliumbgploadbalancerippools.cilium.io
+  - ciliumbgppeeringpolicies.cilium.io
+  - ciliumclusterwideenvoyconfigs.cilium.io
+  - ciliumclusterwidenetworkpolicies.cilium.io
+  - ciliumegressgatewaypolicies.cilium.io
+  - ciliumegressnatpolicies.cilium.io
+  - ciliumendpoints.cilium.io
+  - ciliumendpointslices.cilium.io
+  - ciliumenvoyconfigs.cilium.io
+  - ciliumexternalworkloads.cilium.io
+  - ciliumidentities.cilium.io
+  - ciliumlocalredirectpolicies.cilium.io
+  - ciliumnetworkpolicies.cilium.io
+  - ciliumnodes.cilium.io
 {% endif %}

roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2

@@ -2,11 +2,11 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  name: cilium-operator
+  namespace: kube-system
   labels:
     io.cilium/app: operator
     name: cilium-operator
-  name: cilium-operator
-  namespace: kube-system
 spec:
   replicas: {{ cilium_operator_replicas }}
   selector:
@@ -22,29 +22,22 @@ spec:
     metadata:
 {% if cilium_enable_prometheus %}
       annotations:
-        prometheus.io/port: "6942"
+        prometheus.io/port: "{{ cilium_operator_scrape_port }}"
         prometheus.io/scrape: "true"
 {% endif %}
       labels:
         io.cilium/app: operator
         name: cilium-operator
     spec:
-      # In HA mode, cilium-operator pods must not be scheduled on the same
-      # node as they will clash with each other.
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-              - key: io.cilium/app
-                operator: In
-                values:
-                - operator
-            topologyKey: "kubernetes.io/hostname"
       containers:
-        - args:
+        - name: cilium-operator
-            - --debug=$(CILIUM_DEBUG)
+          image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}"
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          command:
+            - cilium-operator
+          args:
             - --config-dir=/tmp/cilium/config-map
+            - --debug=$(CILIUM_DEBUG)
 {% if cilium_operator_custom_args is string %}
             - {{ cilium_operator_custom_args }}
 {% else %}
@@ -52,14 +45,7 @@ spec:
             - {{ flag }}
 {% endfor %}
 {% endif %}
-          command:
-            - cilium-operator
           env:
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  apiVersion: v1
-                  fieldPath: metadata.namespace
             - name: K8S_NODE_NAME
               valueFrom:
                 fieldRef:
@@ -76,45 +62,23 @@ spec:
                   key: debug
                   name: cilium-config
                   optional: true
-# We are already mounting the whole ConfigMap as a directory.
-# https://github.com/cilium/cilium/pull/10347
-{% if cilium_version | regex_replace('v') is version('1.8', '<') %}
-            - name: CILIUM_CLUSTER_NAME
-              valueFrom:
-                configMapKeyRef:
-                  key: cluster-name
-                  name: cilium-config
-                  optional: true
-            - name: CILIUM_CLUSTER_ID
-              valueFrom:
-                configMapKeyRef:
-                  key: cluster-id
-                  name: cilium-config
-                  optional: true
-            - name: CILIUM_DISABLE_ENDPOINT_CRD
-              valueFrom:
-                configMapKeyRef:
-                  key: disable-endpoint-crd
-                  name: cilium-config
-                  optional: true
-{% endif %}
             - name: AWS_ACCESS_KEY_ID
               valueFrom:
                 secretKeyRef:
-                  key: AWS_ACCESS_KEY_ID
                   name: cilium-aws
+                  key: AWS_ACCESS_KEY_ID
                   optional: true
             - name: AWS_SECRET_ACCESS_KEY
               valueFrom:
                 secretKeyRef:
-                  key: AWS_SECRET_ACCESS_KEY
                   name: cilium-aws
+                  key: AWS_SECRET_ACCESS_KEY
                   optional: true
             - name: AWS_DEFAULT_REGION
               valueFrom:
                 secretKeyRef:
-                  key: AWS_DEFAULT_REGION
                   name: cilium-aws
+                  key: AWS_DEFAULT_REGION
                   optional: true
 {% if cilium_kube_proxy_replacement == 'strict' %}
             - name: KUBERNETES_SERVICE_HOST
@@ -122,14 +86,16 @@ spec:
             - name: KUBERNETES_SERVICE_PORT
               value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
 {% endif %}
-          image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}"
-          imagePullPolicy: {{ k8s_image_pull_policy }}
-          name: cilium-operator
 {% if cilium_enable_prometheus %}
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
           ports:
-            - containerPort: 6942
+            - name: prometheus
-              hostPort: 6942
+              containerPort: {{ cilium_operator_scrape_port }}
-              name: prometheus
+              hostPort: {{ cilium_operator_scrape_port }}
               protocol: TCP
 {% endif %}
           livenessProbe:
@@ -146,46 +112,55 @@ spec:
             periodSeconds: 10
             timeoutSeconds: 3
           volumeMounts:
-{% if cilium_identity_allocation_mode == "kvstore" %}
+            - name: cilium-config-path
-            - mountPath: /var/lib/etcd-config
+              mountPath: /tmp/cilium/config-map
-              name: etcd-config-path
               readOnly: true
-            - mountPath: "{{cilium_cert_dir}}"
+{% if cilium_identity_allocation_mode == "kvstore" %}
-              name: etcd-secrets
+            - name: etcd-config-path
+              mountPath: /var/lib/etcd-config
+              readOnly: true
+            - name: etcd-secrets
+              mountPath: "{{cilium_cert_dir}}"
               readOnly: true
 {% endif %}
-            - mountPath: /tmp/cilium/config-map
-              name: cilium-config-path
-              readOnly: true
 {% for volume_mount in cilium_operator_extra_volume_mounts %}
             - {{ volume_mount | to_nice_yaml(indent=2) | indent(14) }}
 {% endfor %}
-      dnsPolicy: ClusterFirst
+      hostNetwork: true
-      priorityClassName: system-node-critical
+      dnsPolicy: ClusterFirstWithHostNet
       restartPolicy: Always
+      priorityClassName: system-node-critical
       serviceAccount: cilium-operator
       serviceAccountName: cilium-operator
-      hostNetwork: true
+      # In HA mode, cilium-operator pods must not be scheduled on the same
+      # node as they will clash with each other.
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: kubernetes.io/hostname
+            labelSelector:
+              matchLabels:
+                io.cilium/app: operator
       tolerations:
         - operator: Exists
       volumes:
+        - name: cilium-config-path
+          configMap:
+            name: cilium-config
 {% if cilium_identity_allocation_mode == "kvstore" %}
         # To read the etcd config stored in config maps
-        - configMap:
+        - name: etcd-config-path
+          configMap:
+            name: cilium-config
             defaultMode: 420
             items:
               - key: etcd-config
                 path: etcd.config
-            name: cilium-config
-          name: etcd-config-path
           # To read the k8s etcd secrets in case the user might want to use TLS
         - name: etcd-secrets
           hostPath:
             path: "{{cilium_cert_dir}}"
 {% endif %}
-        - configMap:
-            name: cilium-config
-          name: cilium-config-path
 {% for volume in cilium_operator_extra_volumes %}
         - {{ volume | to_nice_yaml(indent=2) | indent(10) }}
 {% endfor %}

roles/network_plugin/cilium/templates/cilium/config.yml.j2

@@ -41,8 +41,8 @@ data:
   # NOTE that this will open the port on ALL nodes where Cilium pods are
   # scheduled.
 {% if cilium_enable_prometheus %}
-  prometheus-serve-addr: ":9090"
+  prometheus-serve-addr: ":{{ cilium_agent_scrape_port }}"
-  operator-prometheus-serve-addr: ":6942"
+  operator-prometheus-serve-addr: ":{{ cilium_operator_scrape_port }}"
   enable-metrics: "true"
 {% endif %}
 
@@ -127,31 +127,6 @@ data:
   cluster-id: "{{ cilium_cluster_id }}"
 {% endif %}
 
-# `tofqdns-enable-poller` is deprecated in 1.8, removed in 1.9
-# https://github.com/cilium/cilium/issues/8604
-{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
-  # DNS Polling periodically issues a DNS lookup for each `matchName` from
-  # cilium-agent. The result is used to regenerate endpoint policy.
-  # DNS lookups are repeated with an interval of 5 seconds, and are made for
-  # A(IPv4) and AAAA(IPv6) addresses. Should a lookup fail, the most recent IP
-  # data is used instead. An IP change will trigger a regeneration of the Cilium
-  # policy for each endpoint and increment the per cilium-agent policy
-  # repository revision.
-  #
-  # This option is disabled by default starting from version 1.4.x in favor
-  # of a more powerful DNS proxy-based implementation, see [0] for details.
-  # Enable this option if you want to use FQDN policies but do not want to use
-  # the DNS proxy.
-  #
-  # To ease upgrade, users may opt to set this option to "true".
-  # Otherwise please refer to the Upgrade Guide [1] which explains how to
-  # prepare policy rules for upgrade.
-  #
-  # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based
-  # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
-  tofqdns-enable-poller: "{{cilium_tofqdns_enable_poller}}"
-{% endif %}
-
 # `wait-bpf-mount` is removed after v1.10.4
 # https://github.com/cilium/cilium/commit/d2217045cb3726a7f823174e086913b69b8090da
 {% if cilium_version | regex_replace('v') is version('1.10.4', '<') %}
@@ -159,14 +134,6 @@ data:
   wait-bpf-mount: "false"
 {% endif %}
 
-# `enable-legacy-services` is deprecated in 1.6, removed in 1.9
-# https://github.com/cilium/cilium/pull/10255
-{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
-  # Enable legacy services (prior v1.5) to prevent from terminating existing
-  # connections with services when upgrading Cilium from < v1.5 to v1.5.
-  enable-legacy-services: "{{cilium_enable_legacy_services}}"
-{% endif %}
-
   kube-proxy-replacement: "{{ cilium_kube_proxy_replacement }}"
 
 # `native-routing-cidr` is deprecated in 1.10, removed in 1.12.
@@ -191,7 +158,7 @@ data:
 {% if cilium_enable_hubble %}
   enable-hubble: "true"
 {% if cilium_enable_hubble_metrics %}
-  hubble-metrics-server: ":9091"
+  hubble-metrics-server: ":{{ cilium_hubble_scrape_port }}"
   hubble-metrics:
 {% for hubble_metrics_cycle in cilium_hubble_metrics %}
     {{ hubble_metrics_cycle }}
@@ -228,9 +195,7 @@ data:
 {% endif %}
 
   # IPAM settings
-{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
   ipam: "{{ cilium_ipam_mode }}"
-{% endif %}
 
   agent-health-port: "{{ cilium_agent_health_port }}"
 
@@ -240,10 +205,8 @@ data:
 
   bpf-map-dynamic-size-ratio: "{{ cilium_bpf_map_dynamic_size_ratio }}"
 
-{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
   enable-ipv4-masquerade: "{{ cilium_enable_ipv4_masquerade }}"
   enable-ipv6-masquerade: "{{ cilium_enable_ipv6_masquerade }}"
-{% endif %}
 
   enable-bpf-masquerade: "{{ cilium_enable_bpf_masquerade }}"
 
@@ -258,7 +221,6 @@ data:
   enable-bpf-clock-probe: "{{ cilium_enable_bpf_clock_probe }}"
 
   disable-cnp-status-updates: "{{ cilium_disable_cnp_status_updates }}"
-
 {% if cilium_ip_masq_agent_enable %}
 ---
 apiVersion: v1
@@ -274,4 +236,4 @@ data:
 {% endfor %}
     masqLinkLocal: {{ cilium_masq_link_local|bool }}
     resyncInterval: "{{ cilium_ip_masq_resync_interval }}"
-{% endif %}
+{% endif %}

roles/network_plugin/cilium/templates/cilium/cr.yml.j2

@@ -7,9 +7,6 @@ rules:
 - apiGroups:
   - networking.k8s.io
   resources:
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
-  - ingresses
-{% endif %}
   - networkpolicies
   verbs:
   - get
@@ -28,34 +25,25 @@ rules:
   resources:
   - namespaces
   - services
-  - nodes
+  - pods
   - endpoints
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
+  - nodes
-  - componentstatuses
-{% endif %}
   verbs:
   - get
   - list
   - watch
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
+{% if cilium_version | regex_replace('v') is version('1.12', '<') %}
-- apiGroups:
-  - extensions
-  resources:
-  - ingresses
-  verbs:
-  - create
-  - get
-  - list
-  - watch
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.7', '>') %}
 - apiGroups:
   - ""
   resources:
+  - pods
   - pods/finalizers
   verbs:
+  - get
+  - list
+  - watch
   - update
-{% endif %}
+  - delete
 - apiGroups:
   - ""
   resources:
@@ -66,6 +54,7 @@ rules:
   - list
   - watch
   - update
+{% endif %}
 - apiGroups:
   - ""
   resources:
@@ -78,47 +67,45 @@ rules:
   resources:
   - customresourcedefinitions
   verbs:
+  # Deprecated for removal in v1.10
   - create
-  - get
   - list
   - watch
   - update
+
+  # This is used when validating policies in preflight. This will need to stay
+  # until we figure out how to avoid "get" inside the preflight, and then
+  # should be removed ideally.
+  - get
 - apiGroups:
   - cilium.io
   resources:
   - ciliumnetworkpolicies
   - ciliumnetworkpolicies/status
-{% if cilium_version | regex_replace('v') is version('1.7', '>=') %}
   - ciliumclusterwidenetworkpolicies
   - ciliumclusterwidenetworkpolicies/status
-{% endif %}
   - ciliumendpoints
   - ciliumendpoints/status
-{% if cilium_version | regex_replace('v') is version('1.6', '>=') %}
   - ciliumnodes
   - ciliumnodes/status
   - ciliumidentities
-  - ciliumidentities/status
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
-  - ciliumnetworkpolicies/finalizers
-  - ciliumclusterwidenetworkpolicies/finalizers
-  - ciliumendpoints/finalizers
-  - ciliumnodes/finalizers
-  - ciliumidentities/finalizers
   - ciliumlocalredirectpolicies
   - ciliumlocalredirectpolicies/status
-  - ciliumlocalredirectpolicies/finalizers
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
   - ciliumegressnatpolicies
-{% endif %}
 {% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
   - ciliumendpointslices
 {% endif %}
 {% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
   - ciliumbgploadbalancerippools
   - ciliumbgppeeringpolicies
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.11.5', '<') %}
+  - ciliumnetworkpolicies/finalizers
+  - ciliumclusterwidenetworkpolicies/finalizers
+  - ciliumendpoints/finalizers
+  - ciliumnodes/finalizers
+  - ciliumidentities/finalizers
+  - ciliumlocalredirectpolicies/finalizers
 {% endif %}
   verbs:
   - '*'
@@ -128,6 +115,7 @@ rules:
   resources:
   - ciliumclusterwideenvoyconfigs
   - ciliumenvoyconfigs
+  - ciliumegressgatewaypolicies
   verbs:
   - list
   - watch

roles/network_plugin/cilium/templates/cilium/crb.yml.j2

@@ -11,8 +11,3 @@ subjects:
 - kind: ServiceAccount
   name: cilium
   namespace: kube-system
-{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
-- apiGroup: rbac.authorization.k8s.io
-  kind: Group
-  name: system:nodes
-{% endif %}

roles/network_plugin/cilium/templates/cilium/ds.yml.j2

@@ -2,37 +2,37 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  labels:
-    k8s-app: cilium
   name: cilium
   namespace: kube-system
+  labels:
+    k8s-app: cilium
 spec:
   selector:
     matchLabels:
       k8s-app: cilium
+  updateStrategy:
+    rollingUpdate:
+      # Specifies the maximum number of Pods that can be unavailable during the update process.
+      maxUnavailable: 2
+    type: RollingUpdate
   template:
     metadata:
       annotations:
 {% if cilium_enable_prometheus %}
-        prometheus.io/port: "9090"
+        prometheus.io/port: "{{ cilium_agent_scrape_port }}"
         prometheus.io/scrape: "true"
 {% endif %}
         scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]'
       labels:
         k8s-app: cilium
     spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-              - key: k8s-app
-                operator: In
-                values:
-                - cilium
-            topologyKey: kubernetes.io/hostname
       containers:
-      - args:
+      - name: cilium-agent
+        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        command:
+        - cilium-agent
+        args:
         - --config-dir=/tmp/cilium/config-map
 {% if cilium_mtu != "" %}
         - --mtu={{ cilium_mtu }}
@@ -44,54 +44,6 @@ spec:
         - {{ flag }}
 {% endfor %}
 {% endif %}
-        command:
-        - cilium-agent
-        env:
-        - name: K8S_NODE_NAME
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: spec.nodeName
-        - name: CILIUM_K8S_NAMESPACE
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: metadata.namespace
-        - name: CILIUM_CLUSTERMESH_CONFIG
-          value: /var/lib/cilium/clustermesh/
-{% if cilium_kube_proxy_replacement == 'strict' %}
-        - name: KUBERNETES_SERVICE_HOST
-          value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
-        - name: KUBERNETES_SERVICE_PORT
-          value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
-{% endif %}
-{% for env_var in cilium_agent_extra_env_vars %}
-        - {{ env_var | to_nice_yaml(indent=2) | indent(10) }}
-{% endfor %}
-        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
-        imagePullPolicy: {{ k8s_image_pull_policy }}
-        resources:
-          limits:
-            cpu: {{ cilium_cpu_limit }}
-            memory: {{ cilium_memory_limit }}
-          requests:
-            cpu: {{ cilium_cpu_requests }}
-            memory: {{ cilium_memory_requests }}
-        lifecycle:
-          postStart:
-            exec:
-              command:
-              - "/cni-install.sh"
-{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
-              - "--cni-exclusive={{ cilium_cni_exclusive }}"
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
-              - "--log-file={{ cilium_cni_log_file }}"
-{% endif %}
-          preStop:
-            exec:
-              command:
-              - /cni-uninstall.sh
         startupProbe:
           httpGet:
             host: '127.0.0.1'
@@ -131,80 +83,109 @@ spec:
           successThreshold: 1
           failureThreshold: 3
           timeoutSeconds: 5
-        name: cilium-agent
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: CILIUM_CLUSTERMESH_CONFIG
+          value: /var/lib/cilium/clustermesh/
+{% if cilium_kube_proxy_replacement == 'strict' %}
+        - name: KUBERNETES_SERVICE_HOST
+          value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
+        - name: KUBERNETES_SERVICE_PORT
+          value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
+{% endif %}
+{% for env_var in cilium_agent_extra_env_vars %}
+        - {{ env_var | to_nice_yaml(indent=2) | indent(10) }}
+{% endfor %}
+        lifecycle:
+          postStart:
+            exec:
+              command:
+              - "/cni-install.sh"
+              - "--cni-exclusive={{ cilium_cni_exclusive | string | lower }}"
+{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
+              - "--enable-debug={{ cilium_debug | string | lower }}"
+              - "--log-file={{ cilium_cni_log_file }}"
+{% endif %}
+          preStop:
+            exec:
+              command:
+              - /cni-uninstall.sh
+        resources:
+          limits:
+            cpu: {{ cilium_cpu_limit }}
+            memory: {{ cilium_memory_limit }}
+          requests:
+            cpu: {{ cilium_cpu_requests }}
+            memory: {{ cilium_memory_requests }}
 {% if cilium_enable_prometheus or cilium_enable_hubble_metrics %}
         ports:
 {% endif %}
 {% if cilium_enable_prometheus %}
-        - containerPort: 9090
+        - name: prometheus
-          hostPort: 9090
+          containerPort: {{ cilium_agent_scrape_port }}
-          name: prometheus
+          hostPort: {{ cilium_agent_scrape_port }}
           protocol: TCP
 {% endif %}
 {% if cilium_enable_hubble_metrics %}
-        - containerPort: 9091
+        - name: hubble-metrics
-          hostPort: 9091
+          containerPort: {{ cilium_hubble_scrape_port }}
-          name: hubble-metrics
+          hostPort: {{ cilium_hubble_scrape_port }}
           protocol: TCP
 {% endif %}
         securityContext:
           privileged: true
         volumeMounts:
-        - mountPath: /sys/fs/bpf
+        - name: bpf-maps
-          name: bpf-maps
+          mountPath: /sys/fs/bpf
           mountPropagation: Bidirectional
-        - mountPath: /var/run/cilium
+        - name: cilium-run
-          name: cilium-run
+          mountPath: /var/run/cilium
-        - mountPath: /host/opt/cni/bin
+        - name: cni-path
-          name: cni-path
+          mountPath: /host/opt/cni/bin
-        - mountPath: /host/etc/cni/net.d
+        - name: etc-cni-netd
-          name: etc-cni-netd
+          mountPath: /host/etc/cni/net.d
-# pkg/workloads was depreca, removed in 1.7
-# https://github.com/cilium/cilium/pull/9447
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
-{% if container_manager == 'docker' %}
-        - mountPath: /var/run/docker.sock
-          name: docker-socket
-          readOnly: true
-{% else %}
-        - name: "{{ container_manager }}-socket"
-          mountPath: {{ cri_socket }}
-          readOnly: true
-{% endif %}
-{% endif %}
 {% if cilium_identity_allocation_mode == "kvstore" %}
-        - mountPath: /var/lib/etcd-config
+        - name: etcd-config-path
-          name: etcd-config-path
+          mountPath: /var/lib/etcd-config
           readOnly: true
-        - mountPath: "{{cilium_cert_dir}}"
+        - name: etcd-secrets
-          name: etcd-secrets
+          mountPath: "{{cilium_cert_dir}}"
           readOnly: true
 {% endif %}
-        - mountPath: /var/lib/cilium/clustermesh
+        - name: clustermesh-secrets
-          name: clustermesh-secrets
+          mountPath: /var/lib/cilium/clustermesh
           readOnly: true
-        - mountPath: /tmp/cilium/config-map
+        - name: cilium-config-path
-          name: cilium-config-path
+          mountPath: /tmp/cilium/config-map
           readOnly: true
-{% if cilium_ip_masq_agent_enable %}
+{% if cilium_ip_masq_agent_enable %}\
         - name: ip-masq-agent
           mountPath: /etc/config
           readOnly: true
 {% endif %}
           # Needed to be able to load kernel modules
-        - mountPath: /lib/modules
+        - name: lib-modules
-          name: lib-modules
+          mountPath: /lib/modules
           readOnly: true
-        - mountPath: /run/xtables.lock
+        - name: xtables-lock
-          name: xtables-lock
+          mountPath: /run/xtables.lock
 {% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %}
-        - mountPath: /etc/ipsec
+        - name: cilium-ipsec-secrets
-          name: cilium-ipsec-secrets
+          mountPath: /etc/ipsec
           readOnly: true
 {% endif %}
 {% if cilium_hubble_install %}
-        - mountPath: /var/lib/cilium/tls/hubble
+        - name: hubble-tls
-          name: hubble-tls
+          mountPath: /var/lib/cilium/tls/hubble
           readOnly: true
 {% endif %}
 {% for volume_mount in cilium_agent_extra_volume_mounts %}
@@ -245,22 +226,50 @@ spec:
         securityContext:
           privileged: true
 {% endif %}
-      - command:
+{% if cilium_version | regex_replace('v') is version('1.11.7', '>=') %}
+      - name: apply-sysctl-overwrites
+        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        env:
+        - name: BIN_PATH
+          value: /opt/cni/bin
+        command:
+        - sh
+        - -ec
+        # The statically linked Go program binary is invoked to avoid any
+        # dependency on utilities like sh that can be missing on certain
+        # distros installed on the underlying host. Copy the binary to the
+        # same directory where we install cilium cni plugin so that exec permissions
+        # are available.
+        - |
+          cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix;
+          nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix";
+          rm /hostbin/cilium-sysctlfix
+        volumeMounts:
+        - name: hostproc
+          mountPath: /hostproc
+        - name: cni-path
+          mountPath: /hostbin
+        securityContext:
+          privileged: true
+{% endif %}
+      - name: clean-cilium-state
+        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        command:
         - /init-container.sh
         env:
         - name: CILIUM_ALL_STATE
           valueFrom:
             configMapKeyRef:
-              key: clean-cilium-state
               name: cilium-config
+              key: clean-cilium-state
               optional: true
-        # CLEAN_CILIUM_BPF_STATE is deprecated in 1.6.
+        - name: CILIUM_BPF_STATE
-        # https://github.com/cilium/cilium/pull/7478
-        - name: "{{ cilium_version | regex_replace('v') is version('1.6', '<')| ternary('CLEAN_CILIUM_BPF_STATE','CILIUM_BPF_STATE') }}"
           valueFrom:
             configMapKeyRef:
-              key: clean-cilium-bpf-state
               name: cilium-config
+              key: clean-cilium-bpf-state
               optional: true
 # Removed in 1.11 and up.
 # https://github.com/cilium/cilium/commit/f7a3f59fd74983c600bfce9cac364b76d20849d9
@@ -278,32 +287,25 @@ spec:
         - name: KUBERNETES_SERVICE_PORT
           value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
 {% endif %}
-{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
-        image: "{{cilium_init_image_repo}}:{{cilium_init_image_tag}}"
-{% else %}
-        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
-{% endif %}
-        imagePullPolicy: {{ k8s_image_pull_policy }}
-        name: clean-cilium-state
         securityContext:
           privileged: true
         volumeMounts:
-        - mountPath: /sys/fs/bpf
+        - name: bpf-maps
-          name: bpf-maps
+          mountPath: /sys/fs/bpf
 {% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
           # Required to mount cgroup filesystem from the host to cilium agent pod
         - name: cilium-cgroup
           mountPath: {{ cilium_cgroup_host_root }}
           mountPropagation: HostToContainer
 {% endif %}
-        - mountPath: /var/run/cilium
+        - name: cilium-run
-          name: cilium-run
+          mountPath: /var/run/cilium
         resources:
           requests:
             cpu: 100m
             memory: 100Mi
-      priorityClassName: system-node-critical
       restartPolicy: Always
+      priorityClassName: system-node-critical
       serviceAccount: cilium
       serviceAccountName: cilium
       terminationGracePeriodSeconds: 1
@@ -312,36 +314,26 @@ spec:
 {% if cilium_identity_allocation_mode == "kvstore" %}
       dnsPolicy: ClusterFirstWithHostNet
 {% endif %}
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: kubernetes.io/hostname
+            labelSelector:
+              matchLabels:
+                k8s-app: cilium
       tolerations:
       - operator: Exists
       volumes:
         # To keep state between restarts / upgrades
-      - hostPath:
+      - name: cilium-run
+        hostPath:
           path: /var/run/cilium
           type: DirectoryOrCreate
-        name: cilium-run
         # To keep state between restarts / upgrades for bpf maps
-      - hostPath:
+      - name: bpf-maps
+        hostPath:
           path: /sys/fs/bpf
           type: DirectoryOrCreate
-        name: bpf-maps
-# pkg/workloads was deprecated in 1.6, removed in 1.7
-# https://github.com/cilium/cilium/pull/9447
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
-{% if container_manager == 'docker' %}
-        # To read docker events from the node
-      - hostPath:
-          path: /var/run/docker.sock
-          type: Socket
-        name: docker-socket
-{% else %}
-        # To read crio events from the node
-      - hostPath:
-          path: {{ cri_socket }}
-          type: Socket
-        name: {{ container_manager }}-socket
-{% endif %}
-{% endif %}
 {% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
       # To mount cgroup2 filesystem on the host
       - name: hostproc
@@ -355,34 +347,34 @@ spec:
           type: DirectoryOrCreate
 {% endif %}
         # To install cilium cni plugin in the host
-      - hostPath:
+      - name: cni-path
+        hostPath:
           path: /opt/cni/bin
           type: DirectoryOrCreate
-        name: cni-path
         # To install cilium cni configuration in the host
-      - hostPath:
+      - name: etc-cni-netd
+        hostPath:
           path: /etc/cni/net.d
           type: DirectoryOrCreate
-        name: etc-cni-netd
         # To be able to load kernel modules
-      - hostPath:
+      - name: lib-modules
+        hostPath:
           path: /lib/modules
-        name: lib-modules
         # To access iptables concurrently with other processes (e.g. kube-proxy)
-      - hostPath:
+      - name: xtables-lock
+        hostPath:
           path: /run/xtables.lock
           type: FileOrCreate
-        name: xtables-lock
 {% if cilium_identity_allocation_mode == "kvstore" %}
         # To read the etcd config stored in config maps
-      - configMap:
+      - name: etcd-config-path
+        configMap:
+          name: cilium-config
           # note: the leading zero means this number is in octal representation: do not remove it
           defaultMode: 0400
           items:
           - key: etcd-config
             path: etcd.config
-          name: cilium-config
-        name: etcd-config-path
         # To read the k8s etcd secrets in case the user might want to use TLS
       - name: etcd-secrets
         hostPath:
@@ -391,21 +383,22 @@ spec:
         # To read the clustermesh configuration
       - name: clustermesh-secrets
         secret:
+          secretName: cilium-clustermesh
           # note: the leading zero means this number is in octal representation: do not remove it
           defaultMode: 0400
           optional: true
-          secretName: cilium-clustermesh
         # To read the configuration from the config map
-      - configMap:
+      - name: cilium-config-path
+        configMap:
           name: cilium-config
-        name: cilium-config-path
 {% if cilium_ip_masq_agent_enable %}
-      - configMap:
+      - name: ip-masq-agent
+        configMap:
           name: ip-masq-agent
+          optional: true
           items:
           - key: config
             path: ip-masq-agent
-        name: ip-masq-agent
 {% endif %}
 {% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %}
       - name: cilium-ipsec-secrets
@@ -420,21 +413,12 @@ spec:
           sources:
           - secret:
               name: hubble-server-certs
+              optional: true
               items:
+                - key: ca.crt
+                  path: client-ca.crt
                 - key: tls.crt
                   path: server.crt
                 - key: tls.key
                   path: server.key
-              optional: true
-          - configMap:
-              name: hubble-ca-cert
-              items:
-                - key: ca.crt
-                  path: client-ca.crt
-              optional: true
 {% endif %}
-  updateStrategy:
-    rollingUpdate:
-      # Specifies the maximum number of Pods that can be unavailable during the update process.
-      maxUnavailable: 2
-    type: RollingUpdate