Cilium 1.12 Upgrade (#9225)

* Drop support for Cilium < 1.10 Signed-off-by: necatican <necaticanyildirim@gmail.com> * Synchronize Cilium templates for 1.11.7 Signed-off-by: necatican <contact@necatican.com> * Set Cilium v1.12.1 as the default version Signed-off-by: necatican <contact@necatican.com> Signed-off-by: necatican <necaticanyildirim@gmail.com> Signed-off-by: necatican <contact@necatican.com>
2022-09-19 12:14:31 +03:00 · 2022-09-19 12:14:31 +03:00 · 7da3dbcb39
commit 7da3dbcb39
parent 680293e79c
12 changed files with 291 additions and 374 deletions
--- a/README.md
+++ b/README.md
@ -144,7 +144,7 @@ Note: Upstart/SysV init based OS types are not supported.
  - [cni-plugins](https://github.com/containernetworking/plugins) v1.1.1
  - [calico](https://github.com/projectcalico/calico) v3.23.3
  - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
-  - [cilium](https://github.com/cilium/cilium) v1.11.7
+  - [cilium](https://github.com/cilium/cilium) v1.12.1
  - [flannel](https://github.com/flannel-io/flannel) v0.18.1
  - [kube-ovn](https://github.com/alauda/kube-ovn) v1.9.7
  - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.5.1
--- a/docs/cilium.md
+++ b/docs/cilium.md
@ -56,7 +56,7 @@ cilium_operator_extra_volume_mounts:
 ## Choose Cilium version

 ```yml
-cilium_version: v1.11.3
+cilium_version: v1.12.1
 ```

 ## Add variable to config
--- a/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml
+++ b/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml
@ -1,5 +1,5 @@
 ---
-# cilium_version: "v1.11.7"
+# cilium_version: "v1.12.1"

 # Log-level
 # cilium_debug: false
@ -118,6 +118,7 @@
 # https://docs.cilium.io/en/stable/concepts/networking/masquerading/
 # By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded
 # cilium_ip_masq_agent_enable: false
+
 ### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded
 # cilium_non_masquerade_cidrs:
 #   - 10.0.0.0/8
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@ -115,7 +115,7 @@ flannel_cni_version: "v1.1.0"
 cni_version: "v1.1.1"
 weave_version: 2.8.1
 pod_infra_version: "3.6"
-cilium_version: "v1.11.7"
+cilium_version: "v1.12.1"
 kube_ovn_version: "v1.9.7"
 kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}"
 kube_router_version: "v1.5.1"
--- a/roles/network_plugin/cilium/defaults/main.yml
+++ b/roles/network_plugin/cilium/defaults/main.yml
@ -1,4 +1,5 @@
 ---
+cilium_min_version_required: "1.10"
 # Log-level
 cilium_debug: false

@ -7,7 +8,7 @@ cilium_enable_ipv4: true
 cilium_enable_ipv6: false

 # Cilium agent health port
-cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879{%- else -%}9876{%- endif -%}"
+cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879 {%- else -%} 9876 {%- endif -%}"

 # Identity allocation mode selects how identities are shared between cilium
 # nodes by setting how they are stored. The options are "crd" or "kvstore".
@ -106,6 +107,7 @@ cilium_wireguard_userspace_fallback: false
 # https://docs.cilium.io/en/stable/concepts/networking/masquerading/
 # By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded
 cilium_ip_masq_agent_enable: false
+
 ### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded
 cilium_non_masquerade_cidrs:
  - 10.0.0.0/8
@ -201,7 +203,7 @@ cilium_cgroup_host_root: "/run/cilium/cgroupv2"

 # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
 # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
-cilium_bpf_map_dynamic_size_ratio: "{%- if cilium_version | regex_replace('v') is version('1.8', '>=') -%}0.0025{%- else -%}0.0{%- endif -%}"
+cilium_bpf_map_dynamic_size_ratio: "0.0025"

 # -- Enables masquerading of IPv4 traffic leaving the node from endpoints.
 # Available for Cilium v1.10 and up
@ -240,3 +242,8 @@ cilium_disable_cnp_status_updates: true
 # Configure how long to wait for the Cilium DaemonSet to be ready again
 cilium_rolling_restart_wait_retries_count: 30
 cilium_rolling_restart_wait_retries_delay_seconds: 10
+
+# Cilium changed the default metrics exporter ports in 1.12
+cilium_agent_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9962', '9090') }}"
+cilium_operator_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9963', '6942') }}"
+cilium_hubble_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9965', '9091') }}"
--- a/roles/network_plugin/cilium/tasks/check.yml
+++ b/roles/network_plugin/cilium/tasks/check.yml
@ -48,13 +48,10 @@
    msg: "cilium_encryption_type must be either 'ipsec' or 'wireguard'"
  when: cilium_encryption_enabled

- name: Stop if `cilium_encryption_type` is set to "wireguard" and cilium_version is < v1.10.0
+- name: Stop if cilium_version is < v1.10.0
  assert:
-    that: cilium_version | regex_replace('v') is version('1.10', '>')
-    msg: "cilium_encryption_type is set to 'wireguard' but cilium_version is < v1.10.0"
-  when:
-    - cilium_encryption_enabled
-    - cilium_encryption_type == "wireguard"
+    that: cilium_version | regex_replace('v') is version(cilium_min_version_required, '>=')
+    msg: "cilium_version is too low. Minimum version {{ cilium_min_version_required }}"

 # TODO: Clean this task up when we drop backward compatibility support for `cilium_ipsec_enabled`
 - name: Set `cilium_encryption_type` to "ipsec" and  if `cilium_ipsec_enabled` is true
--- a/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2
@ -39,7 +39,14 @@ rules:
  - get
  - list
  - watch
-{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
  - ""
  resources:
@ -47,22 +54,14 @@ rules:
  - services/status
  verbs:
  - update
-{% endif %}
 - apiGroups:
  - ""
  resources:
-  # to automatically read from k8s and import the node's pod CIDR to cilium's
-  # etcd so all nodes know how to reach another pod running in in a different
-  # node.
-  - nodes
  # to perform the translation of a CNP that contains `ToGroup` to its endpoints
  - services
  - endpoints
  # to check apiserver connectivity
  - namespaces
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
-  - componentstatuses
-{% endif %}
  verbs:
  - get
  - list
@ -72,26 +71,22 @@ rules:
  resources:
  - ciliumnetworkpolicies
  - ciliumnetworkpolicies/status
+  - ciliumnetworkpolicies/finalizers
  - ciliumclusterwidenetworkpolicies
  - ciliumclusterwidenetworkpolicies/status
+  - ciliumclusterwidenetworkpolicies/finalizers
  - ciliumendpoints
  - ciliumendpoints/status
-{% if cilium_version | regex_replace('v') is version('1.6', '>=') %}
+  - ciliumendpoints/finalizers
  - ciliumnodes
  - ciliumnodes/status
+  - ciliumnodes/finalizers
  - ciliumidentities
  - ciliumidentities/status
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
-  - ciliumnetworkpolicies/finalizers
-  - ciliumclusterwidenetworkpolicies/finalizers
-  - ciliumendpoints/finalizers
-  - ciliumnodes/finalizers
  - ciliumidentities/finalizers
  - ciliumlocalredirectpolicies
  - ciliumlocalredirectpolicies/status
  - ciliumlocalredirectpolicies/finalizers
-{% endif %}
 {% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
  - ciliumendpointslices
 {% endif %}
@ -101,12 +96,7 @@ rules:
  - ciliumenvoyconfigs
 {% endif %}
  verbs:
-  - get
-  - list
-  - watch
-  - create
-  - update
-  - delete
+  - '*'
 - apiGroups:
  - apiextensions.k8s.io
  resources:
@ -117,16 +107,12 @@ rules:
  - list
  - update
  - watch
-{% if cilium_version | regex_replace('v') is version('1.8', '>=') %}
-  # For cilium-operator running in HA mode.
-  #
-  # Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
-  # between mulitple running instances.
-  # The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
-  # common and fewer objects in the cluster watch "all Leases".
-  # The support for leases was introduced in coordination.k8s.io/v1 during Kubernetes 1.14 release.
-  # In Cilium we currently don't support HA mode for K8s version < 1.14. This condition make sure
-  # that we only authorize access to leases resources in supported K8s versions.
+# For cilium-operator running in HA mode.
+#
+# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
+# between multiple running instances.
+# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
+# common and fewer objects in the cluster watch "all Leases".
 - apiGroups:
  - coordination.k8s.io
  resources:
@ -135,4 +121,26 @@ rules:
  - create
  - get
  - update
+{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - update
+  resourceNames:
+  - ciliumbgploadbalancerippools.cilium.io
+  - ciliumbgppeeringpolicies.cilium.io
+  - ciliumclusterwideenvoyconfigs.cilium.io
+  - ciliumclusterwidenetworkpolicies.cilium.io
+  - ciliumegressgatewaypolicies.cilium.io
+  - ciliumegressnatpolicies.cilium.io
+  - ciliumendpoints.cilium.io
+  - ciliumendpointslices.cilium.io
+  - ciliumenvoyconfigs.cilium.io
+  - ciliumexternalworkloads.cilium.io
+  - ciliumidentities.cilium.io
+  - ciliumlocalredirectpolicies.cilium.io
+  - ciliumnetworkpolicies.cilium.io
+  - ciliumnodes.cilium.io
 {% endif %}
--- a/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2
@ -2,11 +2,11 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  name: cilium-operator
+  namespace: kube-system
  labels:
    io.cilium/app: operator
    name: cilium-operator
-  name: cilium-operator
-  namespace: kube-system
 spec:
  replicas: {{ cilium_operator_replicas }}
  selector:
@ -22,29 +22,22 @@ spec:
    metadata:
 {% if cilium_enable_prometheus %}
      annotations:
-        prometheus.io/port: "6942"
+        prometheus.io/port: "{{ cilium_operator_scrape_port }}"
        prometheus.io/scrape: "true"
 {% endif %}
      labels:
        io.cilium/app: operator
        name: cilium-operator
    spec:
-      # In HA mode, cilium-operator pods must not be scheduled on the same
-      # node as they will clash with each other.
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-              - key: io.cilium/app
-                operator: In
-                values:
-                - operator
-            topologyKey: "kubernetes.io/hostname"
      containers:
-        - args:
-            - --debug=$(CILIUM_DEBUG)
+        - name: cilium-operator
+          image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}"
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          command:
+            - cilium-operator
+          args:
            - --config-dir=/tmp/cilium/config-map
+            - --debug=$(CILIUM_DEBUG)
 {% if cilium_operator_custom_args is string %}
            - {{ cilium_operator_custom_args }}
 {% else %}
@ -52,14 +45,7 @@ spec:
            - {{ flag }}
 {% endfor %}
 {% endif %}
-          command:
-            - cilium-operator
          env:
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  apiVersion: v1
-                  fieldPath: metadata.namespace
            - name: K8S_NODE_NAME
              valueFrom:
                fieldRef:
@ -76,45 +62,23 @@ spec:
                  key: debug
                  name: cilium-config
                  optional: true
-# We are already mounting the whole ConfigMap as a directory.
-# https://github.com/cilium/cilium/pull/10347
-{% if cilium_version | regex_replace('v') is version('1.8', '<') %}
-            - name: CILIUM_CLUSTER_NAME
-              valueFrom:
-                configMapKeyRef:
-                  key: cluster-name
-                  name: cilium-config
-                  optional: true
-            - name: CILIUM_CLUSTER_ID
-              valueFrom:
-                configMapKeyRef:
-                  key: cluster-id
-                  name: cilium-config
-                  optional: true
-            - name: CILIUM_DISABLE_ENDPOINT_CRD
-              valueFrom:
-                configMapKeyRef:
-                  key: disable-endpoint-crd
-                  name: cilium-config
-                  optional: true
-{% endif %}
            - name: AWS_ACCESS_KEY_ID
              valueFrom:
                secretKeyRef:
-                  key: AWS_ACCESS_KEY_ID
                  name: cilium-aws
+                  key: AWS_ACCESS_KEY_ID
                  optional: true
            - name: AWS_SECRET_ACCESS_KEY
              valueFrom:
                secretKeyRef:
-                  key: AWS_SECRET_ACCESS_KEY
                  name: cilium-aws
+                  key: AWS_SECRET_ACCESS_KEY
                  optional: true
            - name: AWS_DEFAULT_REGION
              valueFrom:
                secretKeyRef:
-                  key: AWS_DEFAULT_REGION
                  name: cilium-aws
+                  key: AWS_DEFAULT_REGION
                  optional: true
 {% if cilium_kube_proxy_replacement == 'strict' %}
            - name: KUBERNETES_SERVICE_HOST
@ -122,14 +86,16 @@ spec:
            - name: KUBERNETES_SERVICE_PORT
              value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
 {% endif %}
-          image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}"
-          imagePullPolicy: {{ k8s_image_pull_policy }}
-          name: cilium-operator
 {% if cilium_enable_prometheus %}
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
          ports:
-            - containerPort: 6942
-              hostPort: 6942
-              name: prometheus
+            - name: prometheus
+              containerPort: {{ cilium_operator_scrape_port }}
+              hostPort: {{ cilium_operator_scrape_port }}
              protocol: TCP
 {% endif %}
          livenessProbe:
@ -146,46 +112,55 @@ spec:
            periodSeconds: 10
            timeoutSeconds: 3
          volumeMounts:
-{% if cilium_identity_allocation_mode == "kvstore" %}
-            - mountPath: /var/lib/etcd-config
-              name: etcd-config-path
+            - name: cilium-config-path
+              mountPath: /tmp/cilium/config-map
              readOnly: true
-            - mountPath: "{{cilium_cert_dir}}"
-              name: etcd-secrets
+{% if cilium_identity_allocation_mode == "kvstore" %}
+            - name: etcd-config-path
+              mountPath: /var/lib/etcd-config
+              readOnly: true
+            - name: etcd-secrets
+              mountPath: "{{cilium_cert_dir}}"
              readOnly: true
 {% endif %}
-            - mountPath: /tmp/cilium/config-map
-              name: cilium-config-path
-              readOnly: true
 {% for volume_mount in cilium_operator_extra_volume_mounts %}
            - {{ volume_mount | to_nice_yaml(indent=2) | indent(14) }}
 {% endfor %}
-      dnsPolicy: ClusterFirst
-      priorityClassName: system-node-critical
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
      restartPolicy: Always
+      priorityClassName: system-node-critical
      serviceAccount: cilium-operator
      serviceAccountName: cilium-operator
-      hostNetwork: true
+      # In HA mode, cilium-operator pods must not be scheduled on the same
+      # node as they will clash with each other.
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: kubernetes.io/hostname
+            labelSelector:
+              matchLabels:
+                io.cilium/app: operator
      tolerations:
        - operator: Exists
      volumes:
+        - name: cilium-config-path
+          configMap:
+            name: cilium-config
 {% if cilium_identity_allocation_mode == "kvstore" %}
        # To read the etcd config stored in config maps
-        - configMap:
+        - name: etcd-config-path
+          configMap:
+            name: cilium-config
            defaultMode: 420
            items:
              - key: etcd-config
                path: etcd.config
-            name: cilium-config
-          name: etcd-config-path
          # To read the k8s etcd secrets in case the user might want to use TLS
        - name: etcd-secrets
          hostPath:
            path: "{{cilium_cert_dir}}"
 {% endif %}
-        - configMap:
-            name: cilium-config
-          name: cilium-config-path
 {% for volume in cilium_operator_extra_volumes %}
        - {{ volume | to_nice_yaml(indent=2) | indent(10) }}
 {% endfor %}
--- a/roles/network_plugin/cilium/templates/cilium/config.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium/config.yml.j2
@ -41,8 +41,8 @@ data:
  # NOTE that this will open the port on ALL nodes where Cilium pods are
  # scheduled.
 {% if cilium_enable_prometheus %}
-  prometheus-serve-addr: ":9090"
-  operator-prometheus-serve-addr: ":6942"
+  prometheus-serve-addr: ":{{ cilium_agent_scrape_port }}"
+  operator-prometheus-serve-addr: ":{{ cilium_operator_scrape_port }}"
  enable-metrics: "true"
 {% endif %}

@ -127,31 +127,6 @@ data:
  cluster-id: "{{ cilium_cluster_id }}"
 {% endif %}

-# `tofqdns-enable-poller` is deprecated in 1.8, removed in 1.9
-# https://github.com/cilium/cilium/issues/8604
-{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
-  # DNS Polling periodically issues a DNS lookup for each `matchName` from
-  # cilium-agent. The result is used to regenerate endpoint policy.
-  # DNS lookups are repeated with an interval of 5 seconds, and are made for
-  # A(IPv4) and AAAA(IPv6) addresses. Should a lookup fail, the most recent IP
-  # data is used instead. An IP change will trigger a regeneration of the Cilium
-  # policy for each endpoint and increment the per cilium-agent policy
-  # repository revision.
-  #
-  # This option is disabled by default starting from version 1.4.x in favor
-  # of a more powerful DNS proxy-based implementation, see [0] for details.
-  # Enable this option if you want to use FQDN policies but do not want to use
-  # the DNS proxy.
-  #
-  # To ease upgrade, users may opt to set this option to "true".
-  # Otherwise please refer to the Upgrade Guide [1] which explains how to
-  # prepare policy rules for upgrade.
-  #
-  # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based
-  # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
-  tofqdns-enable-poller: "{{cilium_tofqdns_enable_poller}}"
-{% endif %}
-
 # `wait-bpf-mount` is removed after v1.10.4
 # https://github.com/cilium/cilium/commit/d2217045cb3726a7f823174e086913b69b8090da
 {% if cilium_version | regex_replace('v') is version('1.10.4', '<') %}
@ -159,14 +134,6 @@ data:
  wait-bpf-mount: "false"
 {% endif %}

-# `enable-legacy-services` is deprecated in 1.6, removed in 1.9
-# https://github.com/cilium/cilium/pull/10255
-{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
-  # Enable legacy services (prior v1.5) to prevent from terminating existing
-  # connections with services when upgrading Cilium from < v1.5 to v1.5.
-  enable-legacy-services: "{{cilium_enable_legacy_services}}"
-{% endif %}
-
  kube-proxy-replacement: "{{ cilium_kube_proxy_replacement }}"

 # `native-routing-cidr` is deprecated in 1.10, removed in 1.12.
@ -191,7 +158,7 @@ data:
 {% if cilium_enable_hubble %}
  enable-hubble: "true"
 {% if cilium_enable_hubble_metrics %}
-  hubble-metrics-server: ":9091"
+  hubble-metrics-server: ":{{ cilium_hubble_scrape_port }}"
  hubble-metrics:
 {% for hubble_metrics_cycle in cilium_hubble_metrics %}
    {{ hubble_metrics_cycle }}
@ -228,9 +195,7 @@ data:
 {% endif %}

  # IPAM settings
-{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
  ipam: "{{ cilium_ipam_mode }}"
-{% endif %}

  agent-health-port: "{{ cilium_agent_health_port }}"

@ -240,10 +205,8 @@ data:

  bpf-map-dynamic-size-ratio: "{{ cilium_bpf_map_dynamic_size_ratio }}"

-{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
  enable-ipv4-masquerade: "{{ cilium_enable_ipv4_masquerade }}"
  enable-ipv6-masquerade: "{{ cilium_enable_ipv6_masquerade }}"
-{% endif %}

  enable-bpf-masquerade: "{{ cilium_enable_bpf_masquerade }}"

@ -258,7 +221,6 @@ data:
  enable-bpf-clock-probe: "{{ cilium_enable_bpf_clock_probe }}"

  disable-cnp-status-updates: "{{ cilium_disable_cnp_status_updates }}"
-
 {% if cilium_ip_masq_agent_enable %}
 ---
 apiVersion: v1
@ -274,4 +236,4 @@ data:
 {% endfor %}
    masqLinkLocal: {{ cilium_masq_link_local|bool }}
    resyncInterval: "{{ cilium_ip_masq_resync_interval }}"
-{% endif %}
+{% endif %}
--- a/roles/network_plugin/cilium/templates/cilium/cr.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium/cr.yml.j2
@ -7,9 +7,6 @@ rules:
 - apiGroups:
  - networking.k8s.io
  resources:
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
-  - ingresses
-{% endif %}
  - networkpolicies
  verbs:
  - get
@ -28,34 +25,25 @@ rules:
  resources:
  - namespaces
  - services
-  - nodes
+  - pods
  - endpoints
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
-  - componentstatuses
-{% endif %}
+  - nodes
  verbs:
  - get
  - list
  - watch
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
- apiGroups:
-  - extensions
-  resources:
-  - ingresses
-  verbs:
-  - create
-  - get
-  - list
-  - watch
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.7', '>') %}
+{% if cilium_version | regex_replace('v') is version('1.12', '<') %}
 - apiGroups:
  - ""
  resources:
+  - pods
  - pods/finalizers
  verbs:
+  - get
+  - list
+  - watch
  - update
-{% endif %}
+  - delete
 - apiGroups:
  - ""
  resources:
@ -66,6 +54,7 @@ rules:
  - list
  - watch
  - update
+{% endif %}
 - apiGroups:
  - ""
  resources:
@ -78,47 +67,45 @@ rules:
  resources:
  - customresourcedefinitions
  verbs:
+  # Deprecated for removal in v1.10
  - create
-  - get
  - list
  - watch
  - update
+
+  # This is used when validating policies in preflight. This will need to stay
+  # until we figure out how to avoid "get" inside the preflight, and then
+  # should be removed ideally.
+  - get
 - apiGroups:
  - cilium.io
  resources:
  - ciliumnetworkpolicies
  - ciliumnetworkpolicies/status
-{% if cilium_version | regex_replace('v') is version('1.7', '>=') %}
  - ciliumclusterwidenetworkpolicies
  - ciliumclusterwidenetworkpolicies/status
-{% endif %}
  - ciliumendpoints
  - ciliumendpoints/status
-{% if cilium_version | regex_replace('v') is version('1.6', '>=') %}
  - ciliumnodes
  - ciliumnodes/status
  - ciliumidentities
-  - ciliumidentities/status
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
-  - ciliumnetworkpolicies/finalizers
-  - ciliumclusterwidenetworkpolicies/finalizers
-  - ciliumendpoints/finalizers
-  - ciliumnodes/finalizers
-  - ciliumidentities/finalizers
  - ciliumlocalredirectpolicies
  - ciliumlocalredirectpolicies/status
-  - ciliumlocalredirectpolicies/finalizers
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
  - ciliumegressnatpolicies
-{% endif %}
 {% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
  - ciliumendpointslices
 {% endif %}
 {% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
  - ciliumbgploadbalancerippools
  - ciliumbgppeeringpolicies
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.11.5', '<') %}
+  - ciliumnetworkpolicies/finalizers
+  - ciliumclusterwidenetworkpolicies/finalizers
+  - ciliumendpoints/finalizers
+  - ciliumnodes/finalizers
+  - ciliumidentities/finalizers
+  - ciliumlocalredirectpolicies/finalizers
 {% endif %}
  verbs:
  - '*'
@ -128,6 +115,7 @@ rules:
  resources:
  - ciliumclusterwideenvoyconfigs
  - ciliumenvoyconfigs
+  - ciliumegressgatewaypolicies
  verbs:
  - list
  - watch
--- a/roles/network_plugin/cilium/templates/cilium/crb.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium/crb.yml.j2
@ -11,8 +11,3 @@ subjects:
 - kind: ServiceAccount
  name: cilium
  namespace: kube-system
-{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
- apiGroup: rbac.authorization.k8s.io
-  kind: Group
-  name: system:nodes
-{% endif %}
--- a/roles/network_plugin/cilium/templates/cilium/ds.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium/ds.yml.j2
@ -2,37 +2,37 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  labels:
-    k8s-app: cilium
  name: cilium
  namespace: kube-system
+  labels:
+    k8s-app: cilium
 spec:
  selector:
    matchLabels:
      k8s-app: cilium
+  updateStrategy:
+    rollingUpdate:
+      # Specifies the maximum number of Pods that can be unavailable during the update process.
+      maxUnavailable: 2
+    type: RollingUpdate
  template:
    metadata:
      annotations:
 {% if cilium_enable_prometheus %}
-        prometheus.io/port: "9090"
+        prometheus.io/port: "{{ cilium_agent_scrape_port }}"
        prometheus.io/scrape: "true"
 {% endif %}
        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]'
      labels:
        k8s-app: cilium
    spec:
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-          - labelSelector:
-              matchExpressions:
-              - key: k8s-app
-                operator: In
-                values:
-                - cilium
-            topologyKey: kubernetes.io/hostname
      containers:
-      - args:
+      - name: cilium-agent
+        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        command:
+        - cilium-agent
+        args:
        - --config-dir=/tmp/cilium/config-map
 {% if cilium_mtu != "" %}
        - --mtu={{ cilium_mtu }}
@ -44,54 +44,6 @@ spec:
        - {{ flag }}
 {% endfor %}
 {% endif %}
-        command:
-        - cilium-agent
-        env:
-        - name: K8S_NODE_NAME
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: spec.nodeName
-        - name: CILIUM_K8S_NAMESPACE
-          valueFrom:
-            fieldRef:
-              apiVersion: v1
-              fieldPath: metadata.namespace
-        - name: CILIUM_CLUSTERMESH_CONFIG
-          value: /var/lib/cilium/clustermesh/
-{% if cilium_kube_proxy_replacement == 'strict' %}
-        - name: KUBERNETES_SERVICE_HOST
-          value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
-        - name: KUBERNETES_SERVICE_PORT
-          value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
-{% endif %}
-{% for env_var in cilium_agent_extra_env_vars %}
-        - {{ env_var | to_nice_yaml(indent=2) | indent(10) }}
-{% endfor %}
-        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
-        imagePullPolicy: {{ k8s_image_pull_policy }}
-        resources:
-          limits:
-            cpu: {{ cilium_cpu_limit }}
-            memory: {{ cilium_memory_limit }}
-          requests:
-            cpu: {{ cilium_cpu_requests }}
-            memory: {{ cilium_memory_requests }}
-        lifecycle:
-          postStart:
-            exec:
-              command:
-              - "/cni-install.sh"
-{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
-              - "--cni-exclusive={{ cilium_cni_exclusive }}"
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
-              - "--log-file={{ cilium_cni_log_file }}"
-{% endif %}
-          preStop:
-            exec:
-              command:
-              - /cni-uninstall.sh
        startupProbe:
          httpGet:
            host: '127.0.0.1'
@ -131,80 +83,109 @@ spec:
          successThreshold: 1
          failureThreshold: 3
          timeoutSeconds: 5
-        name: cilium-agent
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: CILIUM_CLUSTERMESH_CONFIG
+          value: /var/lib/cilium/clustermesh/
+{% if cilium_kube_proxy_replacement == 'strict' %}
+        - name: KUBERNETES_SERVICE_HOST
+          value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
+        - name: KUBERNETES_SERVICE_PORT
+          value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
+{% endif %}
+{% for env_var in cilium_agent_extra_env_vars %}
+        - {{ env_var | to_nice_yaml(indent=2) | indent(10) }}
+{% endfor %}
+        lifecycle:
+          postStart:
+            exec:
+              command:
+              - "/cni-install.sh"
+              - "--cni-exclusive={{ cilium_cni_exclusive | string | lower }}"
+{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
+              - "--enable-debug={{ cilium_debug | string | lower }}"
+              - "--log-file={{ cilium_cni_log_file }}"
+{% endif %}
+          preStop:
+            exec:
+              command:
+              - /cni-uninstall.sh
+        resources:
+          limits:
+            cpu: {{ cilium_cpu_limit }}
+            memory: {{ cilium_memory_limit }}
+          requests:
+            cpu: {{ cilium_cpu_requests }}
+            memory: {{ cilium_memory_requests }}
 {% if cilium_enable_prometheus or cilium_enable_hubble_metrics %}
        ports:
 {% endif %}
 {% if cilium_enable_prometheus %}
-        - containerPort: 9090
-          hostPort: 9090
-          name: prometheus
+        - name: prometheus
+          containerPort: {{ cilium_agent_scrape_port }}
+          hostPort: {{ cilium_agent_scrape_port }}
          protocol: TCP
 {% endif %}
 {% if cilium_enable_hubble_metrics %}
-        - containerPort: 9091
-          hostPort: 9091
-          name: hubble-metrics
+        - name: hubble-metrics
+          containerPort: {{ cilium_hubble_scrape_port }}
+          hostPort: {{ cilium_hubble_scrape_port }}
          protocol: TCP
 {% endif %}
        securityContext:
          privileged: true
        volumeMounts:
-        - mountPath: /sys/fs/bpf
-          name: bpf-maps
+        - name: bpf-maps
+          mountPath: /sys/fs/bpf
          mountPropagation: Bidirectional
-        - mountPath: /var/run/cilium
-          name: cilium-run
-        - mountPath: /host/opt/cni/bin
-          name: cni-path
-        - mountPath: /host/etc/cni/net.d
-          name: etc-cni-netd
-# pkg/workloads was depreca, removed in 1.7
-# https://github.com/cilium/cilium/pull/9447
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
-{% if container_manager == 'docker' %}
-        - mountPath: /var/run/docker.sock
-          name: docker-socket
-          readOnly: true
-{% else %}
-        - name: "{{ container_manager }}-socket"
-          mountPath: {{ cri_socket }}
-          readOnly: true
-{% endif %}
-{% endif %}
+        - name: cilium-run
+          mountPath: /var/run/cilium
+        - name: cni-path
+          mountPath: /host/opt/cni/bin
+        - name: etc-cni-netd
+          mountPath: /host/etc/cni/net.d
 {% if cilium_identity_allocation_mode == "kvstore" %}
-        - mountPath: /var/lib/etcd-config
-          name: etcd-config-path
+        - name: etcd-config-path
+          mountPath: /var/lib/etcd-config
          readOnly: true
-        - mountPath: "{{cilium_cert_dir}}"
-          name: etcd-secrets
+        - name: etcd-secrets
+          mountPath: "{{cilium_cert_dir}}"
          readOnly: true
 {% endif %}
-        - mountPath: /var/lib/cilium/clustermesh
-          name: clustermesh-secrets
+        - name: clustermesh-secrets
+          mountPath: /var/lib/cilium/clustermesh
          readOnly: true
-        - mountPath: /tmp/cilium/config-map
-          name: cilium-config-path
+        - name: cilium-config-path
+          mountPath: /tmp/cilium/config-map
          readOnly: true
-{% if cilium_ip_masq_agent_enable %}
+{% if cilium_ip_masq_agent_enable %}\
        - name: ip-masq-agent
          mountPath: /etc/config
          readOnly: true
 {% endif %}
          # Needed to be able to load kernel modules
-        - mountPath: /lib/modules
-          name: lib-modules
+        - name: lib-modules
+          mountPath: /lib/modules
          readOnly: true
-        - mountPath: /run/xtables.lock
-          name: xtables-lock
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
 {% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %}
-        - mountPath: /etc/ipsec
-          name: cilium-ipsec-secrets
+        - name: cilium-ipsec-secrets
+          mountPath: /etc/ipsec
          readOnly: true
 {% endif %}
 {% if cilium_hubble_install %}
-        - mountPath: /var/lib/cilium/tls/hubble
-          name: hubble-tls
+        - name: hubble-tls
+          mountPath: /var/lib/cilium/tls/hubble
          readOnly: true
 {% endif %}
 {% for volume_mount in cilium_agent_extra_volume_mounts %}
@ -245,22 +226,50 @@ spec:
        securityContext:
          privileged: true
 {% endif %}
-      - command:
+{% if cilium_version | regex_replace('v') is version('1.11.7', '>=') %}
+      - name: apply-sysctl-overwrites
+        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        env:
+        - name: BIN_PATH
+          value: /opt/cni/bin
+        command:
+        - sh
+        - -ec
+        # The statically linked Go program binary is invoked to avoid any
+        # dependency on utilities like sh that can be missing on certain
+        # distros installed on the underlying host. Copy the binary to the
+        # same directory where we install cilium cni plugin so that exec permissions
+        # are available.
+        - |
+          cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix;
+          nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix";
+          rm /hostbin/cilium-sysctlfix
+        volumeMounts:
+        - name: hostproc
+          mountPath: /hostproc
+        - name: cni-path
+          mountPath: /hostbin
+        securityContext:
+          privileged: true
+{% endif %}
+      - name: clean-cilium-state
+        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        command:
        - /init-container.sh
        env:
        - name: CILIUM_ALL_STATE
          valueFrom:
            configMapKeyRef:
-              key: clean-cilium-state
              name: cilium-config
+              key: clean-cilium-state
              optional: true
-        # CLEAN_CILIUM_BPF_STATE is deprecated in 1.6.
-        # https://github.com/cilium/cilium/pull/7478
-        - name: "{{ cilium_version | regex_replace('v') is version('1.6', '<')| ternary('CLEAN_CILIUM_BPF_STATE','CILIUM_BPF_STATE') }}"
+        - name: CILIUM_BPF_STATE
          valueFrom:
            configMapKeyRef:
-              key: clean-cilium-bpf-state
              name: cilium-config
+              key: clean-cilium-bpf-state
              optional: true
 # Removed in 1.11 and up.
 # https://github.com/cilium/cilium/commit/f7a3f59fd74983c600bfce9cac364b76d20849d9
@ -278,32 +287,25 @@ spec:
        - name: KUBERNETES_SERVICE_PORT
          value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
 {% endif %}
-{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
-        image: "{{cilium_init_image_repo}}:{{cilium_init_image_tag}}"
-{% else %}
-        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
-{% endif %}
-        imagePullPolicy: {{ k8s_image_pull_policy }}
-        name: clean-cilium-state
        securityContext:
          privileged: true
        volumeMounts:
-        - mountPath: /sys/fs/bpf
-          name: bpf-maps
+        - name: bpf-maps
+          mountPath: /sys/fs/bpf
 {% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
          # Required to mount cgroup filesystem from the host to cilium agent pod
        - name: cilium-cgroup
          mountPath: {{ cilium_cgroup_host_root }}
          mountPropagation: HostToContainer
 {% endif %}
-        - mountPath: /var/run/cilium
-          name: cilium-run
+        - name: cilium-run
+          mountPath: /var/run/cilium
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
-      priorityClassName: system-node-critical
      restartPolicy: Always
+      priorityClassName: system-node-critical
      serviceAccount: cilium
      serviceAccountName: cilium
      terminationGracePeriodSeconds: 1
@ -312,36 +314,26 @@ spec:
 {% if cilium_identity_allocation_mode == "kvstore" %}
      dnsPolicy: ClusterFirstWithHostNet
 {% endif %}
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: kubernetes.io/hostname
+            labelSelector:
+              matchLabels:
+                k8s-app: cilium
      tolerations:
      - operator: Exists
      volumes:
        # To keep state between restarts / upgrades
-      - hostPath:
+      - name: cilium-run
+        hostPath:
          path: /var/run/cilium
          type: DirectoryOrCreate
-        name: cilium-run
        # To keep state between restarts / upgrades for bpf maps
-      - hostPath:
+      - name: bpf-maps
+        hostPath:
          path: /sys/fs/bpf
          type: DirectoryOrCreate
-        name: bpf-maps
-# pkg/workloads was deprecated in 1.6, removed in 1.7
-# https://github.com/cilium/cilium/pull/9447
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
-{% if container_manager == 'docker' %}
-        # To read docker events from the node
-      - hostPath:
-          path: /var/run/docker.sock
-          type: Socket
-        name: docker-socket
-{% else %}
-        # To read crio events from the node
-      - hostPath:
-          path: {{ cri_socket }}
-          type: Socket
-        name: {{ container_manager }}-socket
-{% endif %}
-{% endif %}
 {% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
      # To mount cgroup2 filesystem on the host
      - name: hostproc
@ -355,34 +347,34 @@ spec:
          type: DirectoryOrCreate
 {% endif %}
        # To install cilium cni plugin in the host
-      - hostPath:
+      - name: cni-path
+        hostPath:
          path: /opt/cni/bin
          type: DirectoryOrCreate
-        name: cni-path
        # To install cilium cni configuration in the host
-      - hostPath:
+      - name: etc-cni-netd
+        hostPath:
          path: /etc/cni/net.d
          type: DirectoryOrCreate
-        name: etc-cni-netd
        # To be able to load kernel modules
-      - hostPath:
+      - name: lib-modules
+        hostPath:
          path: /lib/modules
-        name: lib-modules
        # To access iptables concurrently with other processes (e.g. kube-proxy)
-      - hostPath:
+      - name: xtables-lock
+        hostPath:
          path: /run/xtables.lock
          type: FileOrCreate
-        name: xtables-lock
 {% if cilium_identity_allocation_mode == "kvstore" %}
        # To read the etcd config stored in config maps
-      - configMap:
+      - name: etcd-config-path
+        configMap:
+          name: cilium-config
          # note: the leading zero means this number is in octal representation: do not remove it
          defaultMode: 0400
          items:
          - key: etcd-config
            path: etcd.config
-          name: cilium-config
-        name: etcd-config-path
        # To read the k8s etcd secrets in case the user might want to use TLS
      - name: etcd-secrets
        hostPath:
@ -391,21 +383,22 @@ spec:
        # To read the clustermesh configuration
      - name: clustermesh-secrets
        secret:
+          secretName: cilium-clustermesh
          # note: the leading zero means this number is in octal representation: do not remove it
          defaultMode: 0400
          optional: true
-          secretName: cilium-clustermesh
        # To read the configuration from the config map
-      - configMap:
+      - name: cilium-config-path
+        configMap:
          name: cilium-config
-        name: cilium-config-path
 {% if cilium_ip_masq_agent_enable %}
-      - configMap:
+      - name: ip-masq-agent
+        configMap:
          name: ip-masq-agent
+          optional: true
          items:
          - key: config
            path: ip-masq-agent
-        name: ip-masq-agent
 {% endif %}
 {% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %}
      - name: cilium-ipsec-secrets
@ -420,21 +413,12 @@ spec:
          sources:
          - secret:
              name: hubble-server-certs
+              optional: true
              items:
+                - key: ca.crt
+                  path: client-ca.crt
                - key: tls.crt
                  path: server.crt
                - key: tls.key
                  path: server.key
-              optional: true
-          - configMap:
-              name: hubble-ca-cert
-              items:
-                - key: ca.crt
-                  path: client-ca.crt
-              optional: true
 {% endif %}
-  updateStrategy:
-    rollingUpdate:
-      # Specifies the maximum number of Pods that can be unavailable during the update process.
-      maxUnavailable: 2
-    type: RollingUpdate