Calico enable support for eBPF (#7618)

* Calico: align manifests with upstream

* allow enabling typha prometheus metrics

* Calico: enable eBPF support

* manage the kubernetes-services-endpoint configmap

* Calico: document the use of eBPF dataplane

* Calico: improve checks before deployment

* enforce disabling kube-proxy when using eBPF dataplane
* ensure calico_version is supported

docs/calico.md

@@ -260,3 +260,73 @@ calico_ipam_host_local: true
 ```
 
 Refer to Project Calico section [Using host-local IPAM](https://docs.projectcalico.org/reference/cni-plugin/configuration#using-host-local-ipam) for further information.
+
+## eBPF Support
+
+Calico supports eBPF for its data plane see [an introduction to the Calico eBPF Dataplane](https://www.projectcalico.org/introducing-the-calico-ebpf-dataplane/) for further information.
+
+Note that it is advisable to always use the latest version of Calico when using the eBPF dataplane.
+
+### Enabling eBPF support
+
+To enable the eBPF dataplane support ensure you add the following to your inventory. Note that the `kube-proxy` is incompatible with running Calico in eBPF mode and the kube-proxy should be removed from the system.
+
+```yaml
+calico_bpf_enabled: true
+kube_proxy_remove: true
+```
+
+### Cleaning up after kube-proxy
+
+Calico node cannot clean up after kube-proxy has run in ipvs mode. If you are converting an existing cluster to eBPF you will need to ensure the `kube-proxy` DaemonSet is deleted and that ipvs rules are cleaned.
+
+To check that kube-proxy was running in ipvs mode:
+
+```ShellSession
+# ipvsadm -l
+```
+
+To clean up any ipvs leftovers:
+
+```ShellSession
+# ipvsadm -C
+```
+
+### Calico access to the kube-api
+
+Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.projectcalico.org/maintenance/ebpf/enabling-bpf) for guidelines on how to do this.
+
+Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](./ha-mode.md).
+
+If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints.
+
+```yaml
+loadbalancer_apiserver_localhost: true
+use_localhost_as_kubeapi_loadbalancer: true
+```
+
+### Tunneled versus Direct Server Return
+
+By default Calico usese Tunneled service mode but it can use direct server return (DSR) in order to optimize the return path for a service.
+
+To configure DSR:
+
+```yaml
+calico_bpf_service_mode: "DSR"
+```
+
+### eBPF Logging and Troubleshooting
+
+In order to enable Calico eBPF mode logging:
+
+```yaml
+calico_bpf_log_level: "Debug"
+```
+
+To view the logs you need to use the `tc` command to read the kernel trace buffer:
+
+```ShellSession
+tc exec bpf debug
+```
+
+Please see [Calico eBPF troubleshooting guide](https://docs.projectcalico.org/maintenance/troubleshoot/troubleshoot-ebpf#ebpf-program-debug-logs).

roles/download/defaults/main.yml

@@ -69,6 +69,7 @@ quay_image_repo: "quay.io"
 calico_version: "v3.18.4"
 calico_ctl_version: "{{ calico_version }}"
 calico_cni_version: "{{ calico_version }}"
+calico_flexvol_version: "{{ calico_version }}"
 calico_policy_version: "{{ calico_version }}"
 calico_typha_version: "{{ calico_version }}"
 typha_enabled: false
@@ -439,6 +440,8 @@ calico_node_image_repo: "{{ quay_image_repo }}/calico/node"
 calico_node_image_tag: "{{ calico_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
 calico_cni_image_repo: "{{ quay_image_repo }}/calico/cni"
 calico_cni_image_tag: "{{ calico_cni_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
+calico_flexvol_image_repo: "{{ quay_image_repo }}/calico/pod2daemon-flexvol"
+calico_flexvol_image_tag: "{{ calico_flexvol_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
 calico_policy_image_repo: "{{ quay_image_repo }}/calico/kube-controllers"
 calico_policy_image_tag: "{{ calico_policy_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}"
 calico_typha_image_repo: "{{ quay_image_repo }}/calico/typha"

roles/kubernetes/preinstall/tasks/0020-verify-settings.yml

@@ -47,6 +47,25 @@
     - kube_network_plugin == 'calico'
     - not ignore_assert_errors
 
+- name: Stop if supported Calico versions
+  assert:
+    that:
+      - "calico_version in calico_crds_archive_checksums.keys()"
+    msg: "Calico version not supported {{ calico_version }} not in {{ calico_crds_archive_checksums.keys() }}"
+  when:
+    - kube_network_plugin == 'calico'
+    - not ignore_assert_errors
+
+- name: Stop if kube-proxy is enabled when using eBPF dataplane
+  assert:
+    that:
+      - kube_proxy_remove
+    msg: "kube-proxy needs to be disabled when using Calico with eBPF dataplane"
+  when:
+    - calico_bpf_enabled | default(false)
+    - kube_network_plugin == 'calico'
+    - not ignore_assert_errors
+
 - name: Stop if unsupported version of Kubernetes
   assert:
     that: kube_version is version(kube_version_min_required, '>=')

roles/network_plugin/calico/defaults/main.yml

@@ -32,6 +32,12 @@ calico_advertise_service_external_ips: []
 # Adveritse Service LoadBalancer IPs
 calico_advertise_service_loadbalancer_ips: []
 
+# Calico eBPF support
+calico_bpf_enabled: false
+calico_bpf_log_level: ""
+# Valid option for service mode: Tunnel (default), DSR=Direct Server Return
+calico_bpf_service_mode: Tunnel
+
 # Limits for apps
 calico_node_memory_limit: 500M
 calico_node_cpu_limit: 300m
@@ -91,6 +97,8 @@ kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
 
 # Use typha (only with kdd)
 typha_enabled: false
+typha_prometheusmetricsenabled: false
+typha_prometheusmetricsport: 9093
 
 # Scaling typha: 1 replica per 100 nodes is adequate
 # Number of typha replicas
@@ -106,3 +114,7 @@ calico_feature_control: {}
 
 # Calico default BGP port
 calico_bgp_listen_port: 179
+
+# Calico FelixConfiguration options
+calico_felix_reporting_interval: 0s
+calico_felix_log_severity_screen: Info

roles/network_plugin/calico/tasks/install.yml

@@ -131,6 +131,27 @@
     - inventory_hostname in groups['kube_control_plane']
     - calico_datastore == "kdd"
 
+- name: Calico | Configure calico FelixConfiguration
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      { "kind": "FelixConfiguration",
+        "apiVersion": "projectcalico.org/v3",
+        "metadata": {
+          "name": "default",
+        },
+        "spec": {
+          "ipipEnabled": {{ calico_ipip_mode != 'Never' | bool }},
+          "reportingInterval": "{{ calico_felix_reporting_interval }}",
+          "bpfLogLevel": "{{ calico_bpf_log_level }}",
+          "bpfEnabled": {{ calico_bpf_enabled | bool }},
+          "bpfExternalServiceMode": "{{ calico_bpf_service_mode }}",
+          "logSeverityScreen": "{{ calico_felix_log_severity_screen }}" }}
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
 - name: Calico | Configure calico network pool
   command:
     cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
@@ -302,6 +323,7 @@
     - {name: calico, file: calico-node-sa.yml, type: sa}
     - {name: calico, file: calico-cr.yml, type: clusterrole}
     - {name: calico, file: calico-crb.yml, type: clusterrolebinding}
+    - {name: kubernetes-services-endpoint, file: kubernetes-services-endpoint.yml, type: cm }
   register: calico_node_manifests
   when:
     - inventory_hostname in groups['kube_control_plane']

roles/network_plugin/calico/templates/calico-cr.yml.j2

@@ -28,6 +28,7 @@ rules:
     resources:
       - nodes/status
     verbs:
+      # Needed for clearing NodeNetworkUnavailable flag.
       - patch
 {% if calico_datastore == "etcd" %}
   - apiGroups:

roles/network_plugin/calico/templates/calico-node.yml.j2

@@ -44,6 +44,11 @@ spec:
         - name: upgrade-ipam
           image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
           command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
+          envFrom:
+            - configMapRef:
+                # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+                name: kubernetes-services-endpoint
+                optional: true
           env:
             - name: KUBERNETES_NODE_NAME
               valueFrom:
@@ -94,12 +99,26 @@ spec:
               name: cni-bin-dir
           securityContext:
             privileged: true
+        # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes
+        # to communicate with Felix over the Policy Sync API.
+        - name: flexvol-driver
+          image: {{ calico_flexvol_image_repo }}:{{ calico_flexvol_image_tag }}
+          volumeMounts:
+            - name: flexvol-driver-host
+              mountPath: /host/driver
+          securityContext:
+            privileged: true
       containers:
         # Runs calico/node container on each Kubernetes node.  This
         # container programs network policy and routes on each
         # host.
         - name: calico-node
           image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }}
+          envFrom:
+            - configMapRef:
+                # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+                name: kubernetes-services-endpoint
+                optional: true
           env:
             # The location of the Calico etcd cluster.
 {% if calico_datastore == "etcd" %}
@@ -231,8 +250,6 @@ spec:
 {% if calico_ip_auto_method is defined %}
             - name: IP_AUTODETECTION_METHOD
               value: "{{ calico_ip_auto_method }}"
-            - name: IP
-              value: "autodetect"
 {% else %}
             - name: NODEIP
               valueFrom:
@@ -240,9 +257,9 @@ spec:
                   fieldPath: status.hostIP
             - name: IP_AUTODETECTION_METHOD
               value: "can-reach=$(NODEIP)"
+{% endif %}
             - name: IP
               value: "autodetect"
-{% endif %}
 {% if enable_dual_stack_networks %}
             - name: IP6
               value: autodetect
@@ -286,10 +303,10 @@ spec:
 {% if calico_network_backend|default("bird") == "bird" %}
               - -bird-live
 {% endif %}
-            initialDelaySeconds: 5
+            periodSeconds: 10
+            initialDelaySeconds: 10
             failureThreshold: 6
           readinessProbe:
-            failureThreshold: 6
             exec:
               command:
               - /bin/calico-node
@@ -297,18 +314,22 @@ spec:
               - -bird-ready
 {% endif %}
               - -felix-ready
+            periodSeconds: 10
+            failureThreshold: 6
           volumeMounts:
             - mountPath: /lib/modules
               name: lib-modules
               readOnly: true
             - mountPath: /var/run/calico
               name: var-run-calico
+              readOnly: false
             - mountPath: /var/lib/calico
               name: var-lib-calico
               readOnly: false
 {% if calico_datastore == "etcd" %}
             - mountPath: /calico-secrets
               name: etcd-certs
+              readOnly: true
 {% endif %}
             - name: xtables-lock
               mountPath: /run/xtables.lock
@@ -324,7 +345,20 @@ spec:
               mountPath: /etc/typha-ca/ca.crt
               readOnly: true
 {% endif %}
-
+            - name: policysync
+              mountPath: /var/run/nodeagent
+{% if calico_bpf_enabled %}
+            # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the
+            # parent directory.
+            - name: sysfs
+              mountPath: /sys/fs/
+              # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host.
+              # If the host is known to mount that filesystem already then Bidirectional can be omitted.
+              mountPropagation: Bidirectional
+{% endif %}
+            - name: cni-log-dir
+              mountPath: /var/log/calico/cni
+              readOnly: true
       volumes:
         # Used by calico/node.
         - name: lib-modules
@@ -375,6 +409,26 @@ spec:
           hostPath:
             path: "/etc/kubernetes/ssl/"
 {% endif %}
+{% if calico_bpf_enabled %}
+        - name: sysfs
+          hostPath:
+            path: /sys/fs/
+            type: DirectoryOrCreate
+{% endif %}
+        # Used to access CNI logs.
+        - name: cni-log-dir
+          hostPath:
+            path: /var/log/calico/cni
+        # Used to create per-pod Unix Domain Sockets
+        - name: policysync
+          hostPath:
+            type: DirectoryOrCreate
+            path: /var/run/nodeagent
+        # Used to install Flex Volume Driver
+        - name: flexvol-driver-host
+          hostPath:
+            type: DirectoryOrCreate
+            path: "{{ kubelet_flexvolumes_plugins_dir | default('/usr/libexec/kubernetes/kubelet-plugins/volume/exec') }}/nodeagent~uds"
   updateStrategy:
     rollingUpdate:
       maxUnavailable: {{ serial | default('20%') }}

roles/network_plugin/calico/templates/calico-typha.yml.j2

@@ -46,6 +46,10 @@ spec:
         k8s-app: calico-typha
       annotations:
         cluster-autoscaler.kubernetes.io/safe-to-evict: 'true'
+{% if typha_prometheusmetricsenabled %}
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: "{{ typha_prometheusmetricsport }}"
+{% endif %}
     spec:
       nodeSelector:
         kubernetes.io/os: linux
@@ -61,6 +65,9 @@ spec:
       # as a host-networked pod.
       serviceAccountName: calico-node
       priorityClassName: system-cluster-critical
+      # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573
+      securityContext:
+        fsGroup: 65534
       containers:
       - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }}
         name: calico-typha
@@ -68,6 +75,11 @@ spec:
         - containerPort: 5473
           name: calico-typha
           protocol: TCP
+        envFrom:
+        - configMapRef:
+            # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
+            name: kubernetes-services-endpoint
+            optional: true
         env:
           # Enable "info" logging by default.  Can be set to "debug" to increase verbosity.
           - name: TYPHA_LOGSEVERITYSCREEN
@@ -105,13 +117,14 @@ spec:
             name: cacert
             readOnly: true
 {% endif %}
-          # Uncomment these lines to enable prometheus metrics.  Since Typha is host-networked,
+{% if typha_prometheusmetricsenabled %}
+          # Since Typha is host-networked,
           # this opens a port on the host, which may need to be secured.
-          #- name: TYPHA_PROMETHEUSMETRICSENABLED
+          - name: TYPHA_PROMETHEUSMETRICSENABLED
-          #  value: "true"
+            value: "true"
-          #- name: TYPHA_PROMETHEUSMETRICSPORT
+          - name: TYPHA_PROMETHEUSMETRICSPORT
-          #  value: "9093"
+            value: "{{ typha_prometheusmetricsport }}"
-
+{% endif %}
           # Needed for version >=3.7 when the 'host-local' ipam is used
           # Should never happen given templates/cni-calico.conflist.j2
           # Configure route aggregation based on pod CIDR.

roles/network_plugin/calico/templates/cni-calico.conflist.j2

@@ -64,6 +64,12 @@
       "capabilities": {
         "portMappings": true
       }
+    },
+    {
+      "type":"bandwidth",
+      "capabilities": {
+        "bandwidth": true
+      }
     }
   ]
 }

roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2

@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: kube-system
+  name: kubernetes-services-endpoint
+data:
+{% if calico_bpf_enabled %}
+{% if loadbalancer_apiserver is defined %}
+  KUBERNETES_SERVICE_HOST: "{{ apiserver_loadbalancer_domain_name }}"
+  KUBERNETES_SERVICE_PORT: "{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}"
+{%- elif use_localhost_as_kubeapi_loadbalancer|default(False)|bool %}
+  KUBERNETES_SERVICE_HOST: "127.0.0.1"
+  KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}"
+{%- else %}
+  KUBERNETES_SERVICE_HOST: "{{ first_kube_master }}"
+  KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}"
+{%- endif %}
+{% endif %}