Fix cilium strict kube proxy replacement in HA (#6473)

* Update the cilium svc proxy test to HA mode

Signed-off-by: Arthur Outhenin-Chalandre <arthur@cri.epita.fr>

* Fix cilium strict kube-proxy in HA

Signed-off-by: Arthur Outhenin-Chalandre <arthur@cri.epita.fr>

* Add a single global endpoint variable

Signed-off-by: Arthur Outhenin-Chalandre <arthur@cri.epita.fr>

* Add cilium docs about kube-proxy replacement

Signed-off-by: Arthur Outhenin-Chalandre <arthur@cri.epita.fr>

* Fix issues in docs

Signed-off-by: Arthur Outhenin-Chalandre <arthur@cri.epita.fr>

docs/cilium.md

@@ -0,0 +1,13 @@
+# Cilium
+
+## Kube-proxy replacement with Cilium
+
+Cilium can run without kube-proxy by setting `cilium_kube_proxy_replacement`
+to `strict`.
+
+Without kube-proxy, cilium needs to know the address of the kube-apiserver
+and this must be set globally for all cilium components (agents and operators).
+Hence, in this configuration in Kubespray, Cilium will always contact
+the external loadbalancer (even from a node in the control plane)
+and if there is no external load balancer It will ignore any local load
+balancer deployed by Kubespray and **only contacts the first master**.

roles/kubespray-defaults/defaults/main.yaml

@@ -448,6 +448,12 @@ loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}"
 loadbalancer_apiserver_type: "nginx"
 # applied if only external loadbalancer_apiserver is defined, otherwise ignored
 apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local"
+kube_apiserver_global_endpoint: |-
+  {% if loadbalancer_apiserver is defined -%}
+      https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }}
+  {%- else -%}
+      https://{{ first_kube_master }}:{{ kube_apiserver_port }}
+  {%- endif %}
 kube_apiserver_endpoint: |-
   {% if loadbalancer_apiserver is defined -%}
       https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }}

roles/network_plugin/cilium/templates/cilium-deploy.yml.j2

@@ -84,6 +84,12 @@ spec:
                   key: AWS_DEFAULT_REGION
                   name: cilium-aws
                   optional: true
+{% if cilium_kube_proxy_replacement == 'strict' %}
+            - name: KUBERNETES_SERVICE_HOST
+              value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
+            - name: KUBERNETES_SERVICE_PORT
+              value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
+{% endif %}
           image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}"
           imagePullPolicy: {{ k8s_image_pull_policy }}
           name: cilium-operator

roles/network_plugin/cilium/templates/cilium-ds.yml.j2

@@ -43,6 +43,12 @@ spec:
               fieldPath: metadata.namespace
         - name: CILIUM_CLUSTERMESH_CONFIG
           value: /var/lib/cilium/clustermesh/
+{% if cilium_kube_proxy_replacement == 'strict' %}
+        - name: KUBERNETES_SERVICE_HOST
+          value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
+        - name: KUBERNETES_SERVICE_PORT
+          value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
+{% endif %}
         image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
         imagePullPolicy: {{ k8s_image_pull_policy }}
         resources:

tests/files/packet_debian10-cilium-svc-proxy.yml

@@ -1,7 +1,7 @@
 ---
 # Instance settings
 cloud_image: debian-10
-mode: separate
+mode: ha
 
 # Kubespray settings
 kube_network_plugin: cilium