Support configuring the Calico iptables insert mode (#5473)

* Support configuring the insert mode

Defaults to the upstream default https://docs.projectcalico.org/v3.9/reference/felix/configuration

so nothing should change for existing deployments.

This allows coexistence with other firewall management technologies.

* Add a note to the sample config

inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml

@@ -63,3 +63,5 @@
 # * interface=INTERFACE-REGEX
 # see https://docs.projectcalico.org/reference/node/configuration
 # calico_ip_auto_method: "interface=eth.*"
+# Choose the iptables insert mode for Calico: "Insert" or "Append".
+# calico_felix_chaininsertmode: Insert

roles/network_plugin/calico/defaults/main.yml

@@ -31,6 +31,7 @@ calicoctl_memory_limit: 170M
 calicoctl_cpu_limit: 100m
 calicoctl_memory_requests: 32M
 calicoctl_cpu_requests: 250m
+calico_felix_chaininsertmode: Insert
 
 # Enable Prometheus Metrics endpoint for felix
 calico_felix_prometheusmetricsenabled: false

roles/network_plugin/calico/templates/calico-node.yml.j2

@@ -228,6 +228,8 @@ spec:
             - name: FELIX_IPINIPMTU
               value: "{{ calico_mtu }}"
 {% endif %}
+            - name: FELIX_CHAININSERTMODE
+              value: "{{ calico_felix_chaininsertmode }}"
             - name: FELIX_PROMETHEUSMETRICSENABLED
               value: "{{ calico_felix_prometheusmetricsenabled }}"
             - name: FELIX_PROMETHEUSMETRICSPORT