[calico] calico rr supports multiple groups (#9134)

* update calico rr

* fix bgppeer conf

* fix yamllint

* fix ansible lint

* fix calico deploy

* fix yamllint

* fix some typo

docs/calico.md

@@ -172,6 +172,8 @@ node5
 
 [rack0:vars]
 cluster_id="1.0.0.1"
+calcio_rr_id=rr1
+calico_group_id=rr1
 ```
 
 The inventory above will deploy the following topology assuming that calico's

roles/network_plugin/calico/rr/tasks/update-node.yml

@@ -6,6 +6,15 @@
     set_fact:
       retry_count: "{{ 0 if retry_count is undefined else retry_count|int + 1 }}"
 
+  - name: Calico | Set label for route reflector  # noqa 301 305
+    shell: "{{ bin_dir }}/calicoctl.sh label node  {{ inventory_hostname }} calico-rr-id={{ calcio_rr_id }} --overwrite"
+    changed_when: false
+    register: calico_rr_id_label
+    until: calico_rr_id_label is succeeded
+    delay: "{{ retry_stagger | random + 3 }}"
+    retries: 10
+    when: calcio_rr_id is defined
+
   - name: Calico-rr | Fetch current node object
     command: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }} -ojson"
     changed_when: false

roles/network_plugin/calico/tasks/install.yml

@@ -346,83 +346,6 @@
   when:
     - inventory_hostname == groups['kube_control_plane'][0]
 
-- name: Calico | Configure peering with router(s) at global scope
-  command:
-    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
-    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
-  vars:
-    stdin: >
-      {"apiVersion": "projectcalico.org/v3",
-      "kind": "BGPPeer",
-      "metadata": {
-        "name": "global-{{ item.name | default(item.router_id|replace(':','-')) }}"
-      },
-      "spec": {
-        "asNumber": "{{ item.as }}",
-        "peerIP": "{{ item.router_id }}"
-      }}
-  register: output
-  retries: 4
-  until: output.rc == 0
-  delay: "{{ retry_stagger | random + 3 }}"
-  with_items:
-    - "{{ peers|selectattr('scope','defined')|selectattr('scope','equalto', 'global')|list|default([]) }}"
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0]
-    - peer_with_router|default(false)
-
-- name: Calico | Configure peering with route reflectors at global scope
-  command:
-    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
-    # revert when it's already a string
-    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
-  vars:
-    stdin: >
-      {"apiVersion": "projectcalico.org/v3",
-      "kind": "BGPPeer",
-      "metadata": {
-        "name": "peer-to-rrs"
-      },
-      "spec": {
-        "nodeSelector": "!has(i-am-a-route-reflector)",
-        "peerSelector": "has(i-am-a-route-reflector)"
-      }}
-  register: output
-  retries: 4
-  until: output.rc == 0
-  delay: "{{ retry_stagger | random + 3 }}"
-  with_items:
-    - "{{ groups['calico_rr'] | default([]) }}"
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0]
-    - peer_with_calico_rr|default(false)
-
-- name: Calico | Configure route reflectors to peer with each other
-  command:
-    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
-    # revert when it's already a string
-    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
-  vars:
-    stdin: >
-      {"apiVersion": "projectcalico.org/v3",
-      "kind": "BGPPeer",
-      "metadata": {
-        "name": "rr-mesh"
-      },
-      "spec": {
-        "nodeSelector": "has(i-am-a-route-reflector)",
-        "peerSelector": "has(i-am-a-route-reflector)"
-      }}
-  register: output
-  retries: 4
-  until: output.rc == 0
-  delay: "{{ retry_stagger | random + 3 }}"
-  with_items:
-    - "{{ groups['calico_rr'] | default([]) }}"
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0]
-    - peer_with_calico_rr|default(false)
-
 - name: Calico | Create calico manifests
   template:
     src: "{{ item.file }}.j2"
@@ -519,61 +442,6 @@
     - inventory_hostname not in groups['kube_control_plane']
     - calico_datastore == "kdd"
 
-- name: Calico | Configure node asNumber for per node peering
-  command:
-    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
-    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
-  vars:
-    stdin: >
-      {"apiVersion": "projectcalico.org/v3",
-      "kind": "Node",
-      "metadata": {
-        "name": "{{ inventory_hostname }}"
-      },
-      "spec": {
-        "bgp": {
-          "asNumber": "{{ local_as }}"
-        },
-        "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}]
-      }}
-  register: output
-  retries: 4
-  until: output.rc == 0
-  delay: "{{ retry_stagger | random + 3 }}"
-  when:
-    - peer_with_router|default(false)
-    - inventory_hostname in groups['k8s_cluster']
-    - local_as is defined
-    - groups['calico_rr'] | default([]) | length == 0
-
-- name: Calico | Configure peering with router(s) at node scope
-  command:
-    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
-    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
-  vars:
-    stdin: >
-      {"apiVersion": "projectcalico.org/v3",
-      "kind": "BGPPeer",
-      "metadata": {
-        "name": "{{ inventory_hostname }}-{{ item.name | default(item.router_id|replace(':','-')) }}"
-      },
-      "spec": {
-        "asNumber": "{{ item.as }}",
-        "node": "{{ inventory_hostname }}",
-        "peerIP": "{{ item.router_id }}",
-        "sourceAddress": "{{ item.sourceaddress|default('UseNodeIP') }}"
-      }}
-  register: output
-  retries: 4
-  until: output.rc == 0
-  delay: "{{ retry_stagger | random + 3 }}"
-  with_items:
-    - "{{ peers|selectattr('scope','undefined')|list|default([]) | union(peers|selectattr('scope','defined')|selectattr('scope','equalto', 'node')|list|default([])) }}"
-  delegate_to: "{{ groups['kube_control_plane'][0] }}"
-  when:
-    - peer_with_router|default(false)
-    - inventory_hostname in groups['k8s_cluster']
-
 - name: Calico | Create Calico ipam manifests
   template:
     src: "{{ item.file }}.j2"
@@ -591,3 +459,11 @@
     state: "latest"
   when:
     - inventory_hostname == groups['kube_control_plane'][0]
+
+- include_tasks: peer_with_calico_rr.yml
+  when:
+    - peer_with_calico_rr|default(false)
+
+- include_tasks: peer_with_router.yml
+  when:
+    - peer_with_router|default(false)

roles/network_plugin/calico/tasks/peer_with_calico_rr.yml

@@ -0,0 +1,86 @@
+---
+- name: Calico | Set lable for groups nodes  # noqa 301 305
+  shell: "{{ bin_dir }}/calicoctl.sh label node  {{ inventory_hostname }} calico-group-id={{ calico_group_id }} --overwrite"
+  changed_when: false
+  register: calico_group_id_label
+  until: calico_group_id_label is succeeded
+  delay: "{{ retry_stagger | random + 3 }}"
+  retries: 10
+  when:
+    - calico_group_id is defined
+
+- name: Calico | Configure peering with route reflectors at global scope
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    # revert when it's already a string
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "{{ calcio_rr_id }}-to-node"
+      },
+      "spec": {
+        "peerSelector": "calico-rr-id == '{{ calcio_rr_id }}'",
+        "nodeSelector": "calico-group-id == '{{ calico_group_id }}'"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  when:
+    - calcio_rr_id is defined
+    - calico_group_id is defined
+    - inventory_hostname in groups['calico_rr']
+
+- name: Calico | Configure peering with route reflectors at global scope
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    # revert when it's already a string
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "peer-to-rrs"
+      },
+      "spec": {
+        "nodeSelector": "!has(i-am-a-route-reflector)",
+        "peerSelector": "has(i-am-a-route-reflector)"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ groups['calico_rr'] | default([]) }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - calcio_rr_id is not defined or calico_group_id is not defined
+
+- name: Calico | Configure route reflectors to peer with each other
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    # revert when it's already a string
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "rr-mesh"
+      },
+      "spec": {
+        "nodeSelector": "has(i-am-a-route-reflector)",
+        "peerSelector": "has(i-am-a-route-reflector)"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ groups['calico_rr'] | default([]) }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]

roles/network_plugin/calico/tasks/peer_with_router.yml

@@ -0,0 +1,77 @@
+---
+- name: Calico | Configure peering with router(s) at global scope
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "global-{{ item.name | default(item.router_id|replace(':','-')) }}"
+      },
+      "spec": {
+        "asNumber": "{{ item.as }}",
+        "peerIP": "{{ item.router_id }}"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ peers|selectattr('scope','defined')|selectattr('scope','equalto', 'global')|list|default([]) }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Calico | Configure node asNumber for per node peering
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "Node",
+      "metadata": {
+        "name": "{{ inventory_hostname }}"
+      },
+      "spec": {
+        "bgp": {
+          "asNumber": "{{ local_as }}"
+        },
+        "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}]
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  when:
+    - inventory_hostname in groups['k8s_cluster']
+    - local_as is defined
+    - groups['calico_rr'] | default([]) | length == 0
+
+- name: Calico | Configure peering with router(s) at node scope
+  command:
+    cmd: "{{ bin_dir }}/calicoctl.sh apply -f -"
+    stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}"
+  vars:
+    stdin: >
+      {"apiVersion": "projectcalico.org/v3",
+      "kind": "BGPPeer",
+      "metadata": {
+        "name": "{{ inventory_hostname }}-{{ item.name | default(item.router_id|replace(':','-')) }}"
+      },
+      "spec": {
+        "asNumber": "{{ item.as }}",
+        "node": "{{ inventory_hostname }}",
+        "peerIP": "{{ item.router_id }}",
+        "sourceAddress": "{{ item.sourceaddress|default('UseNodeIP') }}"
+      }}
+  register: output
+  retries: 4
+  until: output.rc == 0
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ peers|selectattr('scope','undefined')|list|default([]) | union(peers|selectattr('scope','defined')|selectattr('scope','equalto', 'node')|list|default([])) }}"
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  when:
+    - inventory_hostname in groups['k8s_cluster']