diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 6388ca7af..b6fd9d954 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -49,6 +49,11 @@ bin_dir: /usr/local/bin ## The subnets of each nodes will be distributed by the datacenter router #peer_with_router: false +## Enable OSPF peering with list of routers +#calico_ospf_peers: +# - ip: 172.18.188.1 +# eligible: true + ## Upstream dns servers used by dnsmasq #upstream_dns_servers: # - 8.8.8.8 diff --git a/roles/network_plugin/calico/rr/files/bird.cfg.template b/roles/network_plugin/calico/rr/files/bird.cfg.template new file mode 100644 index 000000000..2d1827744 --- /dev/null +++ b/roles/network_plugin/calico/rr/files/bird.cfg.template @@ -0,0 +1,138 @@ +# Generated by confd +router id {{getenv "IP"}}; + +# Watch interface up/down events. +protocol device { + scan time 2; # Scan interfaces every 2 seconds +} + +# Template for all BGP clients +template bgp bgp_template { + debug off; + description "Connection to BGP peer"; + multihop; + import all; # Import all routes, since we don't know what the upstream + # topology is and therefore have to trust the ToR/RR. + export all; # Export all. + source address {{getenv "IP"}}; # The local address we use for the TCP connection + graceful restart; # See comment in kernel section about graceful restart. +} + +{{$our_rr_key := printf "/rr_v4/%s" (getenv "IP")}} +{{if ls $our_rr_key}}{{$our_rr_data := json (getv $our_rr_key)}} + + +{{if ls "/ospf_to_hosts"}} + +# import direct interfaces +protocol direct { + debug { states }; + interface -"cali*", -"docker*", "*"; +} + +# Synchronize them with the kernel +protocol kernel kern1 { + export all; +} + + +# OSPF Export filters +filter export_OSPF { + if ( net ~ 172.27.64.0/18 ) then { + print "export: net accepted:", net; + accept; + } + print "export: net rejected:", net; + reject; +} + +filter import_OSPF { + print "import: net rejected:", net; + reject; +} + +protocol ospf pods_ospf { + debug all; + export filter export_OSPF; + import filter import_OSPF; + area 0.0.0.0 { + interface "eth*" { + hello 10; + retransmit 5; + cost 10; + transmit delay 1; + dead count 4; + wait 40; + type ptp; + strict ptp yes; + authentication none; + priority 0; + neighbors { + {{range gets "/ospf_to_hosts/*"}}{{$ohost := json .Value}}{{$ohost.IP}}{{if $ohost.eligible}} eligible{{end}};{{ end }} + }; + }; + }; +} + +{{ end }} + + + + +# ------------- RR-to-RR full mesh ------------- +{{if ls "/rr_v4"}} +{{range gets "/rr_v4/*"}}{{$data := json .Value}}{{$rr_ip := $data.ip}} +{{$nums := split $rr_ip "."}}{{$id := join $nums "_"}} +# For RR {{$rr_ip}} +{{if eq $rr_ip (getenv "IP") }}# Skipping ourselves +{{else if ne "" $rr_ip}}protocol bgp Mesh_{{$id}} from bgp_template { + local as {{getv "/global/as_num"}}; + neighbor {{$rr_ip}} as {{getv "/global/as_num"}}; +}{{end}}{{end}} +{{end}} + + +# ------------- RR as a global peer ------------- +{{if ls "/global/peer_v4"}} +{{range gets "/global/peer_v4/*"}}{{$data := json .Value}} +{{if eq $data.ip (getenv "IP")}} +# This RR is a global peer with *all* calico nodes. +{{range $cnode := lsdir "/host"}} +{{$cnode_as_key := printf "/host/%s/as_num" $cnode}} +{{$cnode_ip_key := printf "/host/%s/ip_addr_v4" $cnode}}{{$cnode_ip := getv $cnode_ip_key}} +{{$nums := split $cnode_ip "."}}{{$id := join $nums "_"}} +# Peering with Calico node {{$cnode}} +protocol bgp Global_{{$id}} from bgp_template { + local as {{$data.as_num}}; + neighbor {{$cnode_ip}} as {{if exists $cnode_as_key}}{{getv $cnode_as_key}}{{else}}{{getv "/global/as_num"}}{{end}}; + rr client; + {{if $our_rr_data.cluster_id}}rr cluster id {{$our_rr_data.cluster_id}};{{end}} +} +{{end}} +{{end}} +{{end}} +{{end}} + + +# ------------- RR as a node-specific peer ------------- +{{range $cnode := lsdir "/host"}} +{{$node_peers_key := printf "/host/%s/peer_v4" $cnode}} +{{if ls $node_peers_key}} +{{range $peer := gets (printf "%s/*" $node_peers_key)}}{{$data := json $peer.Value}} +{{if eq $data.ip (getenv "IP")}} +{{$cnode_as_key := printf "/host/%s/as_num" $cnode}} +{{$cnode_ip_key := printf "/host/%s/ip_addr_v4" $cnode}}{{$cnode_ip := getv $cnode_ip_key}} +{{$nums := split $cnode_ip "."}}{{$id := join $nums "_"}} +# RR configured as a specific peer for calico node {{$peer.Key}} +protocol bgp Node_{{$id}} from bgp_template { + local as {{$data.as_num}}; + neighbor {{$cnode_ip}} as {{if exists $cnode_as_key}}{{getv $cnode_as_key}}{{else}}{{getv "/global/as_num"}}{{end}}; + rr client; + {{if $our_rr_data.cluster_id}}rr cluster id {{$our_rr_data.cluster_id}};{{end}} +} +{{end}} +{{end}} +{{end}} +{{end}} + +{{end}} diff --git a/roles/network_plugin/calico/rr/files/bird6.cfg.template b/roles/network_plugin/calico/rr/files/bird6.cfg.template new file mode 100644 index 000000000..764ce26e5 --- /dev/null +++ b/roles/network_plugin/calico/rr/files/bird6.cfg.template @@ -0,0 +1,85 @@ +# Generated by confd +router id {{getenv "IP"}}; # Use IPv4 address since router id is 4 octets, even in MP-BGP + +# Watch interface up/down events. +protocol device { + scan time 2; # Scan interfaces every 2 seconds +} + +{{if eq "" (getenv "IP6")}}# IPv6 disabled on this node. +{{else}} +# Template for all BGP clients +template bgp bgp_template { + debug off; + description "Connection to BGP peer"; + multihop; + import all; # Import all routes, since we don't know what the upstream + # topology is and therefore have to trust the ToR/RR. + export all; # Export all. + source address {{getenv "IP6"}}; # The local address we use for the TCP connection + graceful restart; # See comment in kernel section about graceful restart. +} + + +{{$our_rr_key := printf "/rr_v6/%s" (getenv "IP6")}} +{{if ls $our_rr_key}}{{$our_rr_data := json (getv $our_rr_key)}} + +# ------------- RR-to-RR full mesh ------------- +{{if ls "/rr_v6"}} +{{range gets "/rr_v6/*"}}{{$data := json .Value}}{{$rr_ip := $data.ip}} +{{$nums := split $rr_ip ":"}}{{$id := join $nums "_"}} +# For RR {{$rr_ip}} +{{if eq $rr_ip (getenv "IP6") }}# Skipping ourselves +{{else if ne "" $rr_ip}}protocol bgp Mesh_{{$id}} from bgp_template { + local as {{getv "/global/as_num"}}; + neighbor {{$rr_ip}} as {{getv "/global/as_num"}}; +}{{end}}{{end}} +{{end}} + + +# ------------- RR as a global peer ------------- +{{if ls "/global/peer_v6"}} +{{range gets "/global/peer_v6/*"}}{{$data := json .Value}} +{{if eq $data.ip (getenv "IP6")}} +# This RR is a global peer with *all* calico nodes. +{{range $cnode := lsdir "/host"}} +{{$cnode_as_key := printf "/host/%s/as_num" $cnode}} +{{$cnode_ip_key := printf "/host/%s/ip_addr_v6" $cnode}}{{$cnode_ip := getv $cnode_ip_key}} +{{$nums := split $cnode_ip "."}}{{$id := join $nums "_"}} +# Peering with Calico node {{$cnode}} +protocol bgp Global_{{$id}} from bgp_template { + local as {{$data.as_num}}; + neighbor {{$cnode_ip}} as {{if exists $cnode_as_key}}{{getv $cnode_as_key}}{{else}}{{getv "/global/as_num"}}{{end}}; + rr client; + {{if $our_rr_data.cluster_id}}rr cluster id {{$our_rr_data.cluster_id}};{{end}} +} +{{end}} +{{end}} +{{end}} +{{end}} + + +# ------------- RR as a node-specific peer ------------- +{{range $cnode := lsdir "/host"}} +{{$node_peers_key := printf "/host/%s/peer_v6" $cnode}} +{{if ls $node_peers_key}} +{{range $peer := gets (printf "%s/*" $node_peers_key)}}{{$data := json $peer.Value}} +{{if eq $data.ip (getenv "IP")}} +{{$cnode_as_key := printf "/host/%s/as_num" $cnode}} +{{$cnode_ip_key := printf "/host/%s/ip_addr_v6" $cnode}}{{$cnode_ip := getv $cnode_ip_key}} +{{$nums := split $cnode_ip "."}}{{$id := join $nums "_"}} +# RR configured as a specific peer for calico node {{$peer.Key}} +protocol bgp Node_{{$id}} from bgp_template { + local as {{$data.as_num}}; + neighbor {{$cnode_ip}} as {{if exists $cnode_as_key}}{{getv $cnode_as_key}}{{else}}{{getv "/global/as_num"}}{{end}}; + rr client; + {{if $our_rr_data.cluster_id}}rr cluster id {{$our_rr_data.cluster_id}};{{end}} +} +{{end}} +{{end}} +{{end}} +{{end}} + +{{end}} + +{{end}} diff --git a/roles/network_plugin/calico/rr/tasks/main.yml b/roles/network_plugin/calico/rr/tasks/main.yml index 5197aa005..a289e5361 100644 --- a/roles/network_plugin/calico/rr/tasks/main.yml +++ b/roles/network_plugin/calico/rr/tasks/main.yml @@ -26,6 +26,24 @@ - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"} - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"} +- name: Calico-rr | Create confd templates directory + file: + dest: "/templates" + state: directory + mode: 0750 + owner: root + group: root + +- name: Calico-rr | Link confd templates for calico-node + file: + src: "{{ item }}" + dest: "/templates/{{ item }}" + state: hard + force: yes + with_items: + - bird.cfg.template + - bird6.cfg.template + - name: Calico-rr | Create dir for logs file: path: /var/log/calico-rr @@ -56,6 +74,18 @@ }' delegate_to: "{{groups['etcd'][0]}}" +- name: Calico-rr | Configure OSPF Export + command: |- + {{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \ + set /calico/bgp/v1/ospf_to_hosts/{{item.ip}} \ + '{ + "IP": "{{item.ip}}", + "eligible": {{item.eligible}} + }' + delegate_to: "{{groups['etcd'][0]}}" + with_items: calico_ospf_peers + when: calico_ospf_peers|length > 1 + - meta: flush_handlers - name: Calico-rr | Enable calico-rr diff --git a/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 b/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 index f6da04a4d..71fbd5a85 100644 --- a/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 +++ b/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 @@ -15,6 +15,7 @@ ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \ -e ETCD_CERT_FILE=${ETCD_CERT_FILE} \ -e ETCD_KEY_FILE=${ETCD_KEY_FILE} \ -v /var/log/calico-rr:/var/log/calico \ + -v /templates:/templates:ro \ -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \ --memory={{ calico_rr_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_rr_cpu_limit|regex_replace('m', '') }} \ {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }}