From 14cf3e138b4aa2e268c7328cf045be291ccbe548 Mon Sep 17 00:00:00 2001 From: Cristian Calin <6627509+cristicalin@users.noreply.github.com> Date: Wed, 12 May 2021 15:22:17 +0300 Subject: [PATCH] Support Calico advertisement of MetalLB LoadBalancer IPs (#7593) * add initial MetalLB docs * metallb allow disabling the deployment of the metallb speaker * calico>=3.18 allow using calico to advertise service loadbalancer IPs * Document the use of MetalLB and Calico * clean MetalLB docs --- README.md | 2 + docs/metallb.md | 81 +++++++++++++++++++ .../sample/group_vars/k8s_cluster/addons.yml | 1 + .../group_vars/k8s_cluster/k8s-net-calico.yml | 5 ++ roles/kubernetes-apps/metallb/README.md | 17 ---- .../kubernetes-apps/metallb/defaults/main.yml | 1 + .../metallb/templates/metallb.yml.j2 | 10 +++ roles/network_plugin/calico/defaults/main.yml | 3 + roles/network_plugin/calico/tasks/install.yml | 7 ++ 9 files changed, 110 insertions(+), 17 deletions(-) create mode 100644 docs/metallb.md delete mode 100644 roles/kubernetes-apps/metallb/README.md diff --git a/README.md b/README.md index d89e19044..2d5dc6168 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,8 @@ See also [Network checker](docs/netcheck.md). - [nginx](https://kubernetes.github.io/ingress-nginx): the NGINX Ingress Controller. +- [metallb](docs/metallb.md): the MetalLB bare-metal service LoadBalancer provider. + ## Community docs and resources - [kubernetes.io/docs/setup/production-environment/tools/kubespray/](https://kubernetes.io/docs/setup/production-environment/tools/kubespray/) diff --git a/docs/metallb.md b/docs/metallb.md new file mode 100644 index 000000000..e8165d2e8 --- /dev/null +++ b/docs/metallb.md @@ -0,0 +1,81 @@ +# MetalLB + +MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer implementation. +It allows you to create Kubernetes services of type "LoadBalancer" in clusters that don't run on a cloud provider, and thus cannot simply hook into 3rd party products to provide load-balancers. +The default operationg mode of MetalLB is in ["Layer2"](https://metallb.universe.tf/concepts/layer2/) but it can also operate in ["BGP"](https://metallb.universe.tf/concepts/bgp/) mode. + +## Install + +You have to explicitly enable the MetalLB extension and set an IP address range from which to allocate LoadBalancer IPs. + +```yaml +metallb_enabled: true +metallb_speaker_enabled: true +metallb_ip_range: + - 10.5.0.0/16 +``` + +By default only the MetalLB BGP speaker is allowed to run on control plane nodes. If you have a single node cluster or a cluster where control plane are also worker nodes you may need to enable tolerations for the MetalLB controller: + +```yaml +metallb_controller_tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Equal" + value: "" + effect: "NoSchedule" +``` + +## BGP Mode + +When operating in BGP Mode MetalLB needs to have defined upstream peers: + +```yaml +metallb_protocol: bgp +metallb_ip_range: + - 10.5.0.0/16 +metallb_peers: + - peer_address: 192.0.2.1 + peer_asn: 64512 + my_asn: 4200000000 + - peer_address: 192.0.2.2 + peer_asn: 64513 + my_asn: 4200000000 +``` + +When using calico >= 3.18 you can replace MetalLB speaker by calico Service LoadBalancer IP advertisement. +See [calico service IPs advertisement documentation](https://docs.projectcalico.org/archive/v3.18/networking/advertise-service-ips#advertise-service-load-balancer-ip-addresses). +In this scenarion you should disable the MetalLB speaker and configure the `calico_advertise_service_loadbalancer_ips` to match your `metallb_ip_range` + +```yaml +metallb_speaker_enabled: false +metallb_ip_range: + - 10.5.0.0/16 +calico_advertise_service_loadbalancer_ips: "{{ metallb_ip_range }}" +``` + +If you have additional loadbalancer IP pool in `metallb_additional_address_pools`, ensure to add them to the list. + +```yaml +metallb_speaker_enabled: false +metallb_ip_range: + - 10.5.0.0/16 +metallb_additional_address_pools: + kube_service_pool_1: + ip_range: + - 10.6.0.0/16 + protocol: "bgp" + auto_assign: false + kube_service_pool_2: + ip_range: + - 10.10.0.0/16 + protocol: "bgp" + auto_assign: false +calico_advertise_service_loadbalancer_ips: + - 10.5.0.0/16 + - 10.6.0.0/16 + - 10.10.0.0/16 +``` diff --git a/inventory/sample/group_vars/k8s_cluster/addons.yml b/inventory/sample/group_vars/k8s_cluster/addons.yml index cee77d634..1d08337a8 100644 --- a/inventory/sample/group_vars/k8s_cluster/addons.yml +++ b/inventory/sample/group_vars/k8s_cluster/addons.yml @@ -132,6 +132,7 @@ cert_manager_enabled: false # MetalLB deployment metallb_enabled: false +metallb_speaker_enabled: true # metallb_ip_range: # - "10.5.0.50-10.5.0.99" # metallb_speaker_nodeselector: diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml b/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml index 50215e4ca..84f24b414 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml @@ -50,6 +50,11 @@ # - x.x.x.x/24 # - y.y.y.y/32 +# Adveritse Service LoadBalancer IPs +# calico_advertise_service_loadbalancer_ips: +# - x.x.x.x/24 +# - y.y.y.y/16 + # Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore) # calico_datastore: "kdd" diff --git a/roles/kubernetes-apps/metallb/README.md b/roles/kubernetes-apps/metallb/README.md deleted file mode 100644 index 1456a6e8a..000000000 --- a/roles/kubernetes-apps/metallb/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Deploy MetalLB into Kubespray/Kubernetes - -MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer implementation. -In short, it allows you to create Kubernetes services of type "LoadBalancer" in clusters that -don't run on a cloud provider, and thus cannot simply hook into paid products to provide load-balancers. -This addon aims to automate [MetalLB in layer 2 mode](https://metallb.universe.tf/concepts/layer2/) -or [MetalLB in BGP mode](https://metallb.universe.tf/concepts/bgp/). -It deploys MetalLB into Kubernetes and sets up a layer 2 or BGP load-balancer. - -## Install - -In the default, MetalLB is not deployed into your Kubernetes cluster. -You can override the defaults by copying the contents of roles/kubernetes-apps/metallb/defaults/main.yml -to somewhere in inventory/mycluster/group_vars such as inventory/mycluster/groups_vars/k8s_cluster/addons.yml -and updating metallb_enabled option to `true`. -In addition you need to update metallb_ip_range option on the addons.yml at least for suiting your network -environment, because MetalLB allocates external IP addresses from this metallb_ip_range option. diff --git a/roles/kubernetes-apps/metallb/defaults/main.yml b/roles/kubernetes-apps/metallb/defaults/main.yml index 06a509e93..b98106c41 100644 --- a/roles/kubernetes-apps/metallb/defaults/main.yml +++ b/roles/kubernetes-apps/metallb/defaults/main.yml @@ -6,6 +6,7 @@ metallb_port: "7472" metallb_limits_cpu: "100m" metallb_limits_mem: "100Mi" metallb_peers: [] +metallb_speaker_enabled: true metallb_speaker_nodeselector: {} metallb_controller_nodeselector: {} metallb_speaker_tolerations: diff --git a/roles/kubernetes-apps/metallb/templates/metallb.yml.j2 b/roles/kubernetes-apps/metallb/templates/metallb.yml.j2 index 29be2b1a3..5da5d9bfa 100644 --- a/roles/kubernetes-apps/metallb/templates/metallb.yml.j2 +++ b/roles/kubernetes-apps/metallb/templates/metallb.yml.j2 @@ -47,6 +47,7 @@ spec: - secret - emptyDir --- +{% if metallb_speaker_enabled %} apiVersion: policy/v1beta1 kind: PodSecurityPolicy metadata: @@ -85,6 +86,7 @@ spec: - configMap - secret - emptyDir +{% endif %} --- apiVersion: v1 kind: ServiceAccount @@ -94,6 +96,7 @@ metadata: name: controller namespace: metallb-system --- +{% if metallb_speaker_enabled %} apiVersion: v1 kind: ServiceAccount metadata: @@ -101,6 +104,7 @@ metadata: app: metallb name: speaker namespace: metallb-system +{% endif %} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -140,6 +144,7 @@ rules: verbs: - use --- +{% if metallb_speaker_enabled %} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -172,6 +177,7 @@ rules: - podsecuritypolicies verbs: - use +{% endif %} --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -220,6 +226,7 @@ subjects: name: controller namespace: metallb-system --- +{% if metallb_speaker_enabled %} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -234,6 +241,7 @@ subjects: - kind: ServiceAccount name: speaker namespace: metallb-system +{% endif %} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -267,6 +275,7 @@ subjects: - kind: ServiceAccount name: speaker --- +{% if metallb_speaker_enabled %} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -353,6 +362,7 @@ spec: tolerations: {{ metallb_speaker_tolerations | to_nice_yaml(indent=2) | indent(width=8) }} {% endif %} +{% endif %} --- apiVersion: apps/v1 kind: Deployment diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml index 24b28bb7c..b35416221 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico/defaults/main.yml @@ -29,6 +29,9 @@ global_as_num: "64512" # Advertise Service External IPs calico_advertise_service_external_ips: [] +# Adveritse Service LoadBalancer IPs +calico_advertise_service_loadbalancer_ips: [] + # Limits for apps calico_node_memory_limit: 500M calico_node_cpu_limit: 300m diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index 9237cb587..b696d6a95 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -188,6 +188,12 @@ with_items: "{{ calico_advertise_service_external_ips }}" run_once: yes +- name: Populate Service LoadBalancer IPs + set_fact: + _service_loadbalancer_ips: "{{ _service_loadbalancer_ips|default([]) + [ {'cidr': item} ] }}" + with_items: "{{ calico_advertise_service_loadbalancer_ips }}" + run_once: yes + - name: "Determine nodeToNodeMesh needed state" set_fact: nodeToNodeMeshEnabled: "false" @@ -213,6 +219,7 @@ {% if not calico_no_global_as_num|default(false) %}"asNumber": {{ global_as_num }},{% endif %} "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled|default('true') }} , {% if calico_advertise_cluster_ips|default(false) %}"serviceClusterIPs": [{"cidr": "{{ kube_service_addresses }}" }],{% endif %} + {% if calico_version is version('v3.18.0', '>') and calico_advertise_service_loadbalancer_ips|length > 0 %}"serviceLoadBalancerIPs": {{ _service_loadbalancer_ips }},{% endif %} "serviceExternalIPs": {{ _service_external_ips|default([]) }} }} changed_when: false when: