From 996ef98b87d162ee8215219815a8428b91f91191 Mon Sep 17 00:00:00 2001 From: Mathieu Parent Date: Thu, 7 Apr 2022 19:37:57 +0200 Subject: [PATCH] Add support for kube-vip (#8669) Signed-off-by: Mathieu Parent --- docs/_sidebar.md | 1 + docs/ansible.md | 1 + docs/ha-mode.md | 4 +- docs/kube-vip.md | 52 +++++++++++ roles/download/defaults/main.yml | 11 +++ roles/kubernetes/node/defaults/main.yml | 20 ++++ .../node/tasks/loadbalancer/kube-vip.yml | 6 ++ roles/kubernetes/node/tasks/main.yml | 7 ++ .../templates/manifests/kube-vip.manifest.j2 | 93 +++++++++++++++++++ roles/kubespray-defaults/defaults/main.yaml | 2 + 10 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 docs/kube-vip.md create mode 100644 roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml create mode 100644 roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2 diff --git a/docs/_sidebar.md b/docs/_sidebar.md index 0a754206b..859cc1306 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -18,6 +18,7 @@ * [Weave](docs/weave.md) * [Multus](docs/multus.md) * Ingress + * [kube-vip](docs/kube-vip.md) * [ALB Ingress](docs/ingress_controller/alb_ingress_controller.md) * [MetalLB](docs/metallb.md) * [Nginx Ingress](docs/ingress_controller/ingress_nginx.md) diff --git a/docs/ansible.md b/docs/ansible.md index e0bdc6d49..959ba877e 100644 --- a/docs/ansible.md +++ b/docs/ansible.md @@ -156,6 +156,7 @@ The following tags are defined in playbooks: | kubeadm | Roles linked to kubeadm tasks | kube-apiserver | Configuring static pod kube-apiserver | kube-controller-manager | Configuring static pod kube-controller-manager +| kube-vip | Installing and configuring kube-vip | kubectl | Installing kubectl and bash completion | kubelet | Configuring kubelet service | kube-ovn | Network plugin kube-ovn diff --git a/docs/ha-mode.md b/docs/ha-mode.md index 07baceba0..ca924db7d 100644 --- a/docs/ha-mode.md +++ b/docs/ha-mode.md @@ -29,9 +29,7 @@ configure kubelet and kube-proxy on non-master nodes to use the local internal loadbalancer. If you choose to NOT use the local internal loadbalancer, you will need to -configure your own loadbalancer to achieve HA. Note that deploying a -loadbalancer is up to a user and is not covered by ansible roles in Kubespray. -By default, it only configures a non-HA endpoint, which points to the +use the [kube-vip](kube-vip.md) ansible role or configure your own loadbalancer to achieve HA. By default, it only configures a non-HA endpoint, which points to the `access_ip` or IP address of the first server node in the `kube_control_plane` group. It can also configure clients to use endpoints for a given loadbalancer type. The following diagram shows how traffic to the apiserver is directed. diff --git a/docs/kube-vip.md b/docs/kube-vip.md new file mode 100644 index 000000000..e81740ccf --- /dev/null +++ b/docs/kube-vip.md @@ -0,0 +1,52 @@ +# kube-vip + +kube-vip provides Kubernetes clusters with a virtual IP and load balancer for both the control plane (for building a highly-available cluster) and Kubernetes Services of type LoadBalancer without relying on any external hardware or software. + +## Install + +You have to explicitly enable the kube-vip extension: + +```yaml +kube_vip_enabled: true +``` + +You also need to enable +[kube-vip as HA, Load Balancer, or both](https://kube-vip.chipzoller.dev/docs/installation/static/#kube-vip-as-ha-load-balancer-or-both): + +```yaml +# HA for control-plane, requires a VIP +kube_vip_controlplane_enabled: true +kube_vip_address: 10.42.42.42 +loadbalancer_apiserver: + address: "{{ kube_vip_address }}" + port: 6443 +# kube_vip_interface: ens160 + +# LoadBalancer for services +kube_vip_services_enabled: false +# kube_vip_services_interface: ens320 +``` + +> Note: When using `kube-vip` as LoadBalancer for services, +[additionnal manual steps](https://kube-vip.chipzoller.dev/docs/usage/cloud-provider/) +are needed. + +If using [ARP mode](https://kube-vip.chipzoller.dev/docs/installation/static/#arp) : + +```yaml +kube_vip_arp_enabled: true +``` + +If using [BGP mode](https://kube-vip.chipzoller.dev/docs/installation/static/#bgp) : + +```yaml +kube_vip_bgp_enabled: true +kube_vip_local_as: 65000 +kube_vip_bgp_routerid: 192.168.0.2 +kube_vip_bgppeers: +- 192.168.0.10:65000::false +- 192.168.0.11:65000::false +# kube_vip_bgp_peeraddress: +# kube_vip_bgp_peerpass: +# kube_vip_bgp_peeras: +``` diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index e9e15075b..1779b26ae 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -875,6 +875,8 @@ kube_router_image_tag: "{{ kube_router_version }}" multus_image_repo: "{{ github_image_repo }}/k8snetworkplumbingwg/multus-cni" multus_image_tag: "{{ multus_version }}" +kube_vip_image_repo: "{{ github_image_repo }}/kube-vip/kube-vip" +kube_vip_image_tag: v0.4.2 nginx_image_repo: "{{ docker_image_repo }}/library/nginx" nginx_image_tag: 1.21.4 haproxy_image_repo: "{{ docker_image_repo }}/library/haproxy" @@ -1382,6 +1384,15 @@ downloads: groups: - k8s_cluster + kube-vip: + enabled: "{{ kube_vip_enabled }}" + container: true + repo: "{{ kube_vip_image_repo }}" + tag: "{{ kube_vip_image_tag }}" + sha256: "{{ kube_vip_digest_checksum|default(None) }}" + groups: + - kube_control_plane + nginx: enabled: "{{ loadbalancer_apiserver_localhost and loadbalancer_apiserver_type == 'nginx' }}" container: true diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 06a9422e2..800c43f41 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -47,6 +47,26 @@ eviction_hard_control_plane: {} kubelet_status_update_frequency: 10s +# kube-vip +kube_vip_version: v0.4.2 + +kube_vip_arp_enabled: false +kube_vip_interface: +kube_vip_services_interface: +kube_vip_cidr: 32 +kube_vip_controlplane_enabled: false +kube_vip_ddns_enabled: false +kube_vip_services_enabled: false +kube_vip_leader_election_enabled: "{{ kube_vip_arp_enabled }}" +kube_vip_bgp_enabled: false +kube_vip_bgp_routerid: +kube_vip_local_as: 65000 +kube_vip_bgp_peeraddress: +kube_vip_bgp_peerpass: +kube_vip_bgp_peeras: +kube_vip_bgppeers: +kube_vip_address: + # Requests for load balancer app loadbalancer_apiserver_memory_requests: 32M loadbalancer_apiserver_cpu_requests: 25m diff --git a/roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml b/roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml new file mode 100644 index 000000000..0f3f5117f --- /dev/null +++ b/roles/kubernetes/node/tasks/loadbalancer/kube-vip.yml @@ -0,0 +1,6 @@ +--- +- name: kube-vip | Write static pod + template: + src: manifests/kube-vip.manifest.j2 + dest: "{{ kube_manifest_dir }}/kube-vip.yml" + mode: 0640 diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 29042b680..59dc3007a 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -17,6 +17,13 @@ tags: - kubelet +- import_tasks: loadbalancer/kube-vip.yml + when: + - is_kube_master + - kube_vip_enabled + tags: + - kube-vip + - import_tasks: loadbalancer/nginx-proxy.yml when: - not is_kube_master or kube_apiserver_bind_address != '0.0.0.0' diff --git a/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2 new file mode 100644 index 000000000..16246156b --- /dev/null +++ b/roles/kubernetes/node/templates/manifests/kube-vip.manifest.j2 @@ -0,0 +1,93 @@ +# Inspired by https://github.com/kube-vip/kube-vip/blob/v0.4.2/pkg/kubevip/config_generator.go#L13 +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system +spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: {{ kube_vip_arp_enabled | string | to_json }} + - name: port + value: "6443" + {% if kube_vip_interface %} + - name: vip_interface + value: "{{ kube_vip_interface | string | to_json }}" + {% endif %} + {% if kube_vip_services_interface %} + - name: vip_servicesinterface + value: {{ kube_vip_services_interface | string | to_json }} + {% endif %} + {% if kube_vip_cidr %} + - name: vip_cidr + value: {{ kube_vip_cidr | string | to_json }} + {% endif %} + {% if kube_vip_controlplane_enabled %} + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: {{ kube_vip_ddns_enabled | string | to_json }} + {% endif %} + {% if kube_vip_services_enabled %} + - name: svc_enable + value: "true" + {% endif %} + {% if kube_vip_leader_election_enabled %} + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + {% endif %} + {% if kube_vip_bgp_enabled %} + - name: bgp_enable + value: "true" + - name: bgp_routerid + value: {{ kube_vip_bgp_routerid | string | to_json }} + - name: bgp_as + value: {{ kube_vip_local_as | string | to_json }} + - name: bgp_peeraddress + value: {{ kube_vip_bgp_peeraddress | to_json }} + - name: bgp_peerpass + value: {{ kube_vip_bgp_peerpass | to_json }} + - name: bgp_peeras + value: {{ kube_vip_bgp_peeras | to_json }} + {% if kube_vip_bgppeers %} + - name: bgp_peers + value: {{ kube_vip_bgp_peeras | join(',') | to_json }} + {% endif %} + {% endif %} + - name: address + value: {{ kube_vip_address | to_json }} + image: {{ kube_vip_image_repo }}:{{ kube_vip_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig +status: {} + diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index a21233c1f..321e5dc31 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -60,6 +60,8 @@ kube_proxy_nodeport_addresses: >- # Set to true to allow pre-checks to fail and continue deployment ignore_assert_errors: false +kube_vip_enabled: false + # nginx-proxy configure nginx_config_dir: "/etc/nginx"