[metrics_server]: Enabled HA mode by adding 'metrics_server_replicas'… (#9539)

* [metrics_server]: Enabled HA mode by adding 'metrics_server_replicas' variable and adding podAntiAffinity rule

Signed-off-by: Ugur Can Ozturk <57688057+ugur99@users.noreply.github.com>

* [metrics_server]: added namespaces selector

Signed-off-by: Ugur Can Ozturk <57688057+ugur99@users.noreply.github.com>

Signed-off-by: Ugur Can Ozturk <57688057+ugur99@users.noreply.github.com>

inventory/sample/group_vars/k8s_cluster/addons.yml

@@ -19,6 +19,7 @@ metrics_server_enabled: false
 # metrics_server_metric_resolution: 15s
 # metrics_server_kubelet_preferred_address_types: "InternalIP,ExternalIP,Hostname"
 # metrics_server_host_network: false
+# metrics_server_replicas: 1
 
 # Rancher Local Path Provisioner
 local_path_provisioner_enabled: false

roles/kubernetes-apps/metrics_server/defaults/main.yml

@@ -8,3 +8,4 @@ metrics_server_limits_memory: 200Mi
 metrics_server_requests_cpu: 100m
 metrics_server_requests_memory: 200Mi
 metrics_server_host_network: false
+metrics_server_replicas: 1

roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2

@@ -9,6 +9,7 @@ metadata:
     addonmanager.kubernetes.io/mode: Reconcile
     version: {{ metrics_server_version }}
 spec:
+  replicas: {{ metrics_server_replicas }}
   selector:
     matchLabels:
       app.kubernetes.io/name: metrics-server
@@ -91,12 +92,16 @@ spec:
           effect: NoSchedule
 {% endif %}
       affinity:
-        nodeAffinity:
+        podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - weight: 100
-            preference:
+            podAffinityTerm:
+              labelSelector:
                 matchExpressions:
-              - key: node-role.kubernetes.io/control-plane
+                - key: app.kubernetes.io/name
                   operator: In
                   values:
-                - ""
+                  - metrics-server
+              topologyKey: kubernetes.io/hostname
+              namespaces:
+              - kube-system