From 9dced7133c336f15156147b4d3dfc274315c0cbe Mon Sep 17 00:00:00 2001 From: Anthony Bible Date: Mon, 11 Apr 2022 11:26:06 -0600 Subject: [PATCH] Fixes for Hetzner terraform and Hetzner Cloud (#8702) * - add ability to specify the network_zone in hetzner terraform - Export the network id from hetzner terraform the the generated inventory.ini * - Add with_networks variable to allow different deployments of hcloud controller manager - Add network id to hcloud controller secret (added via the inventory) - Don't include extra_args if it's not set --- contrib/terraform/hetzner/README.md | 1 + contrib/terraform/hetzner/default.tfvars | 2 +- contrib/terraform/hetzner/main.tf | 3 ++- .../terraform/hetzner/modules/kubernetes-cluster/main.tf | 2 +- .../hetzner/modules/kubernetes-cluster/output.tf | 4 ++++ .../hetzner/modules/kubernetes-cluster/variables.tf | 3 +++ contrib/terraform/hetzner/templates/inventory.tpl | 3 +++ contrib/terraform/hetzner/variables.tf | 4 ++++ inventory/sample/group_vars/all/hcloud.yml | 2 +- .../external_cloud_controller/hcloud/tasks/main.yml | 4 ++-- ...oud-cloud-controller-manager-ds-with-networks.yml.j2} | 9 ++++++--- ...> external-hcloud-cloud-controller-manager-ds.yml.j2} | 6 ++++-- .../hcloud/templates/external-hcloud-cloud-secret.yml.j2 | 5 ++++- 13 files changed, 36 insertions(+), 12 deletions(-) rename roles/kubernetes-apps/external_cloud_controller/hcloud/templates/{external-hcloud-cloud-controller-manager-ds-with-networks.yaml.j2 => external-hcloud-cloud-controller-manager-ds-with-networks.yml.j2} (91%) rename roles/kubernetes-apps/external_cloud_controller/hcloud/templates/{external-hcloud-cloud-controller-manager-ds.yaml.j2 => external-hcloud-cloud-controller-manager-ds.yml.j2} (94%) diff --git a/contrib/terraform/hetzner/README.md b/contrib/terraform/hetzner/README.md index 747928b33..fdc43f9ff 100644 --- a/contrib/terraform/hetzner/README.md +++ b/contrib/terraform/hetzner/README.md @@ -97,6 +97,7 @@ terraform destroy --var-file default.tfvars ../../contrib/terraform/hetzner * `prefix`: Prefix to add to all resources, if set to "" don't set any prefix * `ssh_public_keys`: List of public SSH keys to install on all machines * `zone`: The zone where to run the cluster +* `network_zone`: the network zone where the cluster is running * `machines`: Machines to provision. Key of this object will be used as the name of the machine * `node_type`: The role of this node *(master|worker)* * `size`: Size of the VM diff --git a/contrib/terraform/hetzner/default.tfvars b/contrib/terraform/hetzner/default.tfvars index cb02b142c..957b2d523 100644 --- a/contrib/terraform/hetzner/default.tfvars +++ b/contrib/terraform/hetzner/default.tfvars @@ -1,6 +1,6 @@ prefix = "default" zone = "hel1" - +network_zone = "eu-central" inventory_file = "inventory.ini" ssh_public_keys = [ diff --git a/contrib/terraform/hetzner/main.tf b/contrib/terraform/hetzner/main.tf index 130e89583..805c7bfb8 100644 --- a/contrib/terraform/hetzner/main.tf +++ b/contrib/terraform/hetzner/main.tf @@ -10,6 +10,7 @@ module "kubernetes" { machines = var.machines ssh_public_keys = var.ssh_public_keys + network_zone = var.network_zone ssh_whitelist = var.ssh_whitelist api_server_whitelist = var.api_server_whitelist @@ -34,9 +35,9 @@ data "template_file" "inventory" { keys(module.kubernetes.worker_ip_addresses), values(module.kubernetes.worker_ip_addresses).*.public_ip, values(module.kubernetes.worker_ip_addresses).*.private_ip)) - list_master = join("\n", keys(module.kubernetes.master_ip_addresses)) list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses)) + network_id = module.kubernetes.network_id } } diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf index e8db4e212..d7ec865d7 100644 --- a/contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf @@ -6,7 +6,7 @@ resource "hcloud_network" "kubernetes" { resource "hcloud_network_subnet" "kubernetes" { type = "cloud" network_id = hcloud_network.kubernetes.id - network_zone = "eu-central" + network_zone = var.network_zone ip_range = var.private_subnet_cidr } diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf index 093647f07..c6bb276da 100644 --- a/contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf @@ -21,3 +21,7 @@ output "worker_ip_addresses" { output "cluster_private_network_cidr" { value = var.private_subnet_cidr } + +output "network_id" { + value = hcloud_network.kubernetes.id +} \ No newline at end of file diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf index 2789ae17b..7486e0806 100644 --- a/contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf +++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf @@ -39,3 +39,6 @@ variable "private_network_cidr" { variable "private_subnet_cidr" { default = "10.0.10.0/24" } +variable "network_zone" { + default = "eu-central" +} diff --git a/contrib/terraform/hetzner/templates/inventory.tpl b/contrib/terraform/hetzner/templates/inventory.tpl index 9c562f4df..ba71f9960 100644 --- a/contrib/terraform/hetzner/templates/inventory.tpl +++ b/contrib/terraform/hetzner/templates/inventory.tpl @@ -14,3 +14,6 @@ ${list_worker} [k8s-cluster:children] kube-master kube-node + +[k8s-cluster:vars] +network_id=${network_id} diff --git a/contrib/terraform/hetzner/variables.tf b/contrib/terraform/hetzner/variables.tf index 978575078..e83676ad8 100644 --- a/contrib/terraform/hetzner/variables.tf +++ b/contrib/terraform/hetzner/variables.tf @@ -1,6 +1,10 @@ variable "zone" { description = "The zone where to run the cluster" } +variable "network_zone" { + description = "The network zone where the cluster is running" + default = "eu-central" +} variable "prefix" { description = "Prefix for resource names" diff --git a/inventory/sample/group_vars/all/hcloud.yml b/inventory/sample/group_vars/all/hcloud.yml index ff90dcc86..c27035c08 100644 --- a/inventory/sample/group_vars/all/hcloud.yml +++ b/inventory/sample/group_vars/all/hcloud.yml @@ -2,7 +2,7 @@ # external_hcloud_cloud: # hcloud_api_token: "" # token_secret_name: hcloud -# +# with_networks: false # Use the hcloud controller-manager with networks support https://github.com/hetznercloud/hcloud-cloud-controller-manager#networks-support # service_account_name: cloud-controller-manager # # controller_image_tag: "latest" diff --git a/roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml b/roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml index adaff2219..e09f99d1f 100644 --- a/roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml +++ b/roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml @@ -9,8 +9,8 @@ - {name: external-hcloud-cloud-secret, file: external-hcloud-cloud-secret.yml} - {name: external-hcloud-cloud-service-account, file: external-hcloud-cloud-service-account.yml} - {name: external-hcloud-cloud-role-bindings, file: external-hcloud-cloud-role-bindings.yml} - - {name: external-hcloud-cloud-controller-manager-ds, file: external-hcloud-cloud-controller-manager-ds.yml} - - {name: external-hcloud-cloud-controller-manager-ds-with-networks, file: external-hcloud-cloud-controller-manager-ds-with-networks.yml} + - {name: "{{ 'external-hcloud-cloud-controller-manager-ds-with-networks' if external_hcloud_cloud.with_networks else 'external-hcloud-cloud-controller-manager-ds' }}", file: "{{ 'external-hcloud-cloud-controller-manager-ds-with-networks.yml' if external_hcloud_cloud.with_networks else 'external-hcloud-cloud-controller-manager-ds.yml' }}"} + register: external_hcloud_manifests when: inventory_hostname == groups['kube_control_plane'][0] tags: external-hcloud diff --git a/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yaml.j2 b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yml.j2 similarity index 91% rename from roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yaml.j2 rename to roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yml.j2 index 3bbe10753..cd796e9b7 100644 --- a/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yaml.j2 +++ b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yml.j2 @@ -1,6 +1,6 @@ --- apiVersion: apps/v1 -kind: DeamonSet +kind: DaemonSet metadata: name: hcloud-cloud-controller-manager namespace: kube-system @@ -44,10 +44,13 @@ spec: - "--allow-untagged-cloud" - "--allocate-node-cidrs=true" - "--cluster-cidr=10.244.0.0/16" +{% if external_hcloud_cloud.controller_extra_args is defined %} + args: {% for key, value in external_hcloud_cloud.controller_extra_args.items() %} - "{{ '--' + key + '=' + value }}" {% endfor %} +{% endif %} resources: requests: cpu: 100m @@ -60,10 +63,10 @@ spec: - name: HCLOUD_TOKEN valueFrom: secretKeyRef: - name: hcloud + name: {{ external_hcloud_cloud.token_secret_name }} key: token - name: HCLOUD_NETWORK valueFrom: secretKeyRef: name: {{ external_hcloud_cloud.token_secret_name }} - key: {{ external_hcloud_cloud.token_secret_key }} + key: network diff --git a/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yaml.j2 b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yml.j2 similarity index 94% rename from roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yaml.j2 rename to roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yml.j2 index fecee8d0a..95473cd59 100644 --- a/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yaml.j2 +++ b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yml.j2 @@ -1,6 +1,6 @@ --- apiVersion: apps/v1 -kind: DeamonSet +kind: DaemonSet metadata: name: hcloud-cloud-controller-manager namespace: kube-system @@ -41,10 +41,12 @@ spec: - "--cloud-provider=hcloud" - "--leader-elect=false" - "--allow-untagged-cloud" +{% if external_hcloud_cloud.controller_extra_args is defined %} args: {% for key, value in external_hcloud_cloud.controller_extra_args.items() %} - "{{ '--' + key + '=' + value }}" {% endfor %} +{% endif %} resources: requests: cpu: 100m @@ -58,4 +60,4 @@ spec: valueFrom: secretKeyRef: name: {{ external_hcloud_cloud.token_secret_name }} - key: {{ external_hcloud_cloud.token_secret_key }} + key: token diff --git a/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2 index 614d27897..c2ea894a9 100644 --- a/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2 +++ b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2 @@ -5,4 +5,7 @@ metadata: name: "{{ external_hcloud_cloud.token_secret_name }}" namespace: kube-system data: - token: "{{ external_hcloud_cloud.hcloud_api_token | base64 }}" + token: "{{ external_hcloud_cloud.hcloud_api_token | b64encode }}" +{% if external_hcloud_cloud.with_networks %} + network: "{{ network_id|b64encode }}" +{% endif %}