From b9e5b0cb533684f82e8aee85b639f8edaac6a9de Mon Sep 17 00:00:00 2001 From: Ajarmar <37733838+Ajarmar@users.noreply.github.com> Date: Wed, 11 May 2022 19:15:03 +0200 Subject: [PATCH] UpCloud server plan, firewall, load balancer integration (#8758) * [upcloud] add option to use preconfigured cpu/mem plan * [upcloud] add option to use firewall rules for API server/SSH access * [upcloud] add option to use managed load balancer --- contrib/terraform/upcloud/README.md | 13 ++ .../terraform/upcloud/cluster-settings.tfvars | 37 ++++ contrib/terraform/upcloud/main.tf | 8 + .../modules/kubernetes-cluster/main.tf | 172 +++++++++++++++++- .../modules/kubernetes-cluster/output.tf | 4 + .../modules/kubernetes-cluster/variables.tf | 36 ++++ .../modules/kubernetes-cluster/versions.tf | 2 +- contrib/terraform/upcloud/output.tf | 4 + .../upcloud/sample-inventory/cluster.tfvars | 37 ++++ contrib/terraform/upcloud/variables.tf | 44 +++++ contrib/terraform/upcloud/versions.tf | 2 +- 11 files changed, 353 insertions(+), 6 deletions(-) diff --git a/contrib/terraform/upcloud/README.md b/contrib/terraform/upcloud/README.md index 6481453fc..0962f8354 100644 --- a/contrib/terraform/upcloud/README.md +++ b/contrib/terraform/upcloud/README.md @@ -104,9 +104,22 @@ terraform destroy --var-file cluster-settings.tfvars \ * `zone`: The zone where to run the cluster * `machines`: Machines to provision. Key of this object will be used as the name of the machine * `node_type`: The role of this node *(master|worker)* + * `plan`: Preconfigured cpu/mem plan to use (disables `cpu` and `mem` attributes below) * `cpu`: number of cpu cores * `mem`: memory size in MB * `disk_size`: The size of the storage in GB * `additional_disks`: Additional disks to attach to the node. * `size`: The size of the additional disk in GB * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm) +* `firewall_enabled`: Enable firewall rules +* `master_allowed_remote_ips`: List of IP ranges that should be allowed to access API of masters + * `start_address`: Start of address range to allow + * `end_address`: End of address range to allow +* `k8s_allowed_remote_ips`: List of IP ranges that should be allowed SSH access to all nodes + * `start_address`: Start of address range to allow + * `end_address`: End of address range to allow +* `loadbalancer_enabled`: Enable managed load balancer +* `loadbalancer_plan`: Plan to use for load balancer *(development|production-small)* +* `loadbalancers`: Ports to load balance and which machines to forward to. Key of this object will be used as the name of the load balancer frontends/backends + * `port`: Port to load balance. + * `backend_servers`: List of servers that traffic to the port should be forwarded to. diff --git a/contrib/terraform/upcloud/cluster-settings.tfvars b/contrib/terraform/upcloud/cluster-settings.tfvars index 217f46acc..b7bdb2302 100644 --- a/contrib/terraform/upcloud/cluster-settings.tfvars +++ b/contrib/terraform/upcloud/cluster-settings.tfvars @@ -20,6 +20,8 @@ ssh_public_keys = [ machines = { "master-0" : { "node_type" : "master", + # plan to use instead of custom cpu/mem + "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB @@ -30,6 +32,8 @@ machines = { }, "worker-0" : { "node_type" : "worker", + # plan to use instead of custom cpu/mem + "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB @@ -49,6 +53,8 @@ machines = { }, "worker-1" : { "node_type" : "worker", + # plan to use instead of custom cpu/mem + "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB @@ -68,6 +74,8 @@ machines = { }, "worker-2" : { "node_type" : "worker", + # plan to use instead of custom cpu/mem + "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB @@ -86,3 +94,32 @@ machines = { } } } + +firewall_enabled = false + +master_allowed_remote_ips = [ + { + "start_address" : "0.0.0.0" + "end_address" : "255.255.255.255" + } +] + +k8s_allowed_remote_ips = [ + { + "start_address" : "0.0.0.0" + "end_address" : "255.255.255.255" + } +] + +loadbalancer_enabled = false +loadbalancer_plan = "development" +loadbalancers = { + # "http" : { + # "port" : 80, + # "backend_servers" : [ + # "worker-0", + # "worker-1", + # "worker-2" + # ] + # } +} diff --git a/contrib/terraform/upcloud/main.tf b/contrib/terraform/upcloud/main.tf index b38171cc1..1acc260fa 100644 --- a/contrib/terraform/upcloud/main.tf +++ b/contrib/terraform/upcloud/main.tf @@ -22,6 +22,14 @@ module "kubernetes" { machines = var.machines ssh_public_keys = var.ssh_public_keys + + firewall_enabled = var.firewall_enabled + master_allowed_remote_ips = var.master_allowed_remote_ips + k8s_allowed_remote_ips = var.k8s_allowed_remote_ips + + loadbalancer_enabled = var.loadbalancer_enabled + loadbalancer_plan = var.loadbalancer_plan + loadbalancers = var.loadbalancers } # diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf index bce7c924a..ed9de9dd6 100644 --- a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf +++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf @@ -10,6 +10,16 @@ locals { ] ]) + lb_backend_servers = flatten([ + for lb_name, loadbalancer in var.loadbalancers : [ + for backend_server in loadbalancer.backend_servers : { + port = loadbalancer.port + lb_name = lb_name + server_name = backend_server + } + ] + ]) + # If prefix is set, all resources will be prefixed with "${var.prefix}-" # Else don't prefix with anything resource-prefix = "%{ if var.prefix != ""}${var.prefix}-%{ endif }" @@ -45,8 +55,9 @@ resource "upcloud_server" "master" { } hostname = "${local.resource-prefix}${each.key}" - cpu = each.value.cpu - mem = each.value.mem + plan = each.value.plan + cpu = each.value.plan == null ? each.value.cpu : null + mem = each.value.plan == null ? each.value.mem : null zone = var.zone template { @@ -69,6 +80,8 @@ resource "upcloud_server" "master" { lifecycle { ignore_changes = [storage_devices] } + + firewall = var.firewall_enabled dynamic "storage_devices" { for_each = { @@ -99,8 +112,9 @@ resource "upcloud_server" "worker" { } hostname = "${local.resource-prefix}${each.key}" - cpu = each.value.cpu - mem = each.value.mem + plan = each.value.plan + cpu = each.value.plan == null ? each.value.cpu : null + mem = each.value.plan == null ? each.value.mem : null zone = var.zone template { @@ -124,6 +138,8 @@ resource "upcloud_server" "worker" { ignore_changes = [storage_devices] } + firewall = var.firewall_enabled + dynamic "storage_devices" { for_each = { for disk_key_name, disk in upcloud_storage.additional_disks : @@ -144,3 +160,151 @@ resource "upcloud_server" "worker" { create_password = false } } + +resource "upcloud_firewall_rules" "master" { + for_each = upcloud_server.master + server_id = each.value.id + + dynamic firewall_rule { + for_each = var.master_allowed_remote_ips + + content { + action = "accept" + comment = "Allow master API access from this network" + destination_port_end = "6443" + destination_port_start = "6443" + direction = "in" + family = "IPv4" + protocol = "tcp" + source_address_end = firewall_rule.value.end_address + source_address_start = firewall_rule.value.start_address + } + } + + dynamic firewall_rule { + for_each = length(var.master_allowed_remote_ips) > 0 ? [1] : [] + + content { + action = "drop" + comment = "Deny master API access from other networks" + destination_port_end = "6443" + destination_port_start = "6443" + direction = "in" + family = "IPv4" + protocol = "tcp" + source_address_end = "255.255.255.255" + source_address_start = "0.0.0.0" + } + } + + dynamic firewall_rule { + for_each = var.k8s_allowed_remote_ips + + content { + action = "accept" + comment = "Allow SSH from this network" + destination_port_end = "22" + destination_port_start = "22" + direction = "in" + family = "IPv4" + protocol = "tcp" + source_address_end = firewall_rule.value.end_address + source_address_start = firewall_rule.value.start_address + } + } + + dynamic firewall_rule { + for_each = length(var.k8s_allowed_remote_ips) > 0 ? [1] : [] + + content { + action = "drop" + comment = "Deny SSH from other networks" + destination_port_end = "22" + destination_port_start = "22" + direction = "in" + family = "IPv4" + protocol = "tcp" + source_address_end = "255.255.255.255" + source_address_start = "0.0.0.0" + } + } +} + +resource "upcloud_firewall_rules" "k8s" { + for_each = upcloud_server.worker + server_id = each.value.id + + dynamic firewall_rule { + for_each = var.k8s_allowed_remote_ips + + content { + action = "accept" + comment = "Allow SSH from this network" + destination_port_end = "22" + destination_port_start = "22" + direction = "in" + family = "IPv4" + protocol = "tcp" + source_address_end = firewall_rule.value.end_address + source_address_start = firewall_rule.value.start_address + } + } + + dynamic firewall_rule { + for_each = length(var.k8s_allowed_remote_ips) > 0 ? [1] : [] + + content { + action = "drop" + comment = "Deny SSH from other networks" + destination_port_end = "22" + destination_port_start = "22" + direction = "in" + family = "IPv4" + protocol = "tcp" + source_address_end = "255.255.255.255" + source_address_start = "0.0.0.0" + } + } +} + +resource "upcloud_loadbalancer" "lb" { + count = var.loadbalancer_enabled ? 1 : 0 + configured_status = "started" + name = "${local.resource-prefix}lb" + plan = var.loadbalancer_plan + zone = var.zone + network = upcloud_network.private.id +} + +resource "upcloud_loadbalancer_backend" "lb_backend" { + for_each = var.loadbalancer_enabled ? var.loadbalancers : {} + + loadbalancer = upcloud_loadbalancer.lb[0].id + name = "lb-backend-${each.key}" +} + +resource "upcloud_loadbalancer_frontend" "lb_frontend" { + for_each = var.loadbalancer_enabled ? var.loadbalancers : {} + + loadbalancer = upcloud_loadbalancer.lb[0].id + name = "lb-frontend-${each.key}" + mode = "tcp" + port = each.value.port + default_backend_name = upcloud_loadbalancer_backend.lb_backend[each.key].name +} + +resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" { + for_each = { + for be_server in local.lb_backend_servers: + "${be_server.server_name}-lb-backend-${be_server.lb_name}" => be_server + if var.loadbalancer_enabled + } + + backend = upcloud_loadbalancer_backend.lb_backend[each.value.lb_name].id + name = "${local.resource-prefix}${each.key}" + ip = merge(upcloud_server.master, upcloud_server.worker)[each.value.server_name].network_interface[1].ip_address + port = each.value.port + weight = 100 + max_sessions = var.loadbalancer_plan == "production-small" ? 50000 : 1000 + enabled = true +} diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf index 7343a80bb..c1f8c7c9c 100644 --- a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf +++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf @@ -18,3 +18,7 @@ output "worker_ip" { } } } + +output "loadbalancer_domain" { + value = var.loadbalancer_enabled ? upcloud_loadbalancer.lb[0].dns_name : null +} diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf index 55abc509b..1fc411b27 100644 --- a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf +++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf @@ -16,6 +16,7 @@ variable "machines" { description = "Cluster machines" type = map(object({ node_type = string + plan = string cpu = string mem = string disk_size = number @@ -29,3 +30,38 @@ variable "machines" { variable "ssh_public_keys" { type = list(string) } + +variable "firewall_enabled" { + type = bool +} + +variable "master_allowed_remote_ips" { + type = list(object({ + start_address = string + end_address = string + })) +} + +variable "k8s_allowed_remote_ips" { + type = list(object({ + start_address = string + end_address = string + })) +} + +variable "loadbalancer_enabled" { + type = bool +} + +variable "loadbalancer_plan" { + type = string +} + +variable "loadbalancers" { + description = "Load balancers" + + type = map(object({ + port = number + backend_servers = list(string) + })) +} diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf index ffe5d32b1..8d87504df 100644 --- a/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf +++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { upcloud = { source = "UpCloudLtd/upcloud" - version = "~>2.0.0" + version = "~>2.4.0" } } required_version = ">= 0.13" diff --git a/contrib/terraform/upcloud/output.tf b/contrib/terraform/upcloud/output.tf index f269c755e..006e3b1cd 100644 --- a/contrib/terraform/upcloud/output.tf +++ b/contrib/terraform/upcloud/output.tf @@ -6,3 +6,7 @@ output "master_ip" { output "worker_ip" { value = module.kubernetes.worker_ip } + +output "loadbalancer_domain" { + value = module.kubernetes.loadbalancer_domain +} diff --git a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars index 1400ed3e4..787fb702c 100644 --- a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars +++ b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars @@ -20,6 +20,8 @@ ssh_public_keys = [ machines = { "master-0" : { "node_type" : "master", + # plan to use instead of custom cpu/mem + "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB @@ -30,6 +32,8 @@ machines = { }, "worker-0" : { "node_type" : "worker", + # plan to use instead of custom cpu/mem + "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB @@ -49,6 +53,8 @@ machines = { }, "worker-1" : { "node_type" : "worker", + # plan to use instead of custom cpu/mem + "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB @@ -68,6 +74,8 @@ machines = { }, "worker-2" : { "node_type" : "worker", + # plan to use instead of custom cpu/mem + "plan" : null, #number of cpu cores "cpu" : "2", #memory size in MB @@ -86,3 +94,32 @@ machines = { } } } + +firewall_enabled = false + +master_allowed_remote_ips = [ + { + "start_address" : "0.0.0.0" + "end_address" : "255.255.255.255" + } +] + +k8s_allowed_remote_ips = [ + { + "start_address" : "0.0.0.0" + "end_address" : "255.255.255.255" + } +] + +loadbalancer_enabled = false +loadbalancer_plan = "development" +loadbalancers = { + # "http" : { + # "port" : 80, + # "backend_servers" : [ + # "worker-0", + # "worker-1", + # "worker-2" + # ] + # } +} diff --git a/contrib/terraform/upcloud/variables.tf b/contrib/terraform/upcloud/variables.tf index 95c90f2f1..60d07c1bf 100644 --- a/contrib/terraform/upcloud/variables.tf +++ b/contrib/terraform/upcloud/variables.tf @@ -28,6 +28,7 @@ variable "machines" { type = map(object({ node_type = string + plan = string cpu = string mem = string disk_size = number @@ -54,3 +55,46 @@ variable "UPCLOUD_USERNAME" { variable "UPCLOUD_PASSWORD" { description = "Password for UpCloud API user" } + +variable "firewall_enabled" { + description = "Enable firewall rules" + default = false +} + +variable "master_allowed_remote_ips" { + description = "List of IP start/end addresses allowed to access API of masters" + type = list(object({ + start_address = string + end_address = string + })) + default = [] +} + +variable "k8s_allowed_remote_ips" { + description = "List of IP start/end addresses allowed to SSH to hosts" + type = list(object({ + start_address = string + end_address = string + })) + default = [] +} + +variable "loadbalancer_enabled" { + description = "Enable load balancer" + default = false +} + +variable "loadbalancer_plan" { + description = "Load balancer plan (development/production-small)" + default = "development" +} + +variable "loadbalancers" { + description = "Load balancers" + + type = map(object({ + port = number + backend_servers = list(string) + })) + default = {} +} diff --git a/contrib/terraform/upcloud/versions.tf b/contrib/terraform/upcloud/versions.tf index c91089879..5250626c3 100644 --- a/contrib/terraform/upcloud/versions.tf +++ b/contrib/terraform/upcloud/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { upcloud = { source = "UpCloudLtd/upcloud" - version = "~>2.0.0" + version = "~>2.4.0" } } required_version = ">= 0.13"