From aa00c1d91ac9f4b6053b10aacd3fd1a6b7f50381 Mon Sep 17 00:00:00 2001
From: Fredrik Liv <fredrik.liv@elastisys.com>
Date: Fri, 10 Sep 2021 22:55:21 +0200
Subject: [PATCH] Updated UpCloud terraform script to use private network and
 dynamic (#7779)

additional disks
---
 contrib/terraform/upcloud/README.md           |  48 +++---
 .../terraform/upcloud/cluster-settings.tfvars |  40 ++++-
 contrib/terraform/upcloud/main.tf             |  16 +-
 .../modules/kubernetes-cluster/main.tf        | 140 +++++++++++++-----
 .../modules/kubernetes-cluster/output.tf      |  10 +-
 .../modules/kubernetes-cluster/variables.tf   |  24 +--
 .../upcloud/sample-inventory/cluster.tfvars   |  40 ++++-
 contrib/terraform/upcloud/variables.tf        |  33 ++++-
 8 files changed, 263 insertions(+), 88 deletions(-)

diff --git a/contrib/terraform/upcloud/README.md b/contrib/terraform/upcloud/README.md
index dcce236fe..6481453fc 100644
--- a/contrib/terraform/upcloud/README.md
+++ b/contrib/terraform/upcloud/README.md
@@ -8,27 +8,29 @@ The setup looks like following
 
 ```text
    Kubernetes cluster
-+-----------------------+
-|   +--------------+    |
-|   | +--------------+  |
-|   | |              |  |
-|   | | Master/etcd  |  |
-|   | | node(s)      |  |
-|   +-+              |  |
-|     +--------------+  |
-|           ^           |
-|           |           |
-|           v           |
-|   +--------------+    |
-|   | +--------------+  |
-|   | |              |  |
-|   | |    Worker    |  |
-|   | |    node(s)   |  |
-|   +-+              |  |
-|     +--------------+  |
-+-----------------------+
++--------------------------+
+|      +--------------+    |
+|      | +--------------+  |
+| -->  | |              |  |
+|      | | Master/etcd  |  |
+|      | | node(s)      |  |
+|      +-+              |  |
+|        +--------------+  |
+|              ^           |
+|              |           |
+|              v           |
+|      +--------------+    |
+|      | +--------------+  |
+| -->  | |              |  |
+|      | |    Worker    |  |
+|      | |    node(s)   |  |
+|      +-+              |  |
+|        +--------------+  |
++--------------------------+
 ```
 
+The nodes uses a private network for node to node communication and a public interface for all external communication.
+
 ## Requirements
 
 * Terraform 0.13.0 or newer
@@ -94,9 +96,10 @@ terraform destroy --var-file cluster-settings.tfvars \
 
 ## Variables
 
-* `hostname`: A valid domain name, e.g. example.com. The maximum length is 128 characters.
+* `prefix`: Prefix to add to all resources, if set to "" don't set any prefix
 * `template_name`: The name or UUID  of a base image
-* `username`: a user to access the nodes
+* `username`: a user to access the nodes, defaults to "ubuntu"
+* `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24"
 * `ssh_public_keys`: List of public SSH keys to install on all machines
 * `zone`: The zone where to run the cluster
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
@@ -104,3 +107,6 @@ terraform destroy --var-file cluster-settings.tfvars \
   * `cpu`: number of cpu cores
   * `mem`: memory size in MB
   * `disk_size`: The size of the storage in GB
+  * `additional_disks`: Additional disks to attach to the node.
+    * `size`: The size of the additional disk in GB
+    * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)
diff --git a/contrib/terraform/upcloud/cluster-settings.tfvars b/contrib/terraform/upcloud/cluster-settings.tfvars
index 08bf5dacf..217f46acc 100644
--- a/contrib/terraform/upcloud/cluster-settings.tfvars
+++ b/contrib/terraform/upcloud/cluster-settings.tfvars
@@ -1,12 +1,11 @@
-
 # See: https://developers.upcloud.com/1.3/5-zones/
 zone     = "fi-hel1"
 username = "ubuntu"
 
-inventory_file = "inventory.ini"
+# Prefix to use for all resources to separate them from other resources
+prefix = "kubespray"
 
-#  A valid domain name, e.g. host.example.com. The maximum length is 128 characters.
-hostname = "example.com"
+inventory_file = "inventory.ini"
 
 #  Set the operating system using UUID or exact name
 template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"
@@ -17,7 +16,7 @@ ssh_public_keys = [
   "ssh-rsa public key 2",
 ]
 
-#check list of available plan https://developers.upcloud.com/1.3/7-plans/
+# check list of available plan https://developers.upcloud.com/1.3/7-plans/
 machines = {
   "master-0" : {
     "node_type" : "master",
@@ -27,6 +26,7 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks" : {}
   },
   "worker-0" : {
     "node_type" : "worker",
@@ -36,6 +36,16 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks" : {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   },
   "worker-1" : {
     "node_type" : "worker",
@@ -45,6 +55,16 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks" : {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   },
   "worker-2" : {
     "node_type" : "worker",
@@ -54,5 +74,15 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks" : {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   }
 }
diff --git a/contrib/terraform/upcloud/main.tf b/contrib/terraform/upcloud/main.tf
index 8ddd46542..b38171cc1 100644
--- a/contrib/terraform/upcloud/main.tf
+++ b/contrib/terraform/upcloud/main.tf
@@ -11,12 +11,14 @@ provider "upcloud" {
 module "kubernetes" {
   source = "./modules/kubernetes-cluster"
 
-  zone     = var.zone
-  hostname = var.hostname
+  prefix = var.prefix
+  zone   = var.zone
 
   template_name = var.template_name
   username      = var.username
 
+  private_network_cidr = var.private_network_cidr
+
   machines = var.machines
 
   ssh_public_keys = var.ssh_public_keys
@@ -30,13 +32,15 @@ data "template_file" "inventory" {
   template = file("${path.module}/templates/inventory.tpl")
 
   vars = {
-    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s etcd_member_name=etcd%d",
+    connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
       keys(module.kubernetes.master_ip),
-      values(module.kubernetes.master_ip),
+      values(module.kubernetes.master_ip).*.public_ip,
+      values(module.kubernetes.master_ip).*.private_ip,
     range(1, length(module.kubernetes.master_ip) + 1)))
-    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s",
+    connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
       keys(module.kubernetes.worker_ip),
-    values(module.kubernetes.worker_ip)))
+      values(module.kubernetes.worker_ip).*.public_ip,
+    values(module.kubernetes.worker_ip).*.private_ip))
     list_master = join("\n", formatlist("%s",
     keys(module.kubernetes.master_ip)))
     list_worker = join("\n", formatlist("%s",
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
index 6a79af720..c72788c69 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
@@ -1,3 +1,41 @@
+locals {
+  # Create a list of all disks to create
+  disks = flatten([
+    for node_name, machine in var.machines : [
+      for disk_name, disk in machine.additional_disks : {
+        disk = disk
+        disk_name = disk_name
+        node_name = node_name
+      }
+    ]
+  ])
+
+  # If prefix is set, all resources will be prefixed with "${var.prefix}-"
+  # Else don't prefix with anything
+  resource-prefix = "%{ if var.prefix != ""}${var.prefix}-%{ endif }"
+}
+
+resource "upcloud_network" "private" {
+  name = "${local.resource-prefix}k8s-network"
+  zone = var.zone
+
+  ip_network {
+    address = var.private_network_cidr
+    dhcp    = true
+    family  = "IPv4"
+  }
+}
+
+resource "upcloud_storage" "additional_disks" {
+  for_each = {
+    for disk in local.disks: "${disk.node_name}_${disk.disk_name}" => disk.disk
+  }
+
+  size  = each.value.size
+  tier  = each.value.tier
+  title = "${local.resource-prefix}${each.key}"
+  zone  = var.zone
+}
 
 resource "upcloud_server" "master" {
   for_each = {
@@ -6,35 +44,48 @@ resource "upcloud_server" "master" {
     if machine.node_type == "master"
   }
 
-  hostname    = "${each.key}.${var.hostname}"
-  cpu            = each.value.cpu
-  mem       = each.value.mem
-  zone            = var.zone
+  hostname = "${local.resource-prefix}${each.key}"
+  cpu      = each.value.cpu
+  mem      = each.value.mem
+  zone     = var.zone
 
   template {
   storage = var.template_name
-  size = each.value.disk_size
+  size    = each.value.disk_size
   }
 
-  # Network interfaces
- network_interface {
-   type = "public"
- }
+  # Public network interface
+  network_interface {
+    type = "public"
+  }
 
- network_interface {
-   type = "utility"
- }
- # Include at least one public SSH key
- login {
-   user = var.username
-   keys = var.ssh_public_keys
-   create_password = false
+  # Private network interface
+  network_interface {
+    type    = "private"
+    network = upcloud_network.private.id
+  }
 
- }
+  dynamic "storage_devices" {
+    for_each = {
+      for disk_key_name, disk in upcloud_storage.additional_disks :
+        disk_key_name => disk
+        # Only add the disk if it matches the node name in the start of its name
+        if length(regexall("^${each.key}_.+", disk_key_name)) > 0
+    }
 
+    content {
+      storage = storage_devices.value.id
+    }
+  }
+
+  # Include at least one public SSH key
+  login {
+    user            = var.username
+    keys            = var.ssh_public_keys
+    create_password = false
+  }
 }
 
-
 resource "upcloud_server" "worker" {
   for_each = {
     for name, machine in var.machines :
@@ -42,25 +93,44 @@ resource "upcloud_server" "worker" {
     if machine.node_type == "worker"
   }
 
-  hostname    = "${each.key}.${var.hostname}"
-  cpu            = each.value.cpu
-  mem       = each.value.mem
-  zone            = var.zone
+  hostname = "${local.resource-prefix}${each.key}"
+  cpu      = each.value.cpu
+  mem      = each.value.mem
+  zone     = var.zone
 
   template {
-  storage = var.template_name
-  size = each.value.disk_size
+    storage = var.template_name
+    size    = each.value.disk_size
   }
 
-  # Network interfaces
- network_interface {
-   type = "public"
- }
+  # Public network interface
+  network_interface {
+    type = "public"
+  }
 
- # Include at least one public SSH key
- login {
-   user = var.username
-   keys = var.ssh_public_keys
-   create_password = false
- }
+  # Private network interface
+  network_interface {
+    type    = "private"
+    network = upcloud_network.private.id
+  }
+
+  dynamic "storage_devices" {
+    for_each = {
+      for disk_key_name, disk in upcloud_storage.additional_disks :
+        disk_key_name => disk
+        # Only add the disk if it matches the node name in the start of its name
+        if length(regexall("^${each.key}_.+", disk_key_name)) > 0
+    }
+
+    content {
+      storage = storage_devices.value.id
+    }
+  }
+
+  # Include at least one public SSH key
+  login {
+    user            = var.username
+    keys            = var.ssh_public_keys
+    create_password = false
+  }
 }
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
index 2661ac013..7343a80bb 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
@@ -2,13 +2,19 @@
 output "master_ip" {
   value = {
     for instance in upcloud_server.master :
-    instance.hostname => instance.network_interface[0].ip_address
+    instance.hostname => {
+      "public_ip": instance.network_interface[0].ip_address
+      "private_ip": instance.network_interface[1].ip_address
+    }
   }
 }
 
 output "worker_ip" {
   value = {
     for instance in upcloud_server.worker :
-    instance.hostname => instance.network_interface[0].ip_address
+    instance.hostname => {
+      "public_ip": instance.network_interface[0].ip_address
+      "private_ip": instance.network_interface[1].ip_address
+    }
   }
 }
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
index 5b130ad10..55abc509b 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
@@ -1,22 +1,28 @@
+variable "prefix" {
+  type = string
+}
+
 variable "zone" {
   type = string
 }
 
-variable "hostname"{
- default ="example.com"
-}
+variable "template_name" {}
 
-variable "template_name"{}
+variable "username" {}
 
-variable "username"{}
+variable "private_network_cidr" {}
 
 variable "machines" {
   description = "Cluster machines"
   type = map(object({
-    node_type = string
-    cpu      = string
-    mem      = string
-    disk_size =  number
+    node_type       = string
+    cpu             = string
+    mem             = string
+    disk_size       =  number
+    additional_disks = map(object({
+      size = number
+      tier = string
+    }))
   }))
 }
 
diff --git a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
index 0a324045e..1400ed3e4 100644
--- a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
+++ b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
@@ -2,20 +2,21 @@
 zone     = "fi-hel1"
 username = "ubuntu"
 
-inventory_file = "inventory.ini"
+# Prefix to use for all resources to separate them from other resources
+prefix = "kubespray"
 
-#  A valid domain name, e.g. host.example.com. The maximum length is 128 characters.
-hostname = "example.com"
+inventory_file = "inventory.ini"
 
 #  Set the operating system using UUID or exact name
 template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"
+
 ssh_public_keys = [
   # Put your public SSH key here
   "ssh-rsa I-did-not-read-the-docs",
   "ssh-rsa I-did-not-read-the-docs 2",
 ]
 
-check list of available plan https://developers.upcloud.com/1.3/7-plans/
+# check list of available plan https://developers.upcloud.com/1.3/7-plans/
 machines = {
   "master-0" : {
     "node_type" : "master",
@@ -25,6 +26,7 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks": {}
   },
   "worker-0" : {
     "node_type" : "worker",
@@ -34,6 +36,16 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks": {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   },
   "worker-1" : {
     "node_type" : "worker",
@@ -43,6 +55,16 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks": {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   },
   "worker-2" : {
     "node_type" : "worker",
@@ -52,5 +74,15 @@ machines = {
     "mem" : "4096"
     # The size of the storage in GB
     "disk_size" : 250
+    "additional_disks": {
+      # "some-disk-name-1": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # },
+      # "some-disk-name-2": {
+      #   "size": 100,
+      #   "tier": "maxiops",
+      # }
+    }
   }
 }
diff --git a/contrib/terraform/upcloud/variables.tf b/contrib/terraform/upcloud/variables.tf
index 60941d572..95c90f2f1 100644
--- a/contrib/terraform/upcloud/variables.tf
+++ b/contrib/terraform/upcloud/variables.tf
@@ -1,23 +1,40 @@
+variable "prefix" {
+  type    = string
+  default = "kubespray"
+
+  description = "Prefix that is used to distinguish these resources from others"
+}
 
 variable "zone" {
   description = "The zone where to run the cluster"
 }
 
-variable "hostname" {
-  default = "example.com"
+variable "template_name" {
+  description = "Block describing the preconfigured operating system"
 }
 
-variable "template_name" {}
+variable "username" {
+  description = "The username to use for the nodes"
+  default     = "ubuntu"
+}
 
-variable "username" {}
+variable "private_network_cidr" {
+  description = "CIDR to use for the private network"
+  default     = "172.16.0.0/24"
+}
 
 variable "machines" {
   description = "Cluster machines"
+
   type = map(object({
     node_type = string
     cpu       = string
     mem       = string
     disk_size = number
+    additional_disks = map(object({
+      size = number
+      tier = string
+    }))
   }))
 }
 
@@ -30,6 +47,10 @@ variable "inventory_file" {
   description = "Where to store the generated inventory file"
 }
 
-variable "UPCLOUD_USERNAME" {}
+variable "UPCLOUD_USERNAME" {
+  description = "UpCloud username with API access"
+}
 
-variable "UPCLOUD_PASSWORD" {}
+variable "UPCLOUD_PASSWORD" {
+  description = "Password for UpCloud API user"
+}