C12s/c12s-kubespray
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Updated UpCloud terraform script to use private network and dynamic (#7779)

Browse source 
additional disks
This commit is contained in:
Fredrik Liv 2021-09-10 22:55:21 +02:00 committed by GitHub
parent a5a88e41af
commit aa00c1d91a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 263 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
contrib/terraform/upcloud/README.md
View file

@ -8,27 +8,29 @@ The setup looks like following
```text ```text
Kubernetes cluster Kubernetes cluster
+-----------------------+ +--------------------------+
| +--------------+ | | +--------------+ |
| | +--------------+ | | | +--------------+ |
| | | | | | --> | | | |
| | | Master/etcd | | | | | Master/etcd | |
| | | node(s) | | | | | node(s) | |
| +-+ | | | +-+ | |
| +--------------+ | | +--------------+ |
| ^ | | ^ |
| | | | | |
| v | | v |
| +--------------+ | | +--------------+ |
| | +--------------+ | | | +--------------+ |
| | | | | | --> | | | |
| | | Worker | | | | | Worker | |
| | | node(s) | | | | | node(s) | |
| +-+ | | | +-+ | |
| +--------------+ | | +--------------+ |
+-----------------------+ +--------------------------+
``` ```
The nodes uses a private network for node to node communication and a public interface for all external communication.
## Requirements ## Requirements
* Terraform 0.13.0 or newer * Terraform 0.13.0 or newer
@ -94,9 +96,10 @@ terraform destroy --var-file cluster-settings.tfvars \
## Variables ## Variables
* `hostname`: A valid domain name, e.g. example.com. The maximum length is 128 characters. * `prefix`: Prefix to add to all resources, if set to "" don't set any prefix
* `template_name`: The name or UUID of a base image * `template_name`: The name or UUID of a base image
* `username`: a user to access the nodes * `username`: a user to access the nodes, defaults to "ubuntu"
* `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24"
* `ssh_public_keys`: List of public SSH keys to install on all machines * `ssh_public_keys`: List of public SSH keys to install on all machines
* `zone`: The zone where to run the cluster * `zone`: The zone where to run the cluster
* `machines`: Machines to provision. Key of this object will be used as the name of the machine * `machines`: Machines to provision. Key of this object will be used as the name of the machine
@ -104,3 +107,6 @@ terraform destroy --var-file cluster-settings.tfvars \
* `cpu`: number of cpu cores * `cpu`: number of cpu cores
* `mem`: memory size in MB * `mem`: memory size in MB
* `disk_size`: The size of the storage in GB * `disk_size`: The size of the storage in GB
* `additional_disks`: Additional disks to attach to the node.
* `size`: The size of the additional disk in GB
* `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)

40
contrib/terraform/upcloud/cluster-settings.tfvars
View file

@ -1,12 +1,11 @@
# See: https://developers.upcloud.com/1.3/5-zones/ # See: https://developers.upcloud.com/1.3/5-zones/
zone = "fi-hel1" zone = "fi-hel1"
username = "ubuntu" username = "ubuntu"
inventory_file = "inventory.ini" # Prefix to use for all resources to separate them from other resources
prefix = "kubespray"
# A valid domain name, e.g. host.example.com. The maximum length is 128 characters. inventory_file = "inventory.ini"
hostname = "example.com"
# Set the operating system using UUID or exact name # Set the operating system using UUID or exact name
template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)" template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"
@ -17,7 +16,7 @@ ssh_public_keys = [
"ssh-rsa public key 2", "ssh-rsa public key 2",
] ]
#check list of available plan https://developers.upcloud.com/1.3/7-plans/ # check list of available plan https://developers.upcloud.com/1.3/7-plans/
machines = { machines = {
"master-0" : { "master-0" : {
"node_type" : "master", "node_type" : "master",
@ -27,6 +26,7 @@ machines = {
"mem" : "4096" "mem" : "4096"
# The size of the storage in GB # The size of the storage in GB
"disk_size" : 250 "disk_size" : 250
"additional_disks" : {}
}, },
"worker-0" : { "worker-0" : {
"node_type" : "worker", "node_type" : "worker",
@ -36,6 +36,16 @@ machines = {
"mem" : "4096" "mem" : "4096"
# The size of the storage in GB # The size of the storage in GB
"disk_size" : 250 "disk_size" : 250
"additional_disks" : {
# "some-disk-name-1": {
# "size": 100,
# "tier": "maxiops",
# },
# "some-disk-name-2": {
# "size": 100,
# "tier": "maxiops",
# }
}
}, },
"worker-1" : { "worker-1" : {
"node_type" : "worker", "node_type" : "worker",
@ -45,6 +55,16 @@ machines = {
"mem" : "4096" "mem" : "4096"
# The size of the storage in GB # The size of the storage in GB
"disk_size" : 250 "disk_size" : 250
"additional_disks" : {
# "some-disk-name-1": {
# "size": 100,
# "tier": "maxiops",
# },
# "some-disk-name-2": {
# "size": 100,
# "tier": "maxiops",
# }
}
}, },
"worker-2" : { "worker-2" : {
"node_type" : "worker", "node_type" : "worker",
@ -54,5 +74,15 @@ machines = {
"mem" : "4096" "mem" : "4096"
# The size of the storage in GB # The size of the storage in GB
"disk_size" : 250 "disk_size" : 250
"additional_disks" : {
# "some-disk-name-1": {
# "size": 100,
# "tier": "maxiops",
# },
# "some-disk-name-2": {
# "size": 100,
# "tier": "maxiops",
# }
}
} }
} }

16
contrib/terraform/upcloud/main.tf
View file

@ -11,12 +11,14 @@ provider "upcloud" {
module "kubernetes" { module "kubernetes" {
source = "./modules/kubernetes-cluster" source = "./modules/kubernetes-cluster"
zone = var.zone prefix = var.prefix
hostname = var.hostname zone = var.zone
template_name = var.template_name template_name = var.template_name
username = var.username username = var.username
private_network_cidr = var.private_network_cidr
machines = var.machines machines = var.machines
ssh_public_keys = var.ssh_public_keys ssh_public_keys = var.ssh_public_keys
@ -30,13 +32,15 @@ data "template_file" "inventory" {
template = file("${path.module}/templates/inventory.tpl") template = file("${path.module}/templates/inventory.tpl")
vars = { vars = {
connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s etcd_member_name=etcd%d", connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
keys(module.kubernetes.master_ip), keys(module.kubernetes.master_ip),
values(module.kubernetes.master_ip), values(module.kubernetes.master_ip).*.public_ip,
values(module.kubernetes.master_ip).*.private_ip,
range(1, length(module.kubernetes.master_ip) + 1))) range(1, length(module.kubernetes.master_ip) + 1)))
connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s", connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
keys(module.kubernetes.worker_ip), keys(module.kubernetes.worker_ip),
values(module.kubernetes.worker_ip))) values(module.kubernetes.worker_ip).*.public_ip,
values(module.kubernetes.worker_ip).*.private_ip))
list_master = join("\n", formatlist("%s", list_master = join("\n", formatlist("%s",
keys(module.kubernetes.master_ip))) keys(module.kubernetes.master_ip)))
list_worker = join("\n", formatlist("%s", list_worker = join("\n", formatlist("%s",

140
contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
View file

@ -1,3 +1,41 @@
locals {
# Create a list of all disks to create
disks = flatten([
for node_name, machine in var.machines : [
for disk_name, disk in machine.additional_disks : {
disk = disk
disk_name = disk_name
node_name = node_name
}
]
])
# If prefix is set, all resources will be prefixed with "${var.prefix}-"
# Else don't prefix with anything
resource-prefix = "%{ if var.prefix != ""}${var.prefix}-%{ endif }"
}
resource "upcloud_network" "private" {
name = "${local.resource-prefix}k8s-network"
zone = var.zone
ip_network {
address = var.private_network_cidr
dhcp = true
family = "IPv4"
}
}
resource "upcloud_storage" "additional_disks" {
for_each = {
for disk in local.disks: "${disk.node_name}_${disk.disk_name}" => disk.disk
}
size = each.value.size
tier = each.value.tier
title = "${local.resource-prefix}${each.key}"
zone = var.zone
}
resource "upcloud_server" "master" { resource "upcloud_server" "master" {
for_each = { for_each = {
@ -6,35 +44,48 @@ resource "upcloud_server" "master" {
if machine.node_type == "master" if machine.node_type == "master"
} }
hostname = "${each.key}.${var.hostname}" hostname = "${local.resource-prefix}${each.key}"
cpu = each.value.cpu cpu = each.value.cpu
mem = each.value.mem mem = each.value.mem
zone = var.zone zone = var.zone
template { template {
storage = var.template_name storage = var.template_name
size = each.value.disk_size size = each.value.disk_size
} }
# Network interfaces # Public network interface
network_interface { network_interface {
type = "public" type = "public"
} }
network_interface { # Private network interface
type = "utility" network_interface {
} type = "private"
# Include at least one public SSH key network = upcloud_network.private.id
login { }
user = var.username
keys = var.ssh_public_keys
create_password = false
} dynamic "storage_devices" {
for_each = {
for disk_key_name, disk in upcloud_storage.additional_disks :
disk_key_name => disk
# Only add the disk if it matches the node name in the start of its name
if length(regexall("^${each.key}_.+", disk_key_name)) > 0
}
content {
storage = storage_devices.value.id
}
}
# Include at least one public SSH key
login {
user = var.username
keys = var.ssh_public_keys
create_password = false
}
} }
resource "upcloud_server" "worker" { resource "upcloud_server" "worker" {
for_each = { for_each = {
for name, machine in var.machines : for name, machine in var.machines :
@ -42,25 +93,44 @@ resource "upcloud_server" "worker" {
if machine.node_type == "worker" if machine.node_type == "worker"
} }
hostname = "${each.key}.${var.hostname}" hostname = "${local.resource-prefix}${each.key}"
cpu = each.value.cpu cpu = each.value.cpu
mem = each.value.mem mem = each.value.mem
zone = var.zone zone = var.zone
template { template {
storage = var.template_name storage = var.template_name
size = each.value.disk_size size = each.value.disk_size
} }
# Network interfaces # Public network interface
network_interface { network_interface {
type = "public" type = "public"
} }
# Include at least one public SSH key # Private network interface
login { network_interface {
user = var.username type = "private"
keys = var.ssh_public_keys network = upcloud_network.private.id
create_password = false }
}
dynamic "storage_devices" {
for_each = {
for disk_key_name, disk in upcloud_storage.additional_disks :
disk_key_name => disk
# Only add the disk if it matches the node name in the start of its name
if length(regexall("^${each.key}_.+", disk_key_name)) > 0
}
content {
storage = storage_devices.value.id
}
}
# Include at least one public SSH key
login {
user = var.username
keys = var.ssh_public_keys
create_password = false
}
} }

10
contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
View file

@ -2,13 +2,19 @@
output "master_ip" { output "master_ip" {
value = { value = {
for instance in upcloud_server.master : for instance in upcloud_server.master :
instance.hostname => instance.network_interface[0].ip_address instance.hostname => {
"public_ip": instance.network_interface[0].ip_address
"private_ip": instance.network_interface[1].ip_address
}
} }
} }
output "worker_ip" { output "worker_ip" {
value = { value = {
for instance in upcloud_server.worker : for instance in upcloud_server.worker :
instance.hostname => instance.network_interface[0].ip_address instance.hostname => {
"public_ip": instance.network_interface[0].ip_address
"private_ip": instance.network_interface[1].ip_address
}
} }
} }

24
contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
View file

@ -1,22 +1,28 @@
variable "prefix" {
type = string
}
variable "zone" { variable "zone" {
type = string type = string
} }
variable "hostname"{ variable "template_name" {}
default ="example.com"
}
variable "template_name"{} variable "username" {}
variable "username"{} variable "private_network_cidr" {}
variable "machines" { variable "machines" {
description = "Cluster machines" description = "Cluster machines"
type = map(object({ type = map(object({
node_type = string node_type = string
cpu = string cpu = string
mem = string mem = string
disk_size = number disk_size = number
additional_disks = map(object({
size = number
tier = string
}))
})) }))
} }

40
contrib/terraform/upcloud/sample-inventory/cluster.tfvars
View file

@ -2,20 +2,21 @@
zone = "fi-hel1" zone = "fi-hel1"
username = "ubuntu" username = "ubuntu"
inventory_file = "inventory.ini" # Prefix to use for all resources to separate them from other resources
prefix = "kubespray"
# A valid domain name, e.g. host.example.com. The maximum length is 128 characters. inventory_file = "inventory.ini"
hostname = "example.com"
# Set the operating system using UUID or exact name # Set the operating system using UUID or exact name
template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)" template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"
ssh_public_keys = [ ssh_public_keys = [
# Put your public SSH key here # Put your public SSH key here
"ssh-rsa I-did-not-read-the-docs", "ssh-rsa I-did-not-read-the-docs",
"ssh-rsa I-did-not-read-the-docs 2", "ssh-rsa I-did-not-read-the-docs 2",
] ]
check list of available plan https://developers.upcloud.com/1.3/7-plans/ # check list of available plan https://developers.upcloud.com/1.3/7-plans/
machines = { machines = {
"master-0" : { "master-0" : {
"node_type" : "master", "node_type" : "master",
@ -25,6 +26,7 @@ machines = {
"mem" : "4096" "mem" : "4096"
# The size of the storage in GB # The size of the storage in GB
"disk_size" : 250 "disk_size" : 250
"additional_disks": {}
}, },
"worker-0" : { "worker-0" : {
"node_type" : "worker", "node_type" : "worker",
@ -34,6 +36,16 @@ machines = {
"mem" : "4096" "mem" : "4096"
# The size of the storage in GB # The size of the storage in GB
"disk_size" : 250 "disk_size" : 250
"additional_disks": {
# "some-disk-name-1": {
# "size": 100,
# "tier": "maxiops",
# },
# "some-disk-name-2": {
# "size": 100,
# "tier": "maxiops",
# }
}
}, },
"worker-1" : { "worker-1" : {
"node_type" : "worker", "node_type" : "worker",
@ -43,6 +55,16 @@ machines = {
"mem" : "4096" "mem" : "4096"
# The size of the storage in GB # The size of the storage in GB
"disk_size" : 250 "disk_size" : 250
"additional_disks": {
# "some-disk-name-1": {
# "size": 100,
# "tier": "maxiops",
# },
# "some-disk-name-2": {
# "size": 100,
# "tier": "maxiops",
# }
}
}, },
"worker-2" : { "worker-2" : {
"node_type" : "worker", "node_type" : "worker",
@ -52,5 +74,15 @@ machines = {
"mem" : "4096" "mem" : "4096"
# The size of the storage in GB # The size of the storage in GB
"disk_size" : 250 "disk_size" : 250
"additional_disks": {
# "some-disk-name-1": {
# "size": 100,
# "tier": "maxiops",
# },
# "some-disk-name-2": {
# "size": 100,
# "tier": "maxiops",
# }
}
} }
} }

33
contrib/terraform/upcloud/variables.tf
View file

@ -1,23 +1,40 @@
variable "prefix" {
type = string
default = "kubespray"
description = "Prefix that is used to distinguish these resources from others"
}
variable "zone" { variable "zone" {
description = "The zone where to run the cluster" description = "The zone where to run the cluster"
} }
variable "hostname" { variable "template_name" {
default = "example.com" description = "Block describing the preconfigured operating system"
} }
variable "template_name" {} variable "username" {
description = "The username to use for the nodes"
default = "ubuntu"
}
variable "username" {} variable "private_network_cidr" {
description = "CIDR to use for the private network"
default = "172.16.0.0/24"
}
variable "machines" { variable "machines" {
description = "Cluster machines" description = "Cluster machines"
type = map(object({ type = map(object({
node_type = string node_type = string
cpu = string cpu = string
mem = string mem = string
disk_size = number disk_size = number
additional_disks = map(object({
size = number
tier = string
}))
})) }))
} }
@ -30,6 +47,10 @@ variable "inventory_file" {
description = "Where to store the generated inventory file" description = "Where to store the generated inventory file"
} }
variable "UPCLOUD_USERNAME" {} variable "UPCLOUD_USERNAME" {
description = "UpCloud username with API access"
}
variable "UPCLOUD_PASSWORD" {} variable "UPCLOUD_PASSWORD" {
description = "Password for UpCloud API user"
}