diff --git a/contrib/terraform/openstack/README.md b/contrib/terraform/openstack/README.md index 1379e5247..ddfc5d695 100644 --- a/contrib/terraform/openstack/README.md +++ b/contrib/terraform/openstack/README.md @@ -88,7 +88,7 @@ binaries available on hyperkube v1.4.3_coreos.0 or higher. ## Requirements -- [Install Terraform](https://www.terraform.io/intro/getting-started/install.html) 0.12 or later +- [Install Terraform](https://www.terraform.io/intro/getting-started/install.html) 0.14 or later - [Install Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html) - you already have a suitable OS image in Glance - you already have a floating IP pool created @@ -284,6 +284,7 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`. |`master_server_group_policy` | Enable and use openstack nova servergroups for masters with set policy, default: "" (disabled) | |`node_server_group_policy` | Enable and use openstack nova servergroups for nodes with set policy, default: "" (disabled) | |`etcd_server_group_policy` | Enable and use openstack nova servergroups for etcd with set policy, default: "" (disabled) | +|`additional_server_groups` | Extra server groups to create. Set "policy" to the policy for the group, expected format is `[{"new-server-group" = {"policy" = "anti-affinity"}}]`, default: [] (to not create any extra groups) | |`use_access_ip` | If 1, nodes with floating IPs will transmit internal cluster traffic via floating IPs; if 0 private IPs will be used instead. Default value is 1. | |`port_security_enabled` | Allow to disable port security by setting this to `false`. `true` by default | |`force_null_port_security` | Set `null` instead of `true` or `false` for `port_security`. `false` by default | @@ -292,12 +293,32 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`. ##### k8s_nodes -Allows a custom definition of worker nodes giving the operator full control over individual node flavor and -availability zone placement. To enable the use of this mode set the `number_of_k8s_nodes` and -`number_of_k8s_nodes_no_floating_ip` variables to 0. Then define your desired worker node configuration -using the `k8s_nodes` variable. The `az`, `flavor` and `floating_ip` parameters are mandatory. +Allows a custom definition of worker nodes giving the operator full control over individual node flavor and availability zone placement. +To enable the use of this mode set the `number_of_k8s_nodes` and `number_of_k8s_nodes_no_floating_ip` variables to 0. +Then define your desired worker node configuration using the `k8s_nodes` variable. +The `az`, `flavor` and `floating_ip` parameters are mandatory. The optional parameter `extra_groups` (a comma-delimited string) can be used to define extra inventory group memberships for specific nodes. +```yaml +k8s_nodes: + node-name: + az: string # Name of the AZ + flavor: string # Flavor ID to use + floating_ip: bool # If floating IPs should be created or not + extra_groups: string # (optional) Additional groups to add for kubespray, defaults to no groups + image_id: string # (optional) Image ID to use, defaults to var.image_id or var.image + root_volume_size_in_gb: number # (optional) Size of the block storage to use as root disk, defaults to var.node_root_volume_size_in_gb or to use volume from flavor otherwise + volume_type: string # (optional) Volume type to use, defaults to var.node_volume_type + server_group: string # (optional) Server group to add this node to. If set, this has to be one specified in additional_server_groups, defaults to use the server group specified in node_server_group_policy + cloudinit: # (optional) Options for cloud-init + extra_partitions: # List of extra partitions (other than the root partition) to setup during creation + volume_path: string # Path to the volume to create partition for (e.g. /dev/vda ) + partition_path: string # Path to the partition (e.g. /dev/vda2 ) + mount_path: string # Path to where the partition should be mounted + partition_start: string # Where the partition should start (e.g. 10GB ). Note, if you set the partition_start to 0 there will be no space left for the root partition + partition_end: string # Where the partition should end (e.g. 10GB or -1 for end of volume) +``` + For example: ```ini diff --git a/contrib/terraform/openstack/kubespray.tf b/contrib/terraform/openstack/kubespray.tf index e4f302f61..a17763432 100644 --- a/contrib/terraform/openstack/kubespray.tf +++ b/contrib/terraform/openstack/kubespray.tf @@ -98,6 +98,7 @@ module "compute" { network_id = module.network.network_id use_existing_network = var.use_existing_network private_subnet_id = module.network.subnet_id + additional_server_groups = var.additional_server_groups depends_on = [ module.network.subnet_id diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf index 7af82e120..0b3508219 100644 --- a/contrib/terraform/openstack/modules/compute/main.tf +++ b/contrib/terraform/openstack/modules/compute/main.tf @@ -18,7 +18,10 @@ data "openstack_images_image_v2" "image_master" { data "cloudinit_config" "cloudinit" { part { content_type = "text/cloud-config" - content = file("${path.module}/templates/cloudinit.yaml") + content = templatefile("${path.module}/templates/cloudinit.yaml", { + # template_file doesn't support lists + extra_partitions = "" + }) } } @@ -170,6 +173,12 @@ resource "openstack_compute_servergroup_v2" "k8s_etcd" { policies = [var.etcd_server_group_policy] } +resource "openstack_compute_servergroup_v2" "k8s_node_additional" { + for_each = var.additional_server_groups + name = "k8s-${each.key}-srvgrp" + policies = [each.value.policy] +} + locals { # master groups master_sec_groups = compact([ @@ -199,6 +208,27 @@ locals { image_to_use_gfs = var.image_gfs_uuid != "" ? var.image_gfs_uuid : var.image_uuid != "" ? var.image_uuid : data.openstack_images_image_v2.gfs_image[0].id # image_master uuidimage_gfs_uuid image_to_use_master = var.image_master_uuid != "" ? var.image_master_uuid : var.image_uuid != "" ? var.image_uuid : data.openstack_images_image_v2.image_master[0].id + + k8s_nodes_settings = { + for name, node in var.k8s_nodes : + name => { + "use_local_disk" = (node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.node_root_volume_size_in_gb) == 0, + "image_id" = node.image_id != null ? node.image_id : local.image_to_use_node, + "volume_size" = node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.node_root_volume_size_in_gb, + "volume_type" = node.volume_type != null ? node.volume_type : var.node_volume_type, + "server_group" = node.server_group != null ? [openstack_compute_servergroup_v2.k8s_node_additional[node.server_group].id] : (var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0].id] : []) + } + } + + k8s_masters_settings = { + for name, node in var.k8s_masters : + name => { + "use_local_disk" = (node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.master_root_volume_size_in_gb) == 0, + "image_id" = node.image_id != null ? node.image_id : local.image_to_use_master, + "volume_size" = node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.master_root_volume_size_in_gb, + "volume_type" = node.volume_type != null ? node.volume_type : var.master_volume_type, + } + } } resource "openstack_networking_port_v2" "bastion_port" { @@ -209,8 +239,11 @@ resource "openstack_networking_port_v2" "bastion_port" { port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.bastion_sec_groups : null no_security_groups = var.port_security_enabled ? null : false - fixed_ip { - subnet_id = var.private_subnet_id + dynamic "fixed_ip" { + for_each = var.private_subnet_id == "" ? [] : [true] + content { + subnet_id = var.private_subnet_id + } } depends_on = [ @@ -262,8 +295,11 @@ resource "openstack_networking_port_v2" "k8s_master_port" { port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.master_sec_groups : null no_security_groups = var.port_security_enabled ? null : false - fixed_ip { - subnet_id = var.private_subnet_id + dynamic "fixed_ip" { + for_each = var.private_subnet_id == "" ? [] : [true] + content { + subnet_id = var.private_subnet_id + } } depends_on = [ @@ -325,8 +361,11 @@ resource "openstack_networking_port_v2" "k8s_masters_port" { port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.master_sec_groups : null no_security_groups = var.port_security_enabled ? null : false - fixed_ip { - subnet_id = var.private_subnet_id + dynamic "fixed_ip" { + for_each = var.private_subnet_id == "" ? [] : [true] + content { + subnet_id = var.private_subnet_id + } } depends_on = [ @@ -338,17 +377,17 @@ resource "openstack_compute_instance_v2" "k8s_masters" { for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? var.k8s_masters : {} name = "${var.cluster_name}-k8s-${each.key}" availability_zone = each.value.az - image_id = var.master_root_volume_size_in_gb == 0 ? local.image_to_use_master : null + image_id = local.k8s_masters_settings[each.key].use_local_disk ? local.k8s_masters_settings[each.key].image_id : null flavor_id = each.value.flavor key_pair = openstack_compute_keypair_v2.k8s.name dynamic "block_device" { - for_each = var.master_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : [] + for_each = !local.k8s_masters_settings[each.key].use_local_disk ? [local.k8s_masters_settings[each.key].image_id] : [] content { - uuid = local.image_to_use_master + uuid = block_device.value source_type = "image" - volume_size = var.master_root_volume_size_in_gb - volume_type = var.master_volume_type + volume_size = local.k8s_masters_settings[each.key].volume_size + volume_type = local.k8s_masters_settings[each.key].volume_type boot_index = 0 destination_type = "volume" delete_on_termination = true @@ -386,8 +425,11 @@ resource "openstack_networking_port_v2" "k8s_master_no_etcd_port" { port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.master_sec_groups : null no_security_groups = var.port_security_enabled ? null : false - fixed_ip { - subnet_id = var.private_subnet_id + dynamic "fixed_ip" { + for_each = var.private_subnet_id == "" ? [] : [true] + content { + subnet_id = var.private_subnet_id + } } depends_on = [ @@ -449,8 +491,11 @@ resource "openstack_networking_port_v2" "etcd_port" { port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.etcd_sec_groups : null no_security_groups = var.port_security_enabled ? null : false - fixed_ip { - subnet_id = var.private_subnet_id + dynamic "fixed_ip" { + for_each = var.private_subnet_id == "" ? [] : [true] + content { + subnet_id = var.private_subnet_id + } } depends_on = [ @@ -506,8 +551,11 @@ resource "openstack_networking_port_v2" "k8s_master_no_floating_ip_port" { port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.master_sec_groups : null no_security_groups = var.port_security_enabled ? null : false - fixed_ip { - subnet_id = var.private_subnet_id + dynamic "fixed_ip" { + for_each = var.private_subnet_id == "" ? [] : [true] + content { + subnet_id = var.private_subnet_id + } } depends_on = [ @@ -563,8 +611,11 @@ resource "openstack_networking_port_v2" "k8s_master_no_floating_ip_no_etcd_port" port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.master_sec_groups : null no_security_groups = var.port_security_enabled ? null : false - fixed_ip { - subnet_id = var.private_subnet_id + dynamic "fixed_ip" { + for_each = var.private_subnet_id == "" ? [] : [true] + content { + subnet_id = var.private_subnet_id + } } depends_on = [ @@ -621,8 +672,11 @@ resource "openstack_networking_port_v2" "k8s_node_port" { port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null no_security_groups = var.port_security_enabled ? null : false - fixed_ip { - subnet_id = var.private_subnet_id + dynamic "fixed_ip" { + for_each = var.private_subnet_id == "" ? [] : [true] + content { + subnet_id = var.private_subnet_id + } } depends_on = [ @@ -684,8 +738,11 @@ resource "openstack_networking_port_v2" "k8s_node_no_floating_ip_port" { port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null no_security_groups = var.port_security_enabled ? null : false - fixed_ip { - subnet_id = var.private_subnet_id + dynamic "fixed_ip" { + for_each = var.private_subnet_id == "" ? [] : [true] + content { + subnet_id = var.private_subnet_id + } } depends_on = [ @@ -720,9 +777,9 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" { } dynamic "scheduler_hints" { - for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : [] + for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0].id] : [] content { - group = openstack_compute_servergroup_v2.k8s_node[0].id + group = scheduler_hints.value } } @@ -742,8 +799,11 @@ resource "openstack_networking_port_v2" "k8s_nodes_port" { port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null no_security_groups = var.port_security_enabled ? null : false - fixed_ip { - subnet_id = var.private_subnet_id + dynamic "fixed_ip" { + for_each = var.private_subnet_id == "" ? [] : [true] + content { + subnet_id = var.private_subnet_id + } } depends_on = [ @@ -755,18 +815,20 @@ resource "openstack_compute_instance_v2" "k8s_nodes" { for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? var.k8s_nodes : {} name = "${var.cluster_name}-k8s-node-${each.key}" availability_zone = each.value.az - image_id = var.node_root_volume_size_in_gb == 0 ? local.image_to_use_node : null + image_id = local.k8s_nodes_settings[each.key].use_local_disk ? local.k8s_nodes_settings[each.key].image_id : null flavor_id = each.value.flavor key_pair = openstack_compute_keypair_v2.k8s.name - user_data = data.cloudinit_config.cloudinit.rendered + user_data = each.value.cloudinit != null ? templatefile("${path.module}/templates/cloudinit.yaml", { + extra_partitions = each.value.cloudinit.extra_partitions + }) : data.cloudinit_config.cloudinit.rendered dynamic "block_device" { - for_each = var.node_root_volume_size_in_gb > 0 ? [local.image_to_use_node] : [] + for_each = !local.k8s_nodes_settings[each.key].use_local_disk ? [local.k8s_nodes_settings[each.key].image_id] : [] content { - uuid = local.image_to_use_node + uuid = block_device.value source_type = "image" - volume_size = var.node_root_volume_size_in_gb - volume_type = var.node_volume_type + volume_size = local.k8s_nodes_settings[each.key].volume_size + volume_type = local.k8s_nodes_settings[each.key].volume_type boot_index = 0 destination_type = "volume" delete_on_termination = true @@ -778,15 +840,15 @@ resource "openstack_compute_instance_v2" "k8s_nodes" { } dynamic "scheduler_hints" { - for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : [] + for_each = local.k8s_nodes_settings[each.key].server_group content { - group = openstack_compute_servergroup_v2.k8s_node[0].id + group = scheduler_hints.value } } metadata = { ssh_user = var.ssh_user - kubespray_groups = "kube_node,k8s_cluster,%{if each.value.floating_ip == false}no_floating,%{endif}${var.supplementary_node_groups},${try(each.value.extra_groups, "")}" + kubespray_groups = "kube_node,k8s_cluster,%{if each.value.floating_ip == false}no_floating,%{endif}${var.supplementary_node_groups}${each.value.extra_groups != null ? ",${each.value.extra_groups}" : ""}" depends_on = var.network_router_id use_access_ip = var.use_access_ip } @@ -804,8 +866,11 @@ resource "openstack_networking_port_v2" "glusterfs_node_no_floating_ip_port" { port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled security_group_ids = var.port_security_enabled ? local.gfs_sec_groups : null no_security_groups = var.port_security_enabled ? null : false - fixed_ip { - subnet_id = var.private_subnet_id + dynamic "fixed_ip" { + for_each = var.private_subnet_id == "" ? [] : [true] + content { + subnet_id = var.private_subnet_id + } } depends_on = [ diff --git a/contrib/terraform/openstack/modules/compute/templates/cloudinit.yaml b/contrib/terraform/openstack/modules/compute/templates/cloudinit.yaml index 396acb9f7..879642cc1 100644 --- a/contrib/terraform/openstack/modules/compute/templates/cloudinit.yaml +++ b/contrib/terraform/openstack/modules/compute/templates/cloudinit.yaml @@ -1,3 +1,24 @@ +%{~ if length(extra_partitions) > 0 } +#cloud-config +bootcmd: +%{~ for idx, partition in extra_partitions } +- [ cloud-init-per, once, move-second-header, sgdisk, --move-second-header, ${partition.volume_path} ] +- [ cloud-init-per, once, create-part-${idx}, parted, --script, ${partition.volume_path}, 'mkpart extended ext4 ${partition.partition_start} ${partition.partition_end}' ] +- [ cloud-init-per, once, create-fs-part-${idx}, mkfs.ext4, ${partition.partition_path} ] +%{~ endfor } + +runcmd: +%{~ for idx, partition in extra_partitions } + - mkdir -p ${partition.mount_path} + - chown nobody:nogroup ${partition.mount_path} + - mount ${partition.partition_path} ${partition.mount_path} +%{~ endfor } + +mounts: +%{~ for idx, partition in extra_partitions } + - [ ${partition.partition_path}, ${partition.mount_path} ] +%{~ endfor } +%{~ else ~} # yamllint disable rule:comments #cloud-config ## in some cases novnc console access is required @@ -15,3 +36,4 @@ #ca-certs: # trusted: | # -----BEGIN CERTIFICATE----- +%{~ endif } diff --git a/contrib/terraform/openstack/modules/compute/variables.tf b/contrib/terraform/openstack/modules/compute/variables.tf index 9259fd967..1f78136e2 100644 --- a/contrib/terraform/openstack/modules/compute/variables.tf +++ b/contrib/terraform/openstack/modules/compute/variables.tf @@ -116,9 +116,46 @@ variable "k8s_allowed_egress_ips" { type = list } -variable "k8s_masters" {} +variable "k8s_masters" { + type = map(object({ + az = string + flavor = string + floating_ip = bool + etcd = bool + image_id = optional(string) + root_volume_size_in_gb = optional(number) + volume_type = optional(string) + })) +} -variable "k8s_nodes" {} +variable "k8s_nodes" { + type = map(object({ + az = string + flavor = string + floating_ip = bool + extra_groups = optional(string) + image_id = optional(string) + root_volume_size_in_gb = optional(number) + volume_type = optional(string) + additional_server_groups = optional(list(string)) + server_group = optional(string) + cloudinit = optional(object({ + extra_partitions = list(object({ + volume_path = string + partition_path = string + partition_start = string + partition_end = string + mount_path = string + })) + })) + })) +} + +variable "additional_server_groups" { + type = map(object({ + policy = string + })) +} variable "supplementary_master_groups" { default = "" diff --git a/contrib/terraform/openstack/modules/compute/versions.tf b/contrib/terraform/openstack/modules/compute/versions.tf index 6c942790d..c268dceeb 100644 --- a/contrib/terraform/openstack/modules/compute/versions.tf +++ b/contrib/terraform/openstack/modules/compute/versions.tf @@ -4,5 +4,6 @@ terraform { source = "terraform-provider-openstack/openstack" } } - required_version = ">= 0.12.26" + experiments = [module_variable_optional_attrs] + required_version = ">= 0.14.0" } diff --git a/contrib/terraform/openstack/variables.tf b/contrib/terraform/openstack/variables.tf index 821e442b8..ff905f3a0 100644 --- a/contrib/terraform/openstack/variables.tf +++ b/contrib/terraform/openstack/variables.tf @@ -294,10 +294,45 @@ variable "router_internal_port_id" { variable "k8s_masters" { default = {} + type = map(object({ + az = string + flavor = string + floating_ip = bool + etcd = bool + image_id = optional(string) + root_volume_size_in_gb = optional(number) + volume_type = optional(string) + })) } variable "k8s_nodes" { default = {} + type = map(object({ + az = string + flavor = string + floating_ip = bool + extra_groups = optional(string) + image_id = optional(string) + root_volume_size_in_gb = optional(number) + volume_type = optional(string) + server_group = optional(string) + cloudinit = optional(object({ + extra_partitions = list(object({ + volume_path = string + partition_path = string + partition_start = string + partition_end = string + mount_path = string + })) + })) + })) +} + +variable "additional_server_groups" { + default = {} + type = map(object({ + policy = string + })) } variable "extra_sec_groups" { diff --git a/contrib/terraform/openstack/versions.tf b/contrib/terraform/openstack/versions.tf index 9541063a2..8a33a62dd 100644 --- a/contrib/terraform/openstack/versions.tf +++ b/contrib/terraform/openstack/versions.tf @@ -5,5 +5,6 @@ terraform { version = "~> 1.17" } } - required_version = ">= 0.12.26" + experiments = [module_variable_optional_attrs] + required_version = ">= 0.14.0" }