C12s/c12s-kubespray
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Added override variable, additional server groups and cloudinit config

Browse source
This commit is contained in:
Fredrik Liv 2022-10-25 11:48:39 +02:00
parent 09748e80e9
commit 657c2de4c3
No known key found for this signature in database
GPG key ID: 55E58068DB18E7F3
8 changed files with 231 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
contrib/terraform/openstack/README.md
View file

@ -88,7 +88,7 @@ binaries available on hyperkube v1.4.3_coreos.0 or higher.
## Requirements
- [Install Terraform](https://www.terraform.io/intro/getting-started/install.html) 0.12 or later
- [Install Terraform](https://www.terraform.io/intro/getting-started/install.html) 0.14 or later
- [Install Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html)
- you already have a suitable OS image in Glance
- you already have a floating IP pool created
@ -284,6 +284,7 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
|`master_server_group_policy` | Enable and use openstack nova servergroups for masters with set policy, default: "" (disabled) |
|`node_server_group_policy` | Enable and use openstack nova servergroups for nodes with set policy, default: "" (disabled) |
|`etcd_server_group_policy` | Enable and use openstack nova servergroups for etcd with set policy, default: "" (disabled) |
|`additional_server_groups` | Extra server groups to create. Set "policy" to the policy for the group, expected format is `[{"new-server-group" = {"policy" = "anti-affinity"}}]`, default: [] (to not create any extra groups) |
|`use_access_ip` | If 1, nodes with floating IPs will transmit internal cluster traffic via floating IPs; if 0 private IPs will be used instead. Default value is 1. |
|`port_security_enabled` | Allow to disable port security by setting this to `false`. `true` by default |
|`force_null_port_security` | Set `null` instead of `true` or `false` for `port_security`. `false` by default |
@ -292,12 +293,32 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
##### k8s_nodes
Allows a custom definition of worker nodes giving the operator full control over individual node flavor and
availability zone placement. To enable the use of this mode set the `number_of_k8s_nodes` and
`number_of_k8s_nodes_no_floating_ip` variables to 0. Then define your desired worker node configuration
using the `k8s_nodes` variable. The `az`, `flavor` and `floating_ip` parameters are mandatory.
Allows a custom definition of worker nodes giving the operator full control over individual node flavor and availability zone placement.
To enable the use of this mode set the `number_of_k8s_nodes` and `number_of_k8s_nodes_no_floating_ip` variables to 0.
Then define your desired worker node configuration using the `k8s_nodes` variable.
The `az`, `flavor` and `floating_ip` parameters are mandatory.
The optional parameter `extra_groups` (a comma-delimited string) can be used to define extra inventory group memberships for specific nodes.
```yaml
k8s_nodes:
node-name:
az: string # Name of the AZ
flavor: string # Flavor ID to use
floating_ip: bool # If floating IPs should be created or not
extra_groups: string # (optional) Additional groups to add for kubespray, defaults to no groups
image_id: string # (optional) Image ID to use, defaults to var.image_id or var.image
root_volume_size_in_gb: number # (optional) Size of the block storage to use as root disk, defaults to var.node_root_volume_size_in_gb or to use volume from flavor otherwise
volume_type: string # (optional) Volume type to use, defaults to var.node_volume_type
server_group: string # (optional) Server group to add this node to. If set, this has to be one specified in additional_server_groups, defaults to use the server group specified in node_server_group_policy
cloudinit: # (optional) Options for cloud-init
extra_partitions: # List of extra partitions (other than the root partition) to setup during creation
volume_path: string # Path to the volume to create partition for (e.g. /dev/vda )
partition_path: string # Path to the partition (e.g. /dev/vda2 )
mount_path: string # Path to where the partition should be mounted
partition_start: string # Where the partition should start (e.g. 10GB ). Note, if you set the partition_start to 0 there will be no space left for the root partition
partition_end: string # Where the partition should end (e.g. 10GB or -1 for end of volume)
```
For example:
```ini

1
contrib/terraform/openstack/kubespray.tf
View file

@ -98,6 +98,7 @@ module "compute" {
network_id = module.network.network_id
use_existing_network = var.use_existing_network
private_subnet_id = module.network.subnet_id
additional_server_groups = var.additional_server_groups
depends_on = [
module.network.subnet_id

121
contrib/terraform/openstack/modules/compute/main.tf
View file

@ -18,7 +18,10 @@ data "openstack_images_image_v2" "image_master" {
data "cloudinit_config" "cloudinit" {
part {
content_type = "text/cloud-config"
content = file("${path.module}/templates/cloudinit.yaml")
content = templatefile("${path.module}/templates/cloudinit.yaml", {
# template_file doesn't support lists
extra_partitions = ""
})
}
}
@ -170,6 +173,12 @@ resource "openstack_compute_servergroup_v2" "k8s_etcd" {
policies = [var.etcd_server_group_policy]
}
resource "openstack_compute_servergroup_v2" "k8s_node_additional" {
for_each = var.additional_server_groups
name = "k8s-${each.key}-srvgrp"
policies = [each.value.policy]
}
locals {
# master groups
master_sec_groups = compact([
@ -199,6 +208,27 @@ locals {
image_to_use_gfs = var.image_gfs_uuid != "" ? var.image_gfs_uuid : var.image_uuid != "" ? var.image_uuid : data.openstack_images_image_v2.gfs_image[0].id
# image_master uuidimage_gfs_uuid
image_to_use_master = var.image_master_uuid != "" ? var.image_master_uuid : var.image_uuid != "" ? var.image_uuid : data.openstack_images_image_v2.image_master[0].id
k8s_nodes_settings = {
for name, node in var.k8s_nodes :
name => {
"use_local_disk" = (node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.node_root_volume_size_in_gb) == 0,
"image_id" = node.image_id != null ? node.image_id : local.image_to_use_node,
"volume_size" = node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.node_root_volume_size_in_gb,
"volume_type" = node.volume_type != null ? node.volume_type : var.node_volume_type,
"server_group" = node.server_group != null ? [openstack_compute_servergroup_v2.k8s_node_additional[node.server_group].id] : (var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0].id] : [])
}
}
k8s_masters_settings = {
for name, node in var.k8s_masters :
name => {
"use_local_disk" = (node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.master_root_volume_size_in_gb) == 0,
"image_id" = node.image_id != null ? node.image_id : local.image_to_use_master,
"volume_size" = node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.master_root_volume_size_in_gb,
"volume_type" = node.volume_type != null ? node.volume_type : var.master_volume_type,
}
}
}
resource "openstack_networking_port_v2" "bastion_port" {
@ -209,9 +239,12 @@ resource "openstack_networking_port_v2" "bastion_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.bastion_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
var.network_router_id
@ -262,9 +295,12 @@ resource "openstack_networking_port_v2" "k8s_master_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.master_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
var.network_router_id
@ -325,9 +361,12 @@ resource "openstack_networking_port_v2" "k8s_masters_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.master_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
var.network_router_id
@ -338,17 +377,17 @@ resource "openstack_compute_instance_v2" "k8s_masters" {
for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? var.k8s_masters : {}
name = "${var.cluster_name}-k8s-${each.key}"
availability_zone = each.value.az
image_id = var.master_root_volume_size_in_gb == 0 ? local.image_to_use_master : null
image_id = local.k8s_masters_settings[each.key].use_local_disk ? local.k8s_masters_settings[each.key].image_id : null
flavor_id = each.value.flavor
key_pair = openstack_compute_keypair_v2.k8s.name
dynamic "block_device" {
for_each = var.master_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : []
for_each = !local.k8s_masters_settings[each.key].use_local_disk ? [local.k8s_masters_settings[each.key].image_id] : []
content {
uuid = local.image_to_use_master
uuid = block_device.value
source_type = "image"
volume_size = var.master_root_volume_size_in_gb
volume_type = var.master_volume_type
volume_size = local.k8s_masters_settings[each.key].volume_size
volume_type = local.k8s_masters_settings[each.key].volume_type
boot_index = 0
destination_type = "volume"
delete_on_termination = true
@ -386,9 +425,12 @@ resource "openstack_networking_port_v2" "k8s_master_no_etcd_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.master_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
var.network_router_id
@ -449,9 +491,12 @@ resource "openstack_networking_port_v2" "etcd_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.etcd_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
var.network_router_id
@ -506,9 +551,12 @@ resource "openstack_networking_port_v2" "k8s_master_no_floating_ip_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.master_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
var.network_router_id
@ -563,9 +611,12 @@ resource "openstack_networking_port_v2" "k8s_master_no_floating_ip_no_etcd_port"
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.master_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
var.network_router_id
@ -621,9 +672,12 @@ resource "openstack_networking_port_v2" "k8s_node_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
var.network_router_id
@ -684,9 +738,12 @@ resource "openstack_networking_port_v2" "k8s_node_no_floating_ip_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
var.network_router_id
@ -720,9 +777,9 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" {
}
dynamic "scheduler_hints" {
for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0].id] : []
content {
group = openstack_compute_servergroup_v2.k8s_node[0].id
group = scheduler_hints.value
}
}
@ -742,9 +799,12 @@ resource "openstack_networking_port_v2" "k8s_nodes_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
var.network_router_id
@ -755,18 +815,20 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? var.k8s_nodes : {}
name = "${var.cluster_name}-k8s-node-${each.key}"
availability_zone = each.value.az
image_id = var.node_root_volume_size_in_gb == 0 ? local.image_to_use_node : null
image_id = local.k8s_nodes_settings[each.key].use_local_disk ? local.k8s_nodes_settings[each.key].image_id : null
flavor_id = each.value.flavor
key_pair = openstack_compute_keypair_v2.k8s.name
user_data = data.cloudinit_config.cloudinit.rendered
user_data = each.value.cloudinit != null ? templatefile("${path.module}/templates/cloudinit.yaml", {
extra_partitions = each.value.cloudinit.extra_partitions
}) : data.cloudinit_config.cloudinit.rendered
dynamic "block_device" {
for_each = var.node_root_volume_size_in_gb > 0 ? [local.image_to_use_node] : []
for_each = !local.k8s_nodes_settings[each.key].use_local_disk ? [local.k8s_nodes_settings[each.key].image_id] : []
content {
uuid = local.image_to_use_node
uuid = block_device.value
source_type = "image"
volume_size = var.node_root_volume_size_in_gb
volume_type = var.node_volume_type
volume_size = local.k8s_nodes_settings[each.key].volume_size
volume_type = local.k8s_nodes_settings[each.key].volume_type
boot_index = 0
destination_type = "volume"
delete_on_termination = true
@ -778,15 +840,15 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
}
dynamic "scheduler_hints" {
for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
for_each = local.k8s_nodes_settings[each.key].server_group
content {
group = openstack_compute_servergroup_v2.k8s_node[0].id
group = scheduler_hints.value
}
}
metadata = {
ssh_user = var.ssh_user
kubespray_groups = "kube_node,k8s_cluster,%{if each.value.floating_ip == false}no_floating,%{endif}${var.supplementary_node_groups},${try(each.value.extra_groups, "")}"
kubespray_groups = "kube_node,k8s_cluster,%{if each.value.floating_ip == false}no_floating,%{endif}${var.supplementary_node_groups}${each.value.extra_groups != null ? ",${each.value.extra_groups}" : ""}"
depends_on = var.network_router_id
use_access_ip = var.use_access_ip
}
@ -804,9 +866,12 @@ resource "openstack_networking_port_v2" "glusterfs_node_no_floating_ip_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.gfs_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
var.network_router_id

22
contrib/terraform/openstack/modules/compute/templates/cloudinit.yaml
View file

@ -1,3 +1,24 @@
%{~ if length(extra_partitions) > 0 }
#cloud-config
bootcmd:
%{~ for idx, partition in extra_partitions }
- [ cloud-init-per, once, move-second-header, sgdisk, --move-second-header, ${partition.volume_path} ]
- [ cloud-init-per, once, create-part-${idx}, parted, --script, ${partition.volume_path}, 'mkpart extended ext4 ${partition.partition_start} ${partition.partition_end}' ]
- [ cloud-init-per, once, create-fs-part-${idx}, mkfs.ext4, ${partition.partition_path} ]
%{~ endfor }
runcmd:
%{~ for idx, partition in extra_partitions }
- mkdir -p ${partition.mount_path}
- chown nobody:nogroup ${partition.mount_path}
- mount ${partition.partition_path} ${partition.mount_path}
%{~ endfor }
mounts:
%{~ for idx, partition in extra_partitions }
- [ ${partition.partition_path}, ${partition.mount_path} ]
%{~ endfor }
%{~ else ~}
# yamllint disable rule:comments
#cloud-config
## in some cases novnc console access is required
@ -15,3 +36,4 @@
#ca-certs:
# trusted: |
# -----BEGIN CERTIFICATE-----
%{~ endif }

41
contrib/terraform/openstack/modules/compute/variables.tf
View file

@ -116,9 +116,46 @@ variable "k8s_allowed_egress_ips" {
type = list
}
variable "k8s_masters" {}
variable "k8s_masters" {
type = map(object({
az = string
flavor = string
floating_ip = bool
etcd = bool
image_id = optional(string)
root_volume_size_in_gb = optional(number)
volume_type = optional(string)
}))
}
variable "k8s_nodes" {}
variable "k8s_nodes" {
type = map(object({
az = string
flavor = string
floating_ip = bool
extra_groups = optional(string)
image_id = optional(string)
root_volume_size_in_gb = optional(number)
volume_type = optional(string)
additional_server_groups = optional(list(string))
server_group = optional(string)
cloudinit = optional(object({
extra_partitions = list(object({
volume_path = string
partition_path = string
partition_start = string
partition_end = string
mount_path = string
}))
}))
}))
}
variable "additional_server_groups" {
type = map(object({
policy = string
}))
}
variable "supplementary_master_groups" {
default = ""

3
contrib/terraform/openstack/modules/compute/versions.tf
View file

@ -4,5 +4,6 @@ terraform {
source = "terraform-provider-openstack/openstack"
}
}
required_version = ">= 0.12.26"
experiments = [module_variable_optional_attrs]
required_version = ">= 0.14.0"
}

35
contrib/terraform/openstack/variables.tf
View file

@ -294,10 +294,45 @@ variable "router_internal_port_id" {
variable "k8s_masters" {
default = {}
type = map(object({
az = string
flavor = string
floating_ip = bool
etcd = bool
image_id = optional(string)
root_volume_size_in_gb = optional(number)
volume_type = optional(string)
}))
}
variable "k8s_nodes" {
default = {}
type = map(object({
az = string
flavor = string
floating_ip = bool
extra_groups = optional(string)
image_id = optional(string)
root_volume_size_in_gb = optional(number)
volume_type = optional(string)
server_group = optional(string)
cloudinit = optional(object({
extra_partitions = list(object({
volume_path = string
partition_path = string
partition_start = string
partition_end = string
mount_path = string
}))
}))
}))
}
variable "additional_server_groups" {
default = {}
type = map(object({
policy = string
}))
}
variable "extra_sec_groups" {

3
contrib/terraform/openstack/versions.tf
View file

@ -5,5 +5,6 @@ terraform {
version = "~> 1.17"
}
}
required_version = ">= 0.12.26"
experiments = [module_variable_optional_attrs]
required_version = ">= 0.14.0"
}