C12s/c12s-kubespray
3
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Merge 646d4d9b87 into fc0d58ff48

Browse source
This commit is contained in:
Fredrik Liv 2022-12-19 18:57:46 -08:00 committed by GitHub
parent fc0d58ff48 646d4d9b87
commit 473650f031
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 259 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
contrib/terraform/openstack/README.md
View file

@ -88,7 +88,7 @@ binaries available on hyperkube v1.4.3_coreos.0 or higher.
## Requirements
- [Install Terraform](https://www.terraform.io/intro/getting-started/install.html) 0.12 or later
- [Install Terraform](https://www.terraform.io/intro/getting-started/install.html) 0.14 or later
- [Install Ansible](http://docs.ansible.com/ansible/latest/intro_installation.html)
- you already have a suitable OS image in Glance
- you already have a floating IP pool created
@ -284,6 +284,7 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
|`master_server_group_policy` | Enable and use openstack nova servergroups for masters with set policy, default: "" (disabled) |
|`node_server_group_policy` | Enable and use openstack nova servergroups for nodes with set policy, default: "" (disabled) |
|`etcd_server_group_policy` | Enable and use openstack nova servergroups for etcd with set policy, default: "" (disabled) |
|`additional_server_groups` | Extra server groups to create. Set "policy" to the policy for the group, expected format is `{"new-server-group" = {"policy" = "anti-affinity"}}`, default: {} (to not create any extra groups) |
|`use_access_ip` | If 1, nodes with floating IPs will transmit internal cluster traffic via floating IPs; if 0 private IPs will be used instead. Default value is 1. |
|`port_security_enabled` | Allow to disable port security by setting this to `false`. `true` by default |
|`force_null_port_security` | Set `null` instead of `true` or `false` for `port_security`. `false` by default |
@ -292,12 +293,33 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
##### k8s_nodes
Allows a custom definition of worker nodes giving the operator full control over individual node flavor and
availability zone placement. To enable the use of this mode set the `number_of_k8s_nodes` and
`number_of_k8s_nodes_no_floating_ip` variables to 0. Then define your desired worker node configuration
using the `k8s_nodes` variable. The `az`, `flavor` and `floating_ip` parameters are mandatory.
Allows a custom definition of worker nodes giving the operator full control over individual node flavor and availability zone placement.
To enable the use of this mode set the `number_of_k8s_nodes` and `number_of_k8s_nodes_no_floating_ip` variables to 0.
Then define your desired worker node configuration using the `k8s_nodes` variable.
The `az`, `flavor` and `floating_ip` parameters are mandatory.
The optional parameter `extra_groups` (a comma-delimited string) can be used to define extra inventory group memberships for specific nodes.
```yaml
k8s_nodes:
node-name:
az: string # Name of the AZ
flavor: string # Flavor ID to use
floating_ip: bool # If floating IPs should be created or not
extra_groups: string # (optional) Additional groups to add for kubespray, defaults to no groups
image_id: string # (optional) Image ID to use, defaults to var.image_id or var.image
root_volume_size_in_gb: number # (optional) Size of the block storage to use as root disk, defaults to var.node_root_volume_size_in_gb or to use volume from flavor otherwise
volume_type: string # (optional) Volume type to use, defaults to var.node_volume_type
network_id: string # (optional) Use this network_id for the node, defaults to either var.network_id or ID of var.network_name
server_group: string # (optional) Server group to add this node to. If set, this has to be one specified in additional_server_groups, defaults to use the server group specified in node_server_group_policy
cloudinit: # (optional) Options for cloud-init
extra_partitions: # List of extra partitions (other than the root partition) to setup during creation
volume_path: string # Path to the volume to create partition for (e.g. /dev/vda )
partition_path: string # Path to the partition (e.g. /dev/vda2 )
mount_path: string # Path to where the partition should be mounted
partition_start: string # Where the partition should start (e.g. 10GB ). Note, if you set the partition_start to 0 there will be no space left for the root partition
partition_end: string # Where the partition should end (e.g. 10GB or -1 for end of volume)
```
For example:
```ini
@ -427,7 +449,7 @@ This should finish fairly quickly telling you Terraform has successfully initial
You can apply cloud-init based customization for the openstack instances before provisioning your cluster.
One common template is used for all instances. Adjust the file shown below:
`contrib/terraform/openstack/modules/compute/templates/cloudinit.yaml`
`contrib/terraform/openstack/modules/compute/templates/cloudinit.yaml.tmpl`
For example, to enable openstack novnc access and ansible_user=root SSH access:
```ShellSession

1
contrib/terraform/openstack/kubespray.tf
View file

@ -98,6 +98,7 @@ module "compute" {
network_id = module.network.network_id
use_existing_network = var.use_existing_network
private_subnet_id = module.network.subnet_id
additional_server_groups = var.additional_server_groups
depends_on = [
module.network.subnet_id

151
contrib/terraform/openstack/modules/compute/main.tf
View file

@ -18,7 +18,10 @@ data "openstack_images_image_v2" "image_master" {
data "cloudinit_config" "cloudinit" {
part {
content_type = "text/cloud-config"
content = file("${path.module}/templates/cloudinit.yaml")
content = templatefile("${path.module}/templates/cloudinit.yaml.tmpl", {
# template_file doesn't support lists
extra_partitions = ""
})
}
}
@ -170,6 +173,12 @@ resource "openstack_compute_servergroup_v2" "k8s_etcd" {
policies = [var.etcd_server_group_policy]
}
resource "openstack_compute_servergroup_v2" "k8s_node_additional" {
for_each = var.additional_server_groups
name = "k8s-${each.key}-srvgrp"
policies = [each.value.policy]
}
locals {
# master groups
master_sec_groups = compact([
@ -199,6 +208,29 @@ locals {
image_to_use_gfs = var.image_gfs_uuid != "" ? var.image_gfs_uuid : var.image_uuid != "" ? var.image_uuid : data.openstack_images_image_v2.gfs_image[0].id
# image_master uuidimage_gfs_uuid
image_to_use_master = var.image_master_uuid != "" ? var.image_master_uuid : var.image_uuid != "" ? var.image_uuid : data.openstack_images_image_v2.image_master[0].id
k8s_nodes_settings = {
for name, node in var.k8s_nodes :
name => {
"use_local_disk" = (node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.node_root_volume_size_in_gb) == 0,
"image_id" = node.image_id != null ? node.image_id : local.image_to_use_node,
"volume_size" = node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.node_root_volume_size_in_gb,
"volume_type" = node.volume_type != null ? node.volume_type : var.node_volume_type,
"network_id" = node.network_id != null ? node.network_id : (var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id)
"server_group" = node.server_group != null ? [openstack_compute_servergroup_v2.k8s_node_additional[node.server_group].id] : (var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0].id] : [])
}
}
k8s_masters_settings = {
for name, node in var.k8s_masters :
name => {
"use_local_disk" = (node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.master_root_volume_size_in_gb) == 0,
"image_id" = node.image_id != null ? node.image_id : local.image_to_use_master,
"volume_size" = node.root_volume_size_in_gb != null ? node.root_volume_size_in_gb : var.master_root_volume_size_in_gb,
"volume_type" = node.volume_type != null ? node.volume_type : var.master_volume_type,
"network_id" = node.network_id != null ? node.network_id : (var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id)
}
}
}
resource "openstack_networking_port_v2" "bastion_port" {
@ -209,8 +241,11 @@ resource "openstack_networking_port_v2" "bastion_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.bastion_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
subnet_id = var.private_subnet_id
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
@ -262,8 +297,11 @@ resource "openstack_networking_port_v2" "k8s_master_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.master_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
subnet_id = var.private_subnet_id
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
@ -320,13 +358,16 @@ resource "openstack_compute_instance_v2" "k8s_master" {
resource "openstack_networking_port_v2" "k8s_masters_port" {
for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? var.k8s_masters : {}
name = "${var.cluster_name}-k8s-${each.key}"
network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id
network_id = local.k8s_masters_settings[each.key].network_id
admin_state_up = "true"
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.master_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
subnet_id = var.private_subnet_id
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
@ -338,17 +379,17 @@ resource "openstack_compute_instance_v2" "k8s_masters" {
for_each = var.number_of_k8s_masters == 0 && var.number_of_k8s_masters_no_etcd == 0 && var.number_of_k8s_masters_no_floating_ip == 0 && var.number_of_k8s_masters_no_floating_ip_no_etcd == 0 ? var.k8s_masters : {}
name = "${var.cluster_name}-k8s-${each.key}"
availability_zone = each.value.az
image_id = var.master_root_volume_size_in_gb == 0 ? local.image_to_use_master : null
image_id = local.k8s_masters_settings[each.key].use_local_disk ? local.k8s_masters_settings[each.key].image_id : null
flavor_id = each.value.flavor
key_pair = openstack_compute_keypair_v2.k8s.name
dynamic "block_device" {
for_each = var.master_root_volume_size_in_gb > 0 ? [local.image_to_use_master] : []
for_each = !local.k8s_masters_settings[each.key].use_local_disk ? [local.k8s_masters_settings[each.key].image_id] : []
content {
uuid = local.image_to_use_master
uuid = block_device.value
source_type = "image"
volume_size = var.master_root_volume_size_in_gb
volume_type = var.master_volume_type
volume_size = local.k8s_masters_settings[each.key].volume_size
volume_type = local.k8s_masters_settings[each.key].volume_type
boot_index = 0
destination_type = "volume"
delete_on_termination = true
@ -374,7 +415,7 @@ resource "openstack_compute_instance_v2" "k8s_masters" {
}
provisioner "local-exec" {
command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ${path.root}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_masters_fips : value.address]), 0)}/ > ${var.group_vars_path}/no_floating.yml%{else}true%{endif}"
command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ${path.module}/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_masters_fips : value.address]), 0)}/ > ${var.group_vars_path}/no_floating.yml%{else}true%{endif}"
}
}
@ -386,8 +427,11 @@ resource "openstack_networking_port_v2" "k8s_master_no_etcd_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.master_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
subnet_id = var.private_subnet_id
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
@ -449,8 +493,11 @@ resource "openstack_networking_port_v2" "etcd_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.etcd_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
subnet_id = var.private_subnet_id
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
@ -506,8 +553,11 @@ resource "openstack_networking_port_v2" "k8s_master_no_floating_ip_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.master_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
subnet_id = var.private_subnet_id
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
@ -563,8 +613,11 @@ resource "openstack_networking_port_v2" "k8s_master_no_floating_ip_no_etcd_port"
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.master_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
subnet_id = var.private_subnet_id
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
@ -621,8 +674,11 @@ resource "openstack_networking_port_v2" "k8s_node_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
subnet_id = var.private_subnet_id
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
@ -684,8 +740,11 @@ resource "openstack_networking_port_v2" "k8s_node_no_floating_ip_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
subnet_id = var.private_subnet_id
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
@ -720,9 +779,9 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" {
}
dynamic "scheduler_hints" {
for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0].id] : []
content {
group = openstack_compute_servergroup_v2.k8s_node[0].id
group = scheduler_hints.value
}
}
@ -737,13 +796,16 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" {
resource "openstack_networking_port_v2" "k8s_nodes_port" {
for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? var.k8s_nodes : {}
name = "${var.cluster_name}-k8s-node-${each.key}"
network_id = var.use_existing_network ? data.openstack_networking_network_v2.k8s_network[0].id : var.network_id
network_id = local.k8s_nodes_settings[each.key].network_id
admin_state_up = "true"
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.worker_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
subnet_id = var.private_subnet_id
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [
@ -755,18 +817,20 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? var.k8s_nodes : {}
name = "${var.cluster_name}-k8s-node-${each.key}"
availability_zone = each.value.az
image_id = var.node_root_volume_size_in_gb == 0 ? local.image_to_use_node : null
image_id = local.k8s_nodes_settings[each.key].use_local_disk ? local.k8s_nodes_settings[each.key].image_id : null
flavor_id = each.value.flavor
key_pair = openstack_compute_keypair_v2.k8s.name
user_data = data.cloudinit_config.cloudinit.rendered
user_data = each.value.cloudinit != null ? templatefile("${path.module}/templates/cloudinit.yaml.tmpl", {
extra_partitions = each.value.cloudinit.extra_partitions
}) : data.cloudinit_config.cloudinit.rendered
dynamic "block_device" {
for_each = var.node_root_volume_size_in_gb > 0 ? [local.image_to_use_node] : []
for_each = !local.k8s_nodes_settings[each.key].use_local_disk ? [local.k8s_nodes_settings[each.key].image_id] : []
content {
uuid = local.image_to_use_node
uuid = block_device.value
source_type = "image"
volume_size = var.node_root_volume_size_in_gb
volume_type = var.node_volume_type
volume_size = local.k8s_nodes_settings[each.key].volume_size
volume_type = local.k8s_nodes_settings[each.key].volume_type
boot_index = 0
destination_type = "volume"
delete_on_termination = true
@ -778,15 +842,15 @@ resource "openstack_compute_instance_v2" "k8s_nodes" {
}
dynamic "scheduler_hints" {
for_each = var.node_server_group_policy != "" ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
for_each = local.k8s_nodes_settings[each.key].server_group
content {
group = openstack_compute_servergroup_v2.k8s_node[0].id
group = scheduler_hints.value
}
}
metadata = {
ssh_user = var.ssh_user
kubespray_groups = "kube_node,k8s_cluster,%{if each.value.floating_ip == false}no_floating,%{endif}${var.supplementary_node_groups},${try(each.value.extra_groups, "")}"
kubespray_groups = "kube_node,k8s_cluster,%{if each.value.floating_ip == false}no_floating,%{endif}${var.supplementary_node_groups}${each.value.extra_groups != null ? ",${each.value.extra_groups}" : ""}"
depends_on = var.network_router_id
use_access_ip = var.use_access_ip
}
@ -804,8 +868,11 @@ resource "openstack_networking_port_v2" "glusterfs_node_no_floating_ip_port" {
port_security_enabled = var.force_null_port_security ? null : var.port_security_enabled
security_group_ids = var.port_security_enabled ? local.gfs_sec_groups : null
no_security_groups = var.port_security_enabled ? null : false
fixed_ip {
subnet_id = var.private_subnet_id
dynamic "fixed_ip" {
for_each = var.private_subnet_id == "" ? [] : [true]
content {
subnet_id = var.private_subnet_id
}
}
depends_on = [

17
contrib/terraform/openstack/modules/compute/templates/cloudinit.yaml
View file

@ -1,17 +0,0 @@
# yamllint disable rule:comments
#cloud-config
## in some cases novnc console access is required
## it requires ssh password to be set
#ssh_pwauth: yes
#chpasswd:
# list: |
# root:secret
# expire: False
## in some cases direct root ssh access via ssh key is required
#disable_root: false
## in some cases additional CA certs are required
#ca-certs:
# trusted: |
# -----BEGIN CERTIFICATE-----

39
contrib/terraform/openstack/modules/compute/templates/cloudinit.yaml.tmpl Normal file
View file

@ -0,0 +1,39 @@
%{~ if length(extra_partitions) > 0 }
#cloud-config
bootcmd:
%{~ for idx, partition in extra_partitions }
- [ cloud-init-per, once, move-second-header, sgdisk, --move-second-header, ${partition.volume_path} ]
- [ cloud-init-per, once, create-part-${idx}, parted, --script, ${partition.volume_path}, 'mkpart extended ext4 ${partition.partition_start} ${partition.partition_end}' ]
- [ cloud-init-per, once, create-fs-part-${idx}, mkfs.ext4, ${partition.partition_path} ]
%{~ endfor }
runcmd:
%{~ for idx, partition in extra_partitions }
- mkdir -p ${partition.mount_path}
- chown nobody:nogroup ${partition.mount_path}
- mount ${partition.partition_path} ${partition.mount_path}
%{~ endfor }
mounts:
%{~ for idx, partition in extra_partitions }
- [ ${partition.partition_path}, ${partition.mount_path} ]
%{~ endfor }
%{~ else ~}
# yamllint disable rule:comments
#cloud-config
## in some cases novnc console access is required
## it requires ssh password to be set
#ssh_pwauth: yes
#chpasswd:
# list: |
# root:secret
# expire: False
## in some cases direct root ssh access via ssh key is required
#disable_root: false
## in some cases additional CA certs are required
#ca-certs:
# trusted: |
# -----BEGIN CERTIFICATE-----
%{~ endif }

43
contrib/terraform/openstack/modules/compute/variables.tf
View file

@ -116,9 +116,48 @@ variable "k8s_allowed_egress_ips" {
type = list
}
variable "k8s_masters" {}
variable "k8s_masters" {
type = map(object({
az = string
flavor = string
floating_ip = bool
etcd = bool
image_id = optional(string)
root_volume_size_in_gb = optional(number)
volume_type = optional(string)
network_id = optional(string)
}))
}
variable "k8s_nodes" {}
variable "k8s_nodes" {
type = map(object({
az = string
flavor = string
floating_ip = bool
extra_groups = optional(string)
image_id = optional(string)
root_volume_size_in_gb = optional(number)
volume_type = optional(string)
network_id = optional(string)
additional_server_groups = optional(list(string))
server_group = optional(string)
cloudinit = optional(object({
extra_partitions = list(object({
volume_path = string
partition_path = string
partition_start = string
partition_end = string
mount_path = string
}))
}))
}))
}
variable "additional_server_groups" {
type = map(object({
policy = string
}))
}
variable "supplementary_master_groups" {
default = ""

3
contrib/terraform/openstack/modules/compute/versions.tf
View file

@ -4,5 +4,6 @@ terraform {
source = "terraform-provider-openstack/openstack"
}
}
required_version = ">= 0.12.26"
experiments = [module_variable_optional_attrs]
required_version = ">= 0.14.0"
}

37
contrib/terraform/openstack/variables.tf
View file

@ -294,10 +294,47 @@ variable "router_internal_port_id" {
variable "k8s_masters" {
default = {}
type = map(object({
az = string
flavor = string
floating_ip = bool
etcd = bool
image_id = optional(string)
root_volume_size_in_gb = optional(number)
volume_type = optional(string)
network_id = optional(string)
}))
}
variable "k8s_nodes" {
default = {}
type = map(object({
az = string
flavor = string
floating_ip = bool
extra_groups = optional(string)
image_id = optional(string)
root_volume_size_in_gb = optional(number)
volume_type = optional(string)
network_id = optional(string)
server_group = optional(string)
cloudinit = optional(object({
extra_partitions = list(object({
volume_path = string
partition_path = string
partition_start = string
partition_end = string
mount_path = string
}))
}))
}))
}
variable "additional_server_groups" {
default = {}
type = map(object({
policy = string
}))
}
variable "extra_sec_groups" {

3
contrib/terraform/openstack/versions.tf
View file

@ -5,5 +5,6 @@ terraform {
version = "~> 1.17"
}
}
required_version = ">= 0.12.26"
experiments = [module_variable_optional_attrs]
required_version = ">= 0.14.0"
}