Proposing fixes for contrib/terraform/vsphere/ #8436 (#8441)

* fixes issues in vSphere Terraform contrib. #8436

* fix formatting

* add variables to the main module and document changes

* add missing newline

contrib/terraform/vsphere/README.md

@@ -105,8 +105,7 @@ ansible-playbook -i inventory.ini ../../cluster.yml -b -v
 * `vsphere_datacenter`: The identifier of vSphere data center
 * `vsphere_compute_cluster`: The identifier of vSphere compute cluster
 * `vsphere_datastore`: The identifier of vSphere data store
-* `vsphere_server`: The address of vSphere server
+* `vsphere_server`: This is the vCenter server name or address for vSphere API operations.
-* `vsphere_hostname`: The IP address of vSphere hostname
 * `ssh_public_keys`: List of public SSH keys to install on all machines
 * `template_name`: The name of a base image (the OVF template be defined in vSphere beforehand)
 
@@ -125,5 +124,7 @@ ansible-playbook -i inventory.ini ../../cluster.yml -b -v
 * `worker_cores`: The number of CPU cores for the worker nodes (default: 16)
 * `worker_memory`: The amount of RAM for the worker nodes in MB (default: 8192)
 * `worker_disk_size`: The amount of disk space for the worker nodes in GB (default: 100)
+* `vapp`: Boolean to set the template type to vapp. (Default: false)
+* `interface_name`: Name of the interface to configure. (Default: ens192)
 
 An example variables file can be found `default.tfvars`

contrib/terraform/vsphere/default.tfvars

@@ -34,6 +34,5 @@ vsphere_datacenter      = "i-did-not-read-the-docs"
 vsphere_compute_cluster = "i-did-not-read-the-docs" # e.g. Cluster
 vsphere_datastore       = "i-did-not-read-the-docs" # e.g. ssd-000000
 vsphere_server          = "i-did-not-read-the-docs" # e.g. vsphere.server.com
-vsphere_hostname        = "i-did-not-read-the-docs" # e.g. 192.168.0.2
 
 template_name = "i-did-not-read-the-docs" # e.g. ubuntu-bionic-18.04-cloudimg

contrib/terraform/vsphere/main.tf

@@ -23,11 +23,6 @@ data "vsphere_network" "network" {
   datacenter_id = data.vsphere_datacenter.dc.id
 }
 
-data "vsphere_host" "host" {
-  name          = var.vsphere_hostname
-  datacenter_id = data.vsphere_datacenter.dc.id
-}
-
 data "vsphere_virtual_machine" "template" {
   name          = var.template_name
   datacenter_id = data.vsphere_datacenter.dc.id
@@ -40,7 +35,7 @@ data "vsphere_compute_cluster" "compute_cluster" {
 
 resource "vsphere_resource_pool" "pool" {
   name                    = "${var.prefix}-cluster-pool"
-  parent_resource_pool_id = data.vsphere_host.host.resource_pool_id
+  parent_resource_pool_id = data.vsphere_compute_cluster.compute_cluster.resource_pool_id
 }
 
 module "kubernetes" {
@@ -74,11 +69,13 @@ module "kubernetes" {
   scsi_type             = data.vsphere_virtual_machine.template.scsi_type
   network_id            = data.vsphere_network.network.id
   adapter_type          = data.vsphere_virtual_machine.template.network_interface_types[0]
+  interface_name        = var.interface_name
   firmware              = var.firmware
   hardware_version      = var.hardware_version
   disk_thin_provisioned = data.vsphere_virtual_machine.template.disks.0.thin_provisioned
 
   template_id = data.vsphere_virtual_machine.template.id
+  vapp        = var.vapp
 
   ssh_public_keys = var.ssh_public_keys
 }
@@ -87,30 +84,17 @@ module "kubernetes" {
 # Generate ansible inventory
 #
 
-data "template_file" "inventory" {
+resource "local_file" "inventory" {
-  template = file("${path.module}/templates/inventory.tpl")
+  content = templatefile("${path.module}/templates/inventory.tpl", {
-
-  vars = {
     connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s etcd_member_name=etcd%d",
       keys(module.kubernetes.master_ip),
       values(module.kubernetes.master_ip),
-    range(1, length(module.kubernetes.master_ip) + 1)))
+    range(1, length(module.kubernetes.master_ip) + 1))),
     connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s",
       keys(module.kubernetes.worker_ip),
-    values(module.kubernetes.worker_ip)))
+    values(module.kubernetes.worker_ip))),
-    list_master = join("\n", formatlist("%s",
+    list_master = join("\n", formatlist("%s", keys(module.kubernetes.master_ip))),
-    keys(module.kubernetes.master_ip)))
+    list_worker = join("\n", formatlist("%s", keys(module.kubernetes.worker_ip)))
-    list_worker = join("\n", formatlist("%s",
+  })
-    keys(module.kubernetes.worker_ip)))
+  filename = var.inventory_file
-  }
-}
-
-resource "null_resource" "inventories" {
-  provisioner "local-exec" {
-    command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}"
-  }
-
-  triggers = {
-    template = data.template_file.inventory.rendered
-  }
 }

contrib/terraform/vsphere/modules/kubernetes-cluster/main.tf

@@ -46,14 +46,30 @@ resource "vsphere_virtual_machine" "worker" {
     client_device = true
   }
 
-  vapp {
+  dynamic "vapp" {
+    for_each = var.vapp ? [1] : []
+
+    content {
       properties = {
-      "user-data" = base64encode(templatefile("${path.module}/templates/cloud-init.tmpl", { ip = each.value.ip,
+        "user-data" = base64encode(templatefile("${path.module}/templates/vapp-cloud-init.tpl", { ssh_public_keys = var.ssh_public_keys }))
+      }
+    }
+  }
+
+  extra_config = {
+    "isolation.tools.copy.disable"         = "FALSE"
+    "isolation.tools.paste.disable"        = "FALSE"
+    "isolation.tools.setGUIOptions.enable" = "TRUE"
+    "guestinfo.userdata"                   = base64encode(templatefile("${path.module}/templates/cloud-init.tpl", { ssh_public_keys = var.ssh_public_keys }))
+    "guestinfo.userdata.encoding"          = "base64"
+    "guestinfo.metadata" = base64encode(templatefile("${path.module}/templates/metadata.tpl", { hostname = "${var.prefix}-${each.key}",
+      interface_name = var.interface_name
+      ip             = each.value.ip,
       netmask        = each.value.netmask,
       gw             = var.gateway,
       dns            = var.dns_primary,
     ssh_public_keys = var.ssh_public_keys }))
-    }
+    "guestinfo.metadata.encoding" = "base64"
   }
 }
 
@@ -105,13 +121,29 @@ resource "vsphere_virtual_machine" "master" {
     client_device = true
   }
 
-  vapp {
+  dynamic "vapp" {
+    for_each = var.vapp ? [1] : []
+
+    content {
       properties = {
-      "user-data" = base64encode(templatefile("${path.module}/templates/cloud-init.tmpl", { ip = each.value.ip,
+        "user-data" = base64encode(templatefile("${path.module}/templates/vapp-cloud-init.tpl", { ssh_public_keys = var.ssh_public_keys }))
+      }
+    }
+  }
+
+  extra_config = {
+    "isolation.tools.copy.disable"         = "FALSE"
+    "isolation.tools.paste.disable"        = "FALSE"
+    "isolation.tools.setGUIOptions.enable" = "TRUE"
+    "guestinfo.userdata"                   = base64encode(templatefile("${path.module}/templates/cloud-init.tpl", { ssh_public_keys = var.ssh_public_keys }))
+    "guestinfo.userdata.encoding"          = "base64"
+    "guestinfo.metadata" = base64encode(templatefile("${path.module}/templates/metadata.tpl", { hostname = "${var.prefix}-${each.key}",
+      interface_name = var.interface_name
+      ip             = each.value.ip,
       netmask        = each.value.netmask,
       gw             = var.gateway,
       dns            = var.dns_primary,
     ssh_public_keys = var.ssh_public_keys }))
-    }
+    "guestinfo.metadata.encoding" = "base64"
   }
 }

contrib/terraform/vsphere/modules/kubernetes-cluster/output.tf

@@ -1,7 +1,7 @@
 output "master_ip" {
   value = {
     for name, machine in var.machines :
-    name => machine.ip
+    "${var.prefix}-${name}" => machine.ip
     if machine.node_type == "master"
   }
 }
@@ -9,8 +9,7 @@ output "master_ip" {
 output "worker_ip" {
   value = {
     for name, machine in var.machines :
-    name => machine.ip
+     "${var.prefix}-${name}" => machine.ip
     if machine.node_type == "worker"
   }
 }
-

contrib/terraform/vsphere/modules/kubernetes-cluster/templates/cloud-init.tpl

@@ -0,0 +1,6 @@
+#cloud-config
+
+ssh_authorized_keys:
+%{ for ssh_public_key in ssh_public_keys ~}
+  - ${ssh_public_key}
+%{ endfor ~}

contrib/terraform/vsphere/modules/kubernetes-cluster/templates/metadata.tpl

@@ -0,0 +1,14 @@
+instance-id: ${hostname}
+local-hostname: ${hostname}
+network:
+  version: 2
+  ethernets:
+    ${interface_name}:
+      match:
+        name: ${interface_name}
+      dhcp4: false
+      addresses:
+        - ${ip}/${netmask}
+      gateway4: ${gw}
+      nameservers:
+        addresses: [${dns}]

contrib/terraform/vsphere/modules/kubernetes-cluster/templates/vapp-cloud-init.tpl

@@ -6,23 +6,12 @@ ssh_authorized_keys:
 %{ endfor ~}
 
 write_files:
-  - path: /etc/netplan/20-internal-network.yaml
-    content: |
-      network:
-        version: 2
-        ethernets:
-          "lo:0":
-            match:
-              name: lo
-            dhcp4: false
-            addresses:
-            - 172.17.0.100/32
   - path: /etc/netplan/10-user-network.yaml
-    content: |
+    content: |.
       network:
         version: 2
         ethernets:
-          ens192:
+          ${interface_name}:
             dhcp4: false #true to use dhcp
             addresses:
             - ${ip}/${netmask}

contrib/terraform/vsphere/modules/kubernetes-cluster/variables.tf

@@ -18,9 +18,13 @@ variable "datastore_id" {}
 variable "guest_id" {}
 variable "scsi_type" {}
 variable "network_id" {}
+variable "interface_name" {}
 variable "adapter_type" {}
 variable "disk_thin_provisioned" {}
 variable "template_id" {}
+variable "vapp" {
+  type = bool
+}
 variable "firmware" {}
 variable "folder" {}
 variable "ssh_public_keys" {

contrib/terraform/vsphere/sample-inventory/cluster.tfvars

@@ -29,6 +29,5 @@ vsphere_datacenter      = "i-did-not-read-the-docs"
 vsphere_compute_cluster = "i-did-not-read-the-docs" # e.g. Cluster
 vsphere_datastore       = "i-did-not-read-the-docs" # e.g. ssd-000000
 vsphere_server          = "i-did-not-read-the-docs" # e.g. vsphere.server.com
-vsphere_hostname        = "i-did-not-read-the-docs" # e.g. 192.168.0.2
 
 template_name = "i-did-not-read-the-docs" # e.g. ubuntu-bionic-18.04-cloudimg

contrib/terraform/vsphere/variables.tf

@@ -27,8 +27,6 @@ variable "vsphere_password" {}
 
 variable "vsphere_server" {}
 
-variable "vsphere_hostname" {}
-
 variable "ssh_public_keys" {
   description = "List of public SSH keys which are injected into the VMs."
   type        = list(string)
@@ -37,6 +35,13 @@ variable "ssh_public_keys" {
 variable "template_name" {}
 
 # Optional variables (ones where reasonable defaults exist)
+variable "vapp" {
+  default = false
+}
+
+variable "interface_name" {
+  default = "ens192"
+}
 
 variable "folder" {
   default = ""