Merge branch 'master' into standalone_kubelet

.gitignore

@@ -3,3 +3,5 @@
 inventory/vagrant_ansible_inventory
 temp
 .idea
+*.tfstate
+*.tfstate.backup

contrib/azurerm/.gitignore

@@ -0,0 +1,2 @@
+.generated
+/inventory

contrib/azurerm/README.md

@@ -0,0 +1,64 @@
+# Kubernetes on Azure with Azure Resource Group Templates
+
+Provision the base infrastructure for a Kubernetes cluster by using [Azure Resource Group Templates](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-authoring-templates)
+
+## Status
+
+This will provision the base infrastructure (vnet, vms, nics, ips, ...) needed for Kubernetes in Azure into the specified
+Resource Group. It will not install Kubernetes itself, this has to be done in a later step by yourself (using kargo of course).
+
+## Requirements
+
+- [Install azure-cli](https://docs.microsoft.com/en-us/azure/xplat-cli-install)
+- [Login with azure-cli](https://docs.microsoft.com/en-us/azure/xplat-cli-connect)
+- Dedicated Resource Group created in the Azure Portal or through azure-cli
+
+## Configuration through group_vars/all
+
+You have to modify at least one variable in group_vars/all, which is the **cluster_name** variable. It must be globally
+unique due to some restrictions in Azure. Most other variables should be self explanatory if you have some basic Kubernetes
+experience.
+
+## Bastion host
+
+You can enable the use of a Bastion Host by changing **use_bastion** in group_vars/all to **true**. The generated
+templates will then include an additional bastion VM which can then be used to connect to the masters and nodes. The option
+also removes all public IPs from all other VMs. 
+
+## Generating and applying
+
+To generate and apply the templates, call:
+
+```shell
+$ ./apply-rg.sh <resource_group_name>
+```
+
+If you change something in the configuration (e.g. number of nodes) later, you can call this again and Azure will
+take care about creating/modifying whatever is needed.
+
+## Clearing a resource group
+
+If you need to delete all resources from a resource group, simply call:
+
+```shell
+$ ./clear-rg.sh <resource_group_name>
+```
+
+**WARNING** this really deletes everything from your resource group, including everything that was later created by you!
+
+
+## Generating an inventory for kargo
+
+After you have applied the templates, you can generate an inventory with this call:
+
+```shell
+$ ./generate-inventory.sh <resource_group_name>
+```
+
+It will create the file ./inventory which can then be used with kargo, e.g.:
+
+```shell
+$ cd kargo-root-dir
+$ ansible-playbook -i contrib/azurerm/inventory -u devops --become -e "@inventory/group_vars/all.yml" cluster.yml
+```
+

contrib/azurerm/apply-rg.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+ansible-playbook generate-templates.yml
+
+azure group deployment create -f ./.generated/network.json -g $AZURE_RESOURCE_GROUP
+azure group deployment create -f ./.generated/storage.json -g $AZURE_RESOURCE_GROUP
+azure group deployment create -f ./.generated/availability-sets.json -g $AZURE_RESOURCE_GROUP
+azure group deployment create -f ./.generated/bastion.json -g $AZURE_RESOURCE_GROUP
+azure group deployment create -f ./.generated/masters.json -g $AZURE_RESOURCE_GROUP
+azure group deployment create -f ./.generated/minions.json -g $AZURE_RESOURCE_GROUP

contrib/azurerm/clear-rg.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+ansible-playbook generate-templates.yml
+
+azure group deployment create -g "$AZURE_RESOURCE_GROUP" -f ./.generated/clear-rg.json -m Complete

contrib/azurerm/generate-inventory.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+AZURE_RESOURCE_GROUP="$1"
+
+if [ "$AZURE_RESOURCE_GROUP" == "" ]; then
+    echo "AZURE_RESOURCE_GROUP is missing"
+    exit 1
+fi
+
+ansible-playbook generate-inventory.yml -e azure_resource_group="$AZURE_RESOURCE_GROUP"

contrib/azurerm/generate-inventory.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - generate-inventory

contrib/azurerm/generate-templates.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  gather_facts: False
+  roles:
+    - generate-templates

contrib/azurerm/group_vars/all

@@ -0,0 +1,26 @@
+
+# Due to some Azure limitations, this name must be globally unique
+cluster_name: example
+
+# Set this to true if you do not want to have public IPs for your masters and minions. This will provision a bastion
+# node that can be used to access the masters and minions
+use_bastion: false
+
+number_of_k8s_masters: 3
+number_of_k8s_nodes: 3
+
+masters_vm_size: Standard_A2
+masters_os_disk_size: 1000
+
+minions_vm_size: Standard_A2
+minions_os_disk_size: 1000
+
+admin_username: devops
+admin_password: changeme
+ssh_public_key: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLRzcxbsFDdEibiyXCSdIFh7bKbXso1NqlKjEyPTptf3aBXHEhVil0lJRjGpTlpfTy7PHvXFbXIOCdv9tOmeH1uxWDDeZawgPFV6VSZ1QneCL+8bxzhjiCn8133wBSPZkN8rbFKd9eEUUBfx8ipCblYblF9FcidylwtMt5TeEmXk8yRVkPiCuEYuDplhc2H0f4PsK3pFb5aDVdaDT3VeIypnOQZZoUxHWqm6ThyHrzLJd3SrZf+RROFWW1uInIDf/SZlXojczUYoffxgT1lERfOJCHJXsqbZWugbxQBwqsVsX59+KPxFFo6nV88h3UQr63wbFx52/MXkX4WrCkAHzN ablock-vwfs@dell-lappy"
+
+# Azure CIDRs
+azure_vnet_cidr: 10.0.0.0/8
+azure_admin_cidr: 10.241.2.0/24
+azure_masters_cidr: 10.0.4.0/24
+azure_minions_cidr: 10.240.0.0/16

contrib/azurerm/roles/generate-inventory/tasks/main.yml

@@ -0,0 +1,11 @@
+---
+
+- name: Query Azure VMs
+  command: azure vm list-ip-address --json {{ azure_resource_group }}
+  register: vm_list_cmd
+
+- set_fact:
+    vm_list: "{{ vm_list_cmd.stdout }}"
+
+- name: Generate inventory
+  template: src=inventory.j2 dest="{{playbook_dir}}/inventory"

contrib/azurerm/roles/generate-inventory/templates/inventory.j2

@@ -0,0 +1,33 @@
+
+{% for vm in vm_list %}
+{% if not use_bastion or vm.name == 'bastion' %}
+{{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].publicIPAddress.expanded.ipAddress }} ip={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }}
+{% else %}
+{{ vm.name }} ansible_ssh_host={{ vm.networkProfile.networkInterfaces[0].expanded.ipConfigurations[0].privateIPAddress }}
+{% endif %}
+{% endfor %}
+
+[kube-master]
+{% for vm in vm_list %}
+{% if 'kube-master' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[etcd]
+{% for vm in vm_list %}
+{% if 'etcd' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[kube-node]
+{% for vm in vm_list %}
+{% if 'kube-node' in vm.tags.roles %}
+{{ vm.name }}
+{% endif %}
+{% endfor %}
+
+[k8s-cluster:children]
+kube-node
+kube-master

contrib/azurerm/roles/generate-templates/defaults/main.yml

@@ -0,0 +1,37 @@
+apiVersion: "2015-06-15"
+
+virtualNetworkName: "KubVNET"
+
+subnetAdminName: "ad-subnet"
+subnetMastersName: "master-subnet"
+subnetMinionsName: "minion-subnet"
+
+routeTableName: "routetable"
+securityGroupName: "secgroup"
+
+nameSuffix: "{{cluster_name}}"
+
+availabilitySetMasters: "master-avs"
+availabilitySetMinions: "minion-avs"
+
+faultDomainCount: 3
+updateDomainCount: 10
+
+bastionVmSize: Standard_A0
+bastionVMName: bastion
+bastionIPAddressName: bastion-pubip
+
+disablePasswordAuthentication: true
+
+sshKeyPath: "/home/{{admin_username}}/.ssh/authorized_keys"
+
+imageReference:
+  publisher: "OpenLogic"
+  offer: "CentOS"
+  sku: "7.2"
+  version: "latest"
+imageReferenceJson: "{{imageReference|to_json}}"
+
+storageAccountName: "sa{{nameSuffix | replace('-', '')}}"
+storageAccountType: "Standard_LRS"
+

contrib/azurerm/roles/generate-templates/tasks/main.yml

@@ -0,0 +1,14 @@
+- set_fact:
+    base_dir: "{{playbook_dir}}/.generated/"
+
+- file: path={{base_dir}} state=directory recurse=true
+
+- template: src={{item}} dest="{{base_dir}}/{{item}}"
+  with_items:
+    - network.json
+    - storage.json
+    - availability-sets.json
+    - bastion.json
+    - masters.json
+    - minions.json
+    - clear-rg.json

contrib/azurerm/roles/generate-templates/templates/availability-sets.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+  },
+  "resources": [
+    {
+      "type": "Microsoft.Compute/availabilitySets",
+      "name": "{{availabilitySetMasters}}",
+      "apiVersion": "{{apiVersion}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "PlatformFaultDomainCount": "{{faultDomainCount}}",
+        "PlatformUpdateDomainCount": "{{updateDomainCount}}"
+      }
+    },
+    {
+      "type": "Microsoft.Compute/availabilitySets",
+      "name": "{{availabilitySetMinions}}",
+      "apiVersion": "{{apiVersion}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "PlatformFaultDomainCount": "{{faultDomainCount}}",
+        "PlatformUpdateDomainCount": "{{updateDomainCount}}"
+      }
+    }
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/bastion.json

@@ -0,0 +1,99 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+    "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
+    "subnetAdminRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetAdminName}}')]"
+  },
+  "resources": [
+    {% if use_bastion %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "{{bastionIPAddressName}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "Static"
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkInterfaces",
+      "name": "{{bastionVMName}}-nic",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/publicIPAddresses/', '{{bastionIPAddressName}}')]"
+      ],
+      "properties": {
+        "ipConfigurations": [
+          {
+            "name": "BastionIpConfig",
+            "properties": {
+              "privateIPAllocationMethod": "Dynamic",
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', '{{bastionIPAddressName}}')]"
+              },
+              "subnet": {
+                "id": "[variables('subnetAdminRef')]"
+              }
+            }
+          }
+        ]
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Compute/virtualMachines",
+      "name": "{{bastionVMName}}",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/networkInterfaces/', '{{bastionVMName}}-nic')]"
+      ],
+      "tags": {
+        "roles": "bastion"
+      },
+      "properties": {
+        "hardwareProfile": {
+          "vmSize": "{{bastionVmSize}}"
+        },
+        "osProfile": {
+          "computerName": "{{bastionVMName}}",
+          "adminUsername": "{{admin_username}}",
+          "adminPassword": "{{admin_password}}",
+          "linuxConfiguration": {
+            "disablePasswordAuthentication": "true",
+            "ssh": {
+              "publicKeys": [
+                {
+                  "path": "{{sshKeyPath}}",
+                  "keyData": "{{ssh_public_key}}"
+                }
+              ]
+            }
+          }
+        },
+        "storageProfile": {
+          "imageReference": {{imageReferenceJson}},
+          "osDisk": {
+            "name": "osdisk",
+            "vhd": {
+              "uri": "[concat('http://', '{{storageAccountName}}', '.blob.core.windows.net/vhds/', '{{bastionVMName}}', '-osdisk.vhd')]"
+            },
+            "caching": "ReadWrite",
+            "createOption": "FromImage"
+          }
+        },
+        "networkProfile": {
+          "networkInterfaces": [
+            {
+              "id": "[resourceId('Microsoft.Network/networkInterfaces', '{{bastionVMName}}-nic')]"
+            }
+          ]
+        }
+      }
+    }
+    {% endif %}
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/clear-rg.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {},
+  "variables": {},
+  "resources": [],
+  "outputs": {}
+}

contrib/azurerm/roles/generate-templates/templates/masters.json

@@ -0,0 +1,196 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+    "lbDomainName": "{{nameSuffix}}-api",
+    "lbPublicIPAddressName": "kubernetes-api-pubip",
+    "lbPublicIPAddressType": "Static",
+    "lbPublicIPAddressID": "[resourceId('Microsoft.Network/publicIPAddresses',variables('lbPublicIPAddressName'))]",
+    "lbName": "kubernetes-api",
+    "lbID": "[resourceId('Microsoft.Network/loadBalancers',variables('lbName'))]",
+
+    "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
+    "kubeMastersSubnetRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetMastersName}}')]"
+  },
+  "resources": [
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "[variables('lbPublicIPAddressName')]",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "[variables('lbPublicIPAddressType')]",
+        "dnsSettings": {
+          "domainNameLabel": "[variables('lbDomainName')]"
+        }
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "name": "[variables('lbName')]",
+      "type": "Microsoft.Network/loadBalancers",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/publicIPAddresses/', variables('lbPublicIPAddressName'))]"
+      ],
+      "properties": {
+        "frontendIPConfigurations": [
+          {
+            "name": "kube-api-frontend",
+            "properties": {
+              "publicIPAddress": {
+                "id": "[variables('lbPublicIPAddressID')]"
+              }
+            }
+          }
+        ],
+        "backendAddressPools": [
+          {
+            "name": "kube-api-backend"
+          }
+        ],
+        "loadBalancingRules": [
+          {
+            "name": "kube-api",
+            "properties": {
+              "frontendIPConfiguration": {
+                "id": "[concat(variables('lbID'), '/frontendIPConfigurations/kube-api-frontend')]"
+              },
+              "backendAddressPool": {
+                "id": "[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]"
+              },
+              "protocol": "tcp",
+              "frontendPort": 443,
+              "backendPort": 443,
+              "enableFloatingIP": false,
+              "idleTimeoutInMinutes": 5,
+              "probe": {
+                "id": "[concat(variables('lbID'), '/probes/kube-api')]"
+              }
+            }
+          }
+        ],
+        "probes": [
+          {
+            "name": "kube-api",
+            "properties": {
+              "protocol": "tcp",
+              "port": 443,
+              "intervalInSeconds": 5,
+              "numberOfProbes": 2
+            }
+          }
+        ]
+      }
+    },
+    {% for i in range(number_of_k8s_masters) %}
+    {% if not use_bastion %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "master-{{i}}-pubip",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "Static"
+      }
+    },
+    {% endif %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkInterfaces",
+      "name": "master-{{i}}-nic",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        {% if not use_bastion %}
+        "[concat('Microsoft.Network/publicIPAddresses/', 'master-{{i}}-pubip')]",
+        {% endif %}
+        "[concat('Microsoft.Network/loadBalancers/', variables('lbName'))]"
+      ],
+      "properties": {
+        "ipConfigurations": [
+          {
+            "name": "MastersIpConfig",
+            "properties": {
+              "privateIPAllocationMethod": "Dynamic",
+              {% if not use_bastion %}
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'master-{{i}}-pubip')]"
+              },
+              {% endif %}
+              "subnet": {
+                "id": "[variables('kubeMastersSubnetRef')]"
+              },
+	          "loadBalancerBackendAddressPools": [
+                {
+                  "id": "[concat(variables('lbID'), '/backendAddressPools/kube-api-backend')]"
+                }
+              ]
+            }
+          }
+        ],
+        "networkSecurityGroup": {
+          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]"
+        },
+        "enableIPForwarding": true
+      }
+    },
+    {
+      "type": "Microsoft.Compute/virtualMachines",
+      "name": "master-{{i}}",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/networkInterfaces/', 'master-{{i}}-nic')]"
+      ],
+      "tags": {
+        "roles": "kube-master,etcd"
+      },
+      "apiVersion": "{{apiVersion}}",
+      "properties": {
+        "availabilitySet": {
+          "id": "[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMasters}}')]"
+        },
+        "hardwareProfile": {
+          "vmSize": "{{masters_vm_size}}"
+        },
+        "osProfile": {
+          "computerName": "master-{{i}}",
+          "adminUsername": "{{admin_username}}",
+          "adminPassword": "{{admin_password}}",
+          "linuxConfiguration": {
+            "disablePasswordAuthentication": "{{disablePasswordAuthentication}}",
+            "ssh": {
+              "publicKeys": [
+                {
+                  "path": "{{sshKeyPath}}",
+                  "keyData": "{{ssh_public_key}}"
+                }
+              ]
+            }
+          }
+        },
+        "storageProfile": {
+          "imageReference": {{imageReferenceJson}},
+          "osDisk": {
+            "name": "ma{{nameSuffix}}{{i}}",
+            "vhd": {
+              "uri": "[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/master-{{i}}.vhd')]"
+            },
+            "caching": "ReadWrite",
+            "createOption": "FromImage",
+            "diskSizeGB": "{{masters_os_disk_size}}"
+          }
+        },
+        "networkProfile": {
+          "networkInterfaces": [
+            {
+              "id": "[resourceId('Microsoft.Network/networkInterfaces', 'master-{{i}}-nic')]"
+            }
+          ]
+        }
+      }
+    } {% if not loop.last %},{% endif %}
+    {% endfor %}
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/minions.json

@@ -0,0 +1,113 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+    "vnetID": "[resourceId('Microsoft.Network/virtualNetworks', '{{virtualNetworkName}}')]",
+    "kubeMinionsSubnetRef": "[concat(variables('vnetID'),'/subnets/', '{{subnetMinionsName}}')]"
+  },
+  "resources": [
+    {% for i in range(number_of_k8s_nodes) %}
+    {% if not use_bastion %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/publicIPAddresses",
+      "name": "minion-{{i}}-pubip",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "publicIPAllocationMethod": "Static"
+      }
+    },
+    {% endif %}
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkInterfaces",
+      "name": "minion-{{i}}-nic",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        {% if not use_bastion %}
+        "[concat('Microsoft.Network/publicIPAddresses/', 'minion-{{i}}-pubip')]"
+        {% endif %}
+      ],
+      "properties": {
+        "ipConfigurations": [
+          {
+            "name": "MinionsIpConfig",
+            "properties": {
+              "privateIPAllocationMethod": "Dynamic",
+              {% if not use_bastion %}
+              "publicIPAddress": {
+                "id": "[resourceId('Microsoft.Network/publicIPAddresses', 'minion-{{i}}-pubip')]"
+              },
+              {% endif %}
+              "subnet": {
+                "id": "[variables('kubeMinionsSubnetRef')]"
+              }
+            }
+          }
+        ],
+        "networkSecurityGroup": {
+          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', '{{securityGroupName}}')]"
+        },
+        "enableIPForwarding": true
+      }
+    },
+    {
+      "type": "Microsoft.Compute/virtualMachines",
+      "name": "minion-{{i}}",
+      "location": "[resourceGroup().location]",
+      "dependsOn": [
+        "[concat('Microsoft.Network/networkInterfaces/', 'minion-{{i}}-nic')]"
+      ],
+      "tags": {
+        "roles": "kube-node"
+      },
+      "apiVersion": "{{apiVersion}}",
+      "properties": {
+        "availabilitySet": {
+          "id": "[resourceId('Microsoft.Compute/availabilitySets', '{{availabilitySetMinions}}')]"
+        },
+        "hardwareProfile": {
+          "vmSize": "{{minions_vm_size}}"
+        },
+        "osProfile": {
+          "computerName": "minion-{{i}}",
+          "adminUsername": "{{admin_username}}",
+          "adminPassword": "{{admin_password}}",
+          "linuxConfiguration": {
+            "disablePasswordAuthentication": "{{disablePasswordAuthentication}}",
+            "ssh": {
+              "publicKeys": [
+                {
+                  "path": "{{sshKeyPath}}",
+                  "keyData": "{{ssh_public_key}}"
+                }
+              ]
+            }
+          }
+        },
+        "storageProfile": {
+          "imageReference": {{imageReferenceJson}},
+          "osDisk": {
+            "name": "mi{{nameSuffix}}{{i}}",
+            "vhd": {
+              "uri": "[concat('http://','{{storageAccountName}}','.blob.core.windows.net/vhds/minion-{{i}}.vhd')]"
+            },
+            "caching": "ReadWrite",
+            "createOption": "FromImage",
+            "diskSizeGB": "{{minions_os_disk_size}}"
+          }
+        },
+        "networkProfile": {
+          "networkInterfaces": [
+            {
+              "id": "[resourceId('Microsoft.Network/networkInterfaces', 'minion-{{i}}-nic')]"
+            }
+          ]
+        }
+      }
+    } {% if not loop.last %},{% endif %}
+    {% endfor %}
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/network.json

@@ -0,0 +1,109 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+  },
+  "resources": [
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/routeTables",
+      "name": "{{routeTableName}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "routes": [
+        ]
+      }
+    },
+    {
+      "type": "Microsoft.Network/virtualNetworks",
+      "name": "{{virtualNetworkName}}",
+      "location": "[resourceGroup().location]",
+      "apiVersion": "{{apiVersion}}",
+      "dependsOn": [
+        "[concat('Microsoft.Network/routeTables/', '{{routeTableName}}')]"
+      ],
+      "properties": {
+        "addressSpace": {
+          "addressPrefixes": [
+            "{{azure_vnet_cidr}}"
+          ]
+        },
+        "subnets": [
+          {
+            "name": "{{subnetMastersName}}",
+            "properties": {
+              "addressPrefix": "{{azure_masters_cidr}}",
+              "routeTable": {
+                "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
+              }
+            }
+          },
+          {
+            "name": "{{subnetMinionsName}}",
+            "properties": {
+              "addressPrefix": "{{azure_minions_cidr}}",
+              "routeTable": {
+                "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
+              }
+            }
+          }
+          {% if use_bastion %}
+          ,{
+            "name": "{{subnetAdminName}}",
+            "properties": {
+              "addressPrefix": "{{azure_admin_cidr}}",
+              "routeTable": {
+                "id": "[resourceId('Microsoft.Network/routeTables', '{{routeTableName}}')]"
+              }
+            }
+          }
+          {% endif %}
+        ]
+      }
+    },
+    {
+      "apiVersion": "{{apiVersion}}",
+      "type": "Microsoft.Network/networkSecurityGroups",
+      "name": "{{securityGroupName}}",
+      "location": "[resourceGroup().location]",
+      "properties": {
+          "securityRules": [
+            {% if not use_bastion %}
+            {
+              "name": "ssh",
+              "properties": {
+                "description": "Allow SSH",
+                "protocol": "Tcp",
+                "sourcePortRange": "*",
+                "destinationPortRange": "22",
+                "sourceAddressPrefix": "Internet",
+                "destinationAddressPrefix": "*",
+                "access": "Allow",
+                "priority": 100,
+                "direction": "Inbound"
+              }
+            },
+            {% endif %}
+            {
+              "name": "kube-api",
+              "properties": {
+                "description": "Allow secure kube-api",
+                "protocol": "Tcp",
+                "sourcePortRange": "*",
+                "destinationPortRange": "443",
+                "sourceAddressPrefix": "Internet",
+                "destinationAddressPrefix": "*",
+                "access": "Allow",
+                "priority": 101,
+                "direction": "Inbound"
+              }
+            }
+          ]
+      },
+      "resources": [],
+      "dependsOn": []
+    }
+  ]
+}

contrib/azurerm/roles/generate-templates/templates/storage.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+  },
+  "variables": {
+  },
+  "resources": [
+    {
+      "type": "Microsoft.Storage/storageAccounts",
+      "name": "{{storageAccountName}}",
+      "location": "[resourceGroup().location]",
+      "apiVersion": "{{apiVersion}}",
+      "properties": {
+        "accountType": "{{storageAccountType}}"
+      }
+    }
+  ]
+}

contrib/terraform/openstack/terraform.tfstate

@@ -1,238 +0,0 @@
-{
-    "version": 1,
-    "serial": 17,
-    "modules": [
-        {
-            "path": [
-                "root"
-            ],
-            "outputs": {},
-            "resources": {
-                "openstack_compute_instance_v2.k8s_master.0": {
-                    "type": "openstack_compute_instance_v2",
-                    "depends_on": [
-                        "openstack_compute_keypair_v2.k8s",
-                        "openstack_compute_secgroup_v2.k8s",
-                        "openstack_compute_secgroup_v2.k8s_master",
-                        "openstack_networking_floatingip_v2.k8s_master"
-                    ],
-                    "primary": {
-                        "id": "f4a44f6e-33ff-4e35-b593-34f3dfd80dc9",
-                        "attributes": {
-                            "access_ip_v4": "173.247.105.12",
-                            "access_ip_v6": "",
-                            "flavor_id": "3",
-                            "flavor_name": "m1.medium",
-                            "floating_ip": "173.247.105.12",
-                            "id": "f4a44f6e-33ff-4e35-b593-34f3dfd80dc9",
-                            "image_id": "1525c3f3-1224-4958-bd07-da9feaedf18b",
-                            "image_name": "ubuntu-14.04",
-                            "key_pair": "kubernetes-example",
-                            "metadata.#": "2",
-                            "metadata.kubespray_groups": "etcd,kube-master,kube-node,k8s-cluster",
-                            "metadata.ssh_user": "ubuntu",
-                            "name": "example-k8s-master-1",
-                            "network.#": "1",
-                            "network.0.access_network": "false",
-                            "network.0.fixed_ip_v4": "10.230.7.86",
-                            "network.0.fixed_ip_v6": "",
-                            "network.0.floating_ip": "173.247.105.12",
-                            "network.0.mac": "fa:16:3e:fb:82:1d",
-                            "network.0.name": "internal",
-                            "network.0.port": "",
-                            "network.0.uuid": "ba0fdd03-72b5-41eb-bb67-fef437fd6cb4",
-                            "security_groups.#": "2",
-                            "security_groups.2779334175": "example-k8s",
-                            "security_groups.3772290257": "example-k8s-master",
-                            "volume.#": "0"
-                        }
-                    }
-                },
-                "openstack_compute_instance_v2.k8s_master.1": {
-                    "type": "openstack_compute_instance_v2",
-                    "depends_on": [
-                        "openstack_compute_keypair_v2.k8s",
-                        "openstack_compute_secgroup_v2.k8s",
-                        "openstack_compute_secgroup_v2.k8s_master",
-                        "openstack_networking_floatingip_v2.k8s_master"
-                    ],
-                    "primary": {
-                        "id": "cbb565fe-a3b6-44ff-8f81-8ec29704d11b",
-                        "attributes": {
-                            "access_ip_v4": "173.247.105.70",
-                            "access_ip_v6": "",
-                            "flavor_id": "3",
-                            "flavor_name": "m1.medium",
-                            "floating_ip": "173.247.105.70",
-                            "id": "cbb565fe-a3b6-44ff-8f81-8ec29704d11b",
-                            "image_id": "1525c3f3-1224-4958-bd07-da9feaedf18b",
-                            "image_name": "ubuntu-14.04",
-                            "key_pair": "kubernetes-example",
-                            "metadata.#": "2",
-                            "metadata.kubespray_groups": "etcd,kube-master,kube-node,k8s-cluster",
-                            "metadata.ssh_user": "ubuntu",
-                            "name": "example-k8s-master-2",
-                            "network.#": "1",
-                            "network.0.access_network": "false",
-                            "network.0.fixed_ip_v4": "10.230.7.85",
-                            "network.0.fixed_ip_v6": "",
-                            "network.0.floating_ip": "173.247.105.70",
-                            "network.0.mac": "fa:16:3e:33:98:e6",
-                            "network.0.name": "internal",
-                            "network.0.port": "",
-                            "network.0.uuid": "ba0fdd03-72b5-41eb-bb67-fef437fd6cb4",
-                            "security_groups.#": "2",
-                            "security_groups.2779334175": "example-k8s",
-                            "security_groups.3772290257": "example-k8s-master",
-                            "volume.#": "0"
-                        }
-                    }
-                },
-                "openstack_compute_instance_v2.k8s_node": {
-                    "type": "openstack_compute_instance_v2",
-                    "depends_on": [
-                        "openstack_compute_keypair_v2.k8s",
-                        "openstack_compute_secgroup_v2.k8s",
-                        "openstack_networking_floatingip_v2.k8s_node"
-                    ],
-                    "primary": {
-                        "id": "39deed7e-8307-4b62-b56c-ce2b405a03fa",
-                        "attributes": {
-                            "access_ip_v4": "173.247.105.76",
-                            "access_ip_v6": "",
-                            "flavor_id": "3",
-                            "flavor_name": "m1.medium",
-                            "floating_ip": "173.247.105.76",
-                            "id": "39deed7e-8307-4b62-b56c-ce2b405a03fa",
-                            "image_id": "1525c3f3-1224-4958-bd07-da9feaedf18b",
-                            "image_name": "ubuntu-14.04",
-                            "key_pair": "kubernetes-example",
-                            "metadata.#": "2",
-                            "metadata.kubespray_groups": "kube-node,k8s-cluster",
-                            "metadata.ssh_user": "ubuntu",
-                            "name": "example-k8s-node-1",
-                            "network.#": "1",
-                            "network.0.access_network": "false",
-                            "network.0.fixed_ip_v4": "10.230.7.84",
-                            "network.0.fixed_ip_v6": "",
-                            "network.0.floating_ip": "173.247.105.76",
-                            "network.0.mac": "fa:16:3e:53:57:bc",
-                            "network.0.name": "internal",
-                            "network.0.port": "",
-                            "network.0.uuid": "ba0fdd03-72b5-41eb-bb67-fef437fd6cb4",
-                            "security_groups.#": "1",
-                            "security_groups.2779334175": "example-k8s",
-                            "volume.#": "0"
-                        }
-                    }
-                },
-                "openstack_compute_keypair_v2.k8s": {
-                    "type": "openstack_compute_keypair_v2",
-                    "primary": {
-                        "id": "kubernetes-example",
-                        "attributes": {
-                            "id": "kubernetes-example",
-                            "name": "kubernetes-example",
-                            "public_key": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC9nU6RPYCabjLH1LvJfpp9L8r8q5RZ6niS92zD95xpm2b2obVydWe0tCSFdmULBuvT8Q8YQ4qOG2g/oJlsGOsia+4CQjYEUV9CgTH9H5HK3vUOwtO5g2eFnYKSmI/4znHa0WYpQFnQK2kSSeCs2beTlJhc8vjfN/2HHmuny6SxNSbnCk/nZdwamxEONIVdjlm3CSBlq4PChT/D/uUqm/nOm0Zqdk9ZlTBkucsjiOCJeEzg4HioKmIH8ewqsKuS7kMADHPH98JMdBhTKbYbLrxTC/RfiaON58WJpmdOA935TT5Td5aVQZoqe/i/5yFRp5fMG239jtfbM0Igu44TEIib pczarkowski@Pauls-MacBook-Pro.local\n"
-                        }
-                    }
-                },
-                "openstack_compute_secgroup_v2.k8s": {
-                    "type": "openstack_compute_secgroup_v2",
-                    "primary": {
-                        "id": "418394e2-b4be-4953-b7a3-b309bf28fbdb",
-                        "attributes": {
-                            "description": "example - Kubernetes",
-                            "id": "418394e2-b4be-4953-b7a3-b309bf28fbdb",
-                            "name": "example-k8s",
-                            "rule.#": "5",
-                            "rule.112275015.cidr": "",
-                            "rule.112275015.from_group_id": "",
-                            "rule.112275015.from_port": "1",
-                            "rule.112275015.id": "597170c9-b35a-45c0-8717-652a342f3fd6",
-                            "rule.112275015.ip_protocol": "tcp",
-                            "rule.112275015.self": "true",
-                            "rule.112275015.to_port": "65535",
-                            "rule.2180185248.cidr": "0.0.0.0/0",
-                            "rule.2180185248.from_group_id": "",
-                            "rule.2180185248.from_port": "-1",
-                            "rule.2180185248.id": "ffdcdd5e-f18b-4537-b502-8849affdfed9",
-                            "rule.2180185248.ip_protocol": "icmp",
-                            "rule.2180185248.self": "false",
-                            "rule.2180185248.to_port": "-1",
-                            "rule.3267409695.cidr": "",
-                            "rule.3267409695.from_group_id": "",
-                            "rule.3267409695.from_port": "-1",
-                            "rule.3267409695.id": "4f91d9ca-940c-4f4d-9ce1-024cbd7d9c54",
-                            "rule.3267409695.ip_protocol": "icmp",
-                            "rule.3267409695.self": "true",
-                            "rule.3267409695.to_port": "-1",
-                            "rule.635693822.cidr": "",
-                            "rule.635693822.from_group_id": "",
-                            "rule.635693822.from_port": "1",
-                            "rule.635693822.id": "c6816e5b-a1a4-4071-acce-d09b92d14d49",
-                            "rule.635693822.ip_protocol": "udp",
-                            "rule.635693822.self": "true",
-                            "rule.635693822.to_port": "65535",
-                            "rule.836640770.cidr": "0.0.0.0/0",
-                            "rule.836640770.from_group_id": "",
-                            "rule.836640770.from_port": "22",
-                            "rule.836640770.id": "8845acba-636b-4c23-b9e2-5bff76d9008d",
-                            "rule.836640770.ip_protocol": "tcp",
-                            "rule.836640770.self": "false",
-                            "rule.836640770.to_port": "22"
-                        }
-                    }
-                },
-                "openstack_compute_secgroup_v2.k8s_master": {
-                    "type": "openstack_compute_secgroup_v2",
-                    "primary": {
-                        "id": "c74aed25-6161-46c4-a488-dfc7f49a228e",
-                        "attributes": {
-                            "description": "example - Kubernetes Master",
-                            "id": "c74aed25-6161-46c4-a488-dfc7f49a228e",
-                            "name": "example-k8s-master",
-                            "rule.#": "0"
-                        }
-                    }
-                },
-                "openstack_networking_floatingip_v2.k8s_master.0": {
-                    "type": "openstack_networking_floatingip_v2",
-                    "primary": {
-                        "id": "2a320c67-214d-4631-a840-2de82505ed3f",
-                        "attributes": {
-                            "address": "173.247.105.12",
-                            "id": "2a320c67-214d-4631-a840-2de82505ed3f",
-                            "pool": "external",
-                            "port_id": ""
-                        }
-                    }
-                },
-                "openstack_networking_floatingip_v2.k8s_master.1": {
-                    "type": "openstack_networking_floatingip_v2",
-                    "primary": {
-                        "id": "3adbfc13-e7ae-4bcf-99d3-3ba9db056e1f",
-                        "attributes": {
-                            "address": "173.247.105.70",
-                            "id": "3adbfc13-e7ae-4bcf-99d3-3ba9db056e1f",
-                            "pool": "external",
-                            "port_id": ""
-                        }
-                    }
-                },
-                "openstack_networking_floatingip_v2.k8s_node": {
-                    "type": "openstack_networking_floatingip_v2",
-                    "primary": {
-                        "id": "a3f77aa6-5c3a-4edf-b97e-ee211dfa81e1",
-                        "attributes": {
-                            "address": "173.247.105.76",
-                            "id": "a3f77aa6-5c3a-4edf-b97e-ee211dfa81e1",
-                            "pool": "external",
-                            "port_id": ""
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}

contrib/terraform/openstack/terraform.tfstate.backup

@@ -1,13 +0,0 @@
-{
-    "version": 1,
-    "serial": 16,
-    "modules": [
-        {
-            "path": [
-                "root"
-            ],
-            "outputs": {},
-            "resources": {}
-        }
-    ]
-}

docs/azure.md

@@ -51,4 +51,6 @@ This is the AppId from the last command
 
 azure\_aad\_client\_id musst be set to the AppId, azure\_aad\_client\_secret is your choosen secret.
 
+## Provisioning Azure with Resource Group Templates
 
+You'll find Resource Group Templates and scripts to provision the required infrastructore to Azure in [*contrib/azurerm*](../contrib/azurerm/README.md)

docs/dns-stack.md

@@ -39,15 +39,17 @@ Later, the nameservers will be reconfigured to the DNS service IP that Kargo
 configures for K8s cluster.
 
 Also note, existing records will be purged from the `/etc/resolv.conf`,
-including base/head/cloud-init config files and those that come from dhclient.
+including resolvconf's base/head/cloud-init config files and those that come from dhclient.
 This is required for hostnet pods networking and for [kubelet to not exceed search domains
 limits](https://github.com/kubernetes/kubernetes/issues/9229).
 
-New search, nameserver records and options will be defined from the aforementioned vars:
+Instead, new domain, search, nameserver records and options will be defined from the
-* Via resolvconf's head file, if resolvconf installed.
+aforementioned vars:
-* Via dhclient's DNS update hook.
+* Superseded via dhclient's DNS update hook.
-* Via cloud-init (CoreOS only).
+* Generated via cloud-init (CoreOS only).
-* Statically in the `/etc/resolv.conf`, if none of above is applicable.
+* Statically defined in the `/etc/resolv.conf`, if none of above is applicable.
+* Resolvconf's head/base files are disabled from populating anything into the
+  `/etc/resolv.conf`.
 
 DNS configuration details
 -------------------------

inventory/group_vars/all.yml

@@ -26,6 +26,9 @@ kube_token_dir: "{{ kube_config_dir }}/tokens"
 # This is where to save basic auth file
 kube_users_dir: "{{ kube_config_dir }}/users"
 
+## Change this to use another Kubernetes version, e.g. a current beta release
+kube_version: 1.4.6
+
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
 local_release_dir: "/tmp/releases"
@@ -43,6 +46,11 @@ kube_cert_group: kube-cert
 # Cluster Loglevel configuration
 kube_log_level: 2
 
+# Kubernetes 1.5 added a new flag to the apiserver to disable anonymous auth. In previos versions, anonymous auth was
+# not implemented. As the new flag defaults to true, we have to explicetely disable it. Change this line if you want the
+# 1.5 default behavior. The flag is actually only added if the used kubernetes version is >= 1.5
+kube_api_anonymous_auth: false
+
 # Users to create for basic auth in Kubernetes API via HTTP
 kube_api_pwd: "changeme"
 kube_users:

roles/download/defaults/main.yml

@@ -15,8 +15,6 @@ download_compress: 9
 download_localhost: False
 
 # Versions
-kube_version: v1.4.6
-
 etcd_version: v3.0.6
 #TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -48,6 +48,9 @@ spec:
     - --cloud-config={{ kube_config_dir }}/cloud_config
 {% elif cloud_provider is defined and cloud_provider == "aws" %}
     - --cloud-provider={{ cloud_provider }}
+{% endif %}
+{% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=')  %}
+    - --anonymous-auth={{ kube_api_anonymous_auth }}
 {% endif %}
     livenessProbe:
       httpGet:

roles/kubernetes/preinstall/tasks/dhclient-hooks.yml

@@ -1,8 +1,8 @@
 ---
-- name: Configure dhclient to prepend nameservers and supersede search/domain
+- name: Configure dhclient to supersede search/domain/nameservers
   blockinfile:
     block: |-
-      {% for item in [ supersede_domain, supersede_search, prepend_nameserver ] -%}
+      {% for item in [ supersede_domain, supersede_search, supersede_nameserver ] -%}
       {{ item }}
       {% endfor %}
     dest: "{{dhclientconffile}}"

roles/kubernetes/preinstall/tasks/set_resolv_facts.yml

@@ -64,18 +64,13 @@
     supersede_domain:
       supersede domain-name "{{ dns_domain }}";
 
-- name: decide on dns server IP
-  set_fact:
-    dns_server_real: >-
-      {%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%}
-
 - name: pick dnsmasq cluster IP or default resolver
   set_fact:
     dnsmasq_server: |-
       {%- if skip_dnsmasq|bool and not dns_early|bool -%}
         {{ [ skydns_server ] + upstream_dns_servers|default([]) }}
       {%- elif dns_early|bool -%}
-        {{ [ dns_server_real ] + upstream_dns_servers|default([]) }}
+        {{ upstream_dns_servers|default([default_resolver]) }}
       {%- else -%}
         {{ [ dns_server ] }}
       {%- endif -%}
@@ -83,6 +78,6 @@
 - name: generate nameservers to resolvconf
   set_fact:
     nameserverentries:
-      nameserver {{( dnsmasq_server|default([]) + nameservers|default([])) | join(',nameserver ')}}
+      nameserver {{( dnsmasq_server + nameservers|default([])) | join(',nameserver ')}}
-    prepend_nameserver:
+    supersede_nameserver:
-      prepend domain-name-servers {{( dnsmasq_server|default([]) + nameservers|default([])) | join(', ') }};
+      supersede domain-name-servers {{( dnsmasq_server + nameservers|default([])) | join(', ') }};

roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2

@@ -5,7 +5,7 @@
 #
 if [ $reason = "BOUND" ]; then
   if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
-    RESOLV_CONF=$(cat /etc/resolv.conf)
+    RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d')
     OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
 
     printf "%b\n" "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf

roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2

@@ -5,7 +5,7 @@
 #
 zdnsupdate_config() {
   if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
-    RESOLV_CONF=$(cat /etc/resolv.conf)
+    RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d')
     OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
 
     echo -e "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf

roles/reset/tasks/main.yml

@@ -24,12 +24,12 @@
   shell: docker ps -aq | xargs -r docker rm -fv
 
 - name: reset | gather mounted kubelet dirs
-  shell: mount | grep /var/lib/kubelet | awk '{print $3}'
+  shell: mount | grep /var/lib/kubelet | awk '{print $3}' | tac
   register: mounted_dirs
 
 - name: reset | unmount kubelet dirs
   command: umount {{item}}
-  with_items: '{{ mounted_dirs.stdout_lines | reverse }}'
+  with_items: '{{ mounted_dirs.stdout_lines }}'
 
 - name: reset | delete some files and directories
   file: path={{ item }} state=absent

roles/uploads/defaults/main.yml

@@ -2,8 +2,6 @@
 local_release_dir: /tmp
 
 # Versions
-kube_version: v1.4.6
-
 etcd_version: v3.0.6
 calico_version: v0.23.0
 calico_cni_version: v1.4.2