From d7f5fdc0c5ab516b33beaf4d159363b1b6733adb Mon Sep 17 00:00:00 2001 From: "Cesarini, Daniele" Date: Wed, 8 Mar 2017 14:42:25 +0000 Subject: [PATCH 01/47] Adding /O=system:masters to admin certificate Issue #1125. Make RBAC authorization plugin work out of the box. "When bootstrapping, superuser credentials should include the system:masters group, for example by creating a client cert with /O=system:masters. This gives those credentials full access to the API and allows an admin to then set up bindings for other users." --- roles/kubernetes/secrets/files/make-ssl.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh index 3cea6d79e..55ea13d1e 100755 --- a/roles/kubernetes/secrets/files/make-ssl.sh +++ b/roles/kubernetes/secrets/files/make-ssl.sh @@ -85,7 +85,7 @@ if [ -n "$MASTERS" ]; then cn="${host%%.*}" # admin key openssl genrsa -out admin-${host}-key.pem 2048 > /dev/null 2>&1 - openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=kube-admin-${cn}" > /dev/null 2>&1 + openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=kube-admin-${cn}/O=system:masters" > /dev/null 2>&1 openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days 3650 > /dev/null 2>&1 done fi From c90578d3adacee230538967233478d000171df42 Mon Sep 17 00:00:00 2001 From: Connz Date: Thu, 9 Mar 2017 11:10:25 +0100 Subject: [PATCH 02/47] Fixed nova command to get available flavors The nova command for getting the flavors is not nova list-flavors but nova flavor-list --- contrib/terraform/openstack/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/terraform/openstack/README.md b/contrib/terraform/openstack/README.md index 9df7abd9f..e98b8068a 100644 --- a/contrib/terraform/openstack/README.md +++ b/contrib/terraform/openstack/README.md @@ -86,7 +86,7 @@ This will provision one VM as master using a floating ip, two additional masters Additionally, now the terraform based installation supports provisioning of a GlusterFS shared file system based on a separate set of VMs, running either a Debian or RedHat based set of VMs. To enable this, you need to add to your `my-terraform-vars.tfvars` the following variables: ``` -# Flavour depends on your openstack installation, you can get available flavours through `nova list-flavors` +# Flavour depends on your openstack installation, you can get available flavours through `nova flavor-list` flavor_gfs_node = "af659280-5b8a-42b5-8865-a703775911da" # This is the name of an image already available in your openstack installation. image_gfs = "Ubuntu 15.10" From a6a90be6ba3922c584da3c82c0901243962d1030 Mon Sep 17 00:00:00 2001 From: Justin Downing Date: Wed, 29 Mar 2017 22:00:52 -0400 Subject: [PATCH 03/47] Update upgrades.md Clarify that the `kube_version` environment variable is needed for the CLI "graceful upgrade". Also add and example to check that the upgrade was successful. --- docs/upgrades.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/upgrades.md b/docs/upgrades.md index c37cad54a..cb431d4c0 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -44,7 +44,15 @@ deployed. ``` git fetch origin git checkout origin/master -ansible-playbook upgrade-cluster.yml -b -i inventory/inventory.cfg +ansible-playbook upgrade-cluster.yml -b -i inventory/inventory.cfg -e kube_version=v1.6.0 +``` + +After a successul upgrade, the Server Version should be updated: + +``` +$ kubectl version +Client Version: version.Info{Major:"1", Minor:"6", GitVersion:"v1.6.0", GitCommit:"fff5156092b56e6bd60fff75aad4dc9de6b6ef37", GitTreeState:"clean", BuildDate:"2017-03-28T19:15:41Z", GoVersion:"go1.8", Compiler:"gc", Platform:"darwin/amd64"} +Server Version: version.Info{Major:"1", Minor:"6", GitVersion:"v1.6.0+coreos.0", GitCommit:"8031716957d697332f9234ddf85febb07ac6c3e3", GitTreeState:"clean", BuildDate:"2017-03-29T04:33:09Z", GoVersion:"go1.7.5", Compiler:"gc", Platform:"linux/amd64"} ``` #### Upgrade order From 4ef5f03d7b1398e1cca19082b3ca2c6f721f9722 Mon Sep 17 00:00:00 2001 From: Brad Beam Date: Tue, 11 Apr 2017 20:52:04 -0500 Subject: [PATCH 04/47] Updating calico versions --- roles/download/defaults/main.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index e64845460..2f5c8358a 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -22,8 +22,8 @@ kube_version: v1.6.1 etcd_version: v3.0.17 #TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download -calico_version: "v1.1.0-rc8" -calico_cni_version: "v1.5.6" +calico_version: "v1.1.0" +calico_cni_version: "v1.6.2" calico_policy_version: "v0.5.4" weave_version: 1.8.2 flannel_version: v0.6.2 @@ -50,10 +50,8 @@ calico_cni_image_repo: "calico/cni" calico_cni_image_tag: "{{ calico_cni_version }}" calico_policy_image_repo: "calico/kube-policy-controller" calico_policy_image_tag: "{{ calico_policy_version }}" -# TODO(adidenko): switch to "calico/routereflector" when -# https://github.com/projectcalico/calico-bird/pull/27 is merged -calico_rr_image_repo: "quay.io/l23network/routereflector" -calico_rr_image_tag: "v0.1" +calico_rr_image_repo: "quay.io/calico/routereflector" +calico_rr_image_tag: "v0.3.0" exechealthz_version: 1.1 exechealthz_image_repo: "gcr.io/google_containers/exechealthz-amd64" exechealthz_image_tag: "{{ exechealthz_version }}" From f121058f0bed8c763684db4515ba6bfdf8537dda Mon Sep 17 00:00:00 2001 From: Vincent Schwarzer Date: Wed, 12 Apr 2017 15:11:39 +0200 Subject: [PATCH 05/47] Fixes for AWS Terraform Deployment and Updated Readme --- contrib/terraform/aws/README.md | 34 +++++++++++++++---- .../terraform/aws/create-infrastructure.tf | 1 + contrib/terraform/aws/output.tf | 4 +++ contrib/terraform/aws/templates/inventory.tpl | 1 + .../terraform/aws/terraform.tfvars.example | 6 ++-- contrib/terraform/aws/variables.tf | 4 +++ 6 files changed, 41 insertions(+), 9 deletions(-) diff --git a/contrib/terraform/aws/README.md b/contrib/terraform/aws/README.md index 03bc4e23e..de858b2a9 100644 --- a/contrib/terraform/aws/README.md +++ b/contrib/terraform/aws/README.md @@ -14,20 +14,42 @@ This project will create: **How to Use:** -- Export the variables for your AWS credentials or edit credentials.tfvars: +- Export the variables for your AWS credentials or edit `credentials.tfvars`: ``` -export aws_access_key="xxx" -export aws_secret_key="yyy" -export aws_ssh_key_name="zzz" +export AWS_ACCESS_KEY_ID="www" +export AWS_SECRET_ACCESS_KEY ="xxx" +export AWS_SSH_KEY_NAME="yyy" +export AWS_DEFAULT_REGION="zzz" ``` +- Rename `contrib/terraform/aws/terraform.tfvars.example` to `terraform.tfvars` -- Update contrib/terraform/aws/terraform.tfvars with your data +- Update `contrib/terraform/aws/terraform.tfvars` with your data + - Allocate new AWS Elastic IPs: Depending on # of Availability Zones used (2 for each AZ) + - Create an AWS EC2 SSH Key -- Run with `terraform apply -var-file="credentials.tfvars"` or `terraform apply` depending if you exported your AWS credentials + +- Run with `terraform apply --var-file="credentials.tfvars"` or `terraform apply` depending if you exported your AWS credentials + +- Terraform automatically creates an Ansible Inventory file called `hosts` with the created infrastructure in the directory `inventory` - Once the infrastructure is created, you can run the kargo playbooks and supply inventory/hosts with the `-i` flag. +**Troubleshooting** + +***Remaining AWS IAM Instance Profile***: + +If the cluster was destroyed without using Terraform it is possible that +the AWS IAM Instance Profiles still remain. To delete them you can use +the `AWS CLI` with the following command: +``` +aws iam delete-instance-profile --region --instance-profile-name +``` + +***Ansible Inventory doesnt get created:*** + +It could happen that Terraform doesnt create an Ansible Inventory file automatically. If this is the case copy the output after `inventory=` and create a file named `hosts`in the directory `inventory` and paste the inventory into the file. + **Architecture** Pictured is an AWS Infrastructure created with this Terraform project distributed over two Availability Zones. diff --git a/contrib/terraform/aws/create-infrastructure.tf b/contrib/terraform/aws/create-infrastructure.tf index 14da95492..781edea86 100644 --- a/contrib/terraform/aws/create-infrastructure.tf +++ b/contrib/terraform/aws/create-infrastructure.tf @@ -173,6 +173,7 @@ data "template_file" "inventory" { list_etcd = "${join("\n",aws_instance.k8s-etcd.*.tags.Name)}" elb_api_fqdn = "apiserver_loadbalancer_domain_name=\"${module.aws-elb.aws_elb_api_fqdn}\"" elb_api_port = "loadbalancer_apiserver.port=${var.aws_elb_api_port}" + kube_insecure_apiserver_address = "kube_apiserver_insecure_bind_address: ${var.kube_insecure_apiserver_address}" } } diff --git a/contrib/terraform/aws/output.tf b/contrib/terraform/aws/output.tf index fbe74f262..fabc0d218 100644 --- a/contrib/terraform/aws/output.tf +++ b/contrib/terraform/aws/output.tf @@ -18,3 +18,7 @@ output "etcd" { output "aws_elb_api_fqdn" { value = "${module.aws-elb.aws_elb_api_fqdn}:${var.aws_elb_api_port}" } + +output "inventory" { + value = "${data.template_file.inventory.rendered}" +} diff --git a/contrib/terraform/aws/templates/inventory.tpl b/contrib/terraform/aws/templates/inventory.tpl index 4140aa768..8d5afd1cf 100644 --- a/contrib/terraform/aws/templates/inventory.tpl +++ b/contrib/terraform/aws/templates/inventory.tpl @@ -25,3 +25,4 @@ kube-master [k8s-cluster:vars] ${elb_api_fqdn} ${elb_api_port} +${kube_insecure_apiserver_address} diff --git a/contrib/terraform/aws/terraform.tfvars.example b/contrib/terraform/aws/terraform.tfvars.example index 214ef89db..666b21db2 100644 --- a/contrib/terraform/aws/terraform.tfvars.example +++ b/contrib/terraform/aws/terraform.tfvars.example @@ -1,6 +1,5 @@ #Global Vars aws_cluster_name = "devtest" -aws_region = "eu-central-1" #VPC Vars aws_vpc_cidr_block = "10.250.192.0/18" @@ -28,5 +27,6 @@ aws_cluster_ami = "ami-903df7ff" #Settings AWS ELB -aws_elb_api_port = 443 -k8s_secure_api_port = 443 +aws_elb_api_port = 6443 +k8s_secure_api_port = 6443 +kube_insecure_apiserver_address = 0.0.0.0 diff --git a/contrib/terraform/aws/variables.tf b/contrib/terraform/aws/variables.tf index 82e2fb018..c740e6472 100644 --- a/contrib/terraform/aws/variables.tf +++ b/contrib/terraform/aws/variables.tf @@ -95,3 +95,7 @@ variable "aws_elb_api_port" { variable "k8s_secure_api_port" { description = "Secure Port of K8S API Server" } + +variable "kube_insecure_apiserver_address" { + description= "Bind Address for insecure Port of K8s API Server" +} From 74c43c290a9d635c152cfc7e94e01a187e73125e Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Wed, 12 Apr 2017 13:37:50 +0300 Subject: [PATCH 06/47] Skip vault cert task evaluation completely when using script cert generation --- roles/etcd/tasks/gen_certs_vault.yml | 16 +++++++++++++--- roles/etcd/tasks/main.yml | 15 +-------------- .../kubernetes/secrets/tasks/gen_certs_vault.yml | 13 ++++++++++--- roles/kubernetes/secrets/tasks/main.yml | 8 -------- roles/vault/tasks/bootstrap/start_vault_temp.yml | 3 +++ 5 files changed, 27 insertions(+), 28 deletions(-) diff --git a/roles/etcd/tasks/gen_certs_vault.yml b/roles/etcd/tasks/gen_certs_vault.yml index 144e3b658..e45b2d02d 100644 --- a/roles/etcd/tasks/gen_certs_vault.yml +++ b/roles/etcd/tasks/gen_certs_vault.yml @@ -1,4 +1,12 @@ --- +- include: sync_etcd_master_certs.yml + when: inventory_hostname in groups.etcd + tags: etcd-secrets + +- include: sync_etcd_node_certs.yml + when: inventory_hostname in etcd_node_cert_hosts + tags: etcd-secrets + - name: gen_certs_vault | Read in the local credentials command: cat /etc/vault/roles/etcd/userpass @@ -15,7 +23,7 @@ url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ etcd_vault_creds.username }}" headers: Accept: application/json - Content-Type: application/json + Content-Type: application/json method: POST body_format: json body: @@ -37,7 +45,7 @@ issue_cert_copy_ca: "{{ item == etcd_master_certs_needed|first }}" issue_cert_file_group: "{{ etcd_cert_group }}" issue_cert_file_owner: kube - issue_cert_headers: "{{ etcd_vault_headers }}" + issue_cert_headers: "{{ etcd_vault_headers }}" issue_cert_hosts: "{{ groups.etcd }}" issue_cert_ip_sans: >- [ @@ -60,7 +68,7 @@ issue_cert_copy_ca: "{{ item == etcd_node_certs_needed|first }}" issue_cert_file_group: "{{ etcd_cert_group }}" issue_cert_file_owner: kube - issue_cert_headers: "{{ etcd_vault_headers }}" + issue_cert_headers: "{{ etcd_vault_headers }}" issue_cert_hosts: "{{ etcd_node_cert_hosts }}" issue_cert_ip_sans: >- [ @@ -75,3 +83,5 @@ with_items: "{{ etcd_node_certs_needed|d([]) }}" when: inventory_hostname in etcd_node_cert_hosts notify: set etcd_secret_changed + + diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index d917b56ac..afd5fa883 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -7,20 +7,7 @@ when: cert_management == "script" tags: [etcd-secrets, facts] -- include: gen_certs_script.yml - when: cert_management == "script" - tags: etcd-secrets - -- include: sync_etcd_master_certs.yml - when: cert_management == "vault" and inventory_hostname in groups.etcd - tags: etcd-secrets - -- include: sync_etcd_node_certs.yml - when: cert_management == "vault" and inventory_hostname in etcd_node_cert_hosts - tags: etcd-secrets - -- include: gen_certs_vault.yml - when: cert_management == "vault" and (etcd_master_certs_needed|d() or etcd_node_certs_needed|d()) +- include: "gen_certs_{{ cert_management }}.yml" tags: etcd-secrets - include: "install_{{ etcd_deployment_type }}.yml" diff --git a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml index 5a7c4827b..ac8e128b4 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml @@ -1,4 +1,11 @@ --- +- include: sync_kube_master_certs.yml + when: inventory_hostname in groups['kube-master'] + tags: k8s-secrets + +- include: sync_kube_node_certs.yml + when: inventory_hostname in groups['k8s-cluster'] + tags: k8s-secrets - name: gen_certs_vault | Read in the local credentials command: cat /etc/vault/roles/kube/userpass @@ -15,7 +22,7 @@ url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ kube_vault_creds.username }}" headers: Accept: application/json - Content-Type: application/json + Content-Type: application/json method: POST body_format: json body: @@ -54,7 +61,7 @@ }} issue_cert_file_group: "{{ kube_cert_group }}" issue_cert_file_owner: kube - issue_cert_headers: "{{ kube_vault_headers }}" + issue_cert_headers: "{{ kube_vault_headers }}" issue_cert_hosts: "{{ groups['kube-master'] }}" issue_cert_ip_sans: >- [ @@ -75,7 +82,7 @@ issue_cert_copy_ca: "{{ item == kube_node_certs_needed|first }}" issue_cert_file_group: "{{ kube_cert_group }}" issue_cert_file_owner: kube - issue_cert_headers: "{{ kube_vault_headers }}" + issue_cert_headers: "{{ kube_vault_headers }}" issue_cert_hosts: "{{ groups['k8s-cluster'] }}" issue_cert_path: "{{ item }}" issue_cert_role: kube diff --git a/roles/kubernetes/secrets/tasks/main.yml b/roles/kubernetes/secrets/tasks/main.yml index 919ed0df7..fb4c38f38 100644 --- a/roles/kubernetes/secrets/tasks/main.yml +++ b/roles/kubernetes/secrets/tasks/main.yml @@ -74,13 +74,5 @@ - include: "gen_certs_{{ cert_management }}.yml" tags: k8s-secrets -- include: sync_kube_master_certs.yml - when: cert_management == "vault" and inventory_hostname in groups['kube-master'] - tags: k8s-secrets - -- include: sync_kube_node_certs.yml - when: cert_management == "vault" and inventory_hostname in groups['k8s-cluster'] - tags: k8s-secrets - - include: gen_tokens.yml tags: k8s-secrets diff --git a/roles/vault/tasks/bootstrap/start_vault_temp.yml b/roles/vault/tasks/bootstrap/start_vault_temp.yml index eeaaad535..daebf1a6d 100644 --- a/roles/vault/tasks/bootstrap/start_vault_temp.yml +++ b/roles/vault/tasks/bootstrap/start_vault_temp.yml @@ -13,6 +13,9 @@ -v /etc/vault:/etc/vault {{ vault_image_repo }}:{{ vault_version }} server +- name: bootstrap/start_vault_temp | Start again single node Vault with file backend + command: docker start {{ vault_temp_container_name }} + - name: bootstrap/start_vault_temp | Initialize vault-temp uri: url: "http://localhost:{{ vault_port }}/v1/sys/init" From 4eb5a0e67a68c4f7f9bb80fb3ecae81817fc4f2d Mon Sep 17 00:00:00 2001 From: Joe Duhamel Date: Thu, 13 Apr 2017 14:55:25 -0400 Subject: [PATCH 07/47] Update kubedns-autoscaler change target The target was a replicationcontroller but kubedns is currently a deployment --- roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml index 65dee527f..c0f519e2c 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml +++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml @@ -42,7 +42,7 @@ spec: - --namespace=kube-system - --configmap=kubedns-autoscaler # Should keep target in sync with cluster/addons/dns/kubedns-controller.yaml.base - - --target=replicationcontroller/kubedns + - --target=Deployment/kubedns - --default-params={"linear":{"nodesPerReplica":{{ kubedns_nodes_per_replica }},"min":{{ kubedns_min_replicas }}}} - --logtostderr=true - --v=2 From f945c8d88576b59d1b813cb028a9c10953668b00 Mon Sep 17 00:00:00 2001 From: Joe Duhamel Date: Thu, 13 Apr 2017 15:07:06 -0400 Subject: [PATCH 08/47] Update dnsmasq-autoscaler changed target to be a deployment rather than a replicationcontroller. --- roles/dnsmasq/templates/dnsmasq-autoscaler.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/dnsmasq/templates/dnsmasq-autoscaler.yml b/roles/dnsmasq/templates/dnsmasq-autoscaler.yml index ca65c2dab..4e5e2ddcc 100644 --- a/roles/dnsmasq/templates/dnsmasq-autoscaler.yml +++ b/roles/dnsmasq/templates/dnsmasq-autoscaler.yml @@ -41,7 +41,7 @@ spec: - /cluster-proportional-autoscaler - --namespace=kube-system - --configmap=dnsmasq-autoscaler - - --target=ReplicationController/dnsmasq + - --target=Deployment/dnsmasq # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate. # If using small nodes, "nodesPerReplica" should dominate. - --default-params={"linear":{"nodesPerReplica":{{ dnsmasq_nodes_per_replica }},"preventSinglePointFailure":true}} From 2ab365825f0b64ff5426be2b8a8662b43e83de9b Mon Sep 17 00:00:00 2001 From: Sergii Golovatiuk Date: Fri, 14 Apr 2017 10:49:16 +0200 Subject: [PATCH 09/47] Reschedule netchecker-server in case of HW failure. Pod opbject is not reschedulable by kubernetes. It means that if node with netchecker-server goes down, netchecker-server won't be scheduled somewhere. This commit changes the type of netchecker-server to Deployment, so netchecker-server will be scheduled on other nodes in case of failures. --- .../ansible/tasks/netchecker.yml | 2 +- .../netchecker-server-deployment.yml.j2 | 33 +++++++++++++++++++ .../templates/netchecker-server-pod.yml.j2 | 28 ---------------- 3 files changed, 34 insertions(+), 29 deletions(-) create mode 100644 roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 delete mode 100644 roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml.j2 diff --git a/roles/kubernetes-apps/ansible/tasks/netchecker.yml b/roles/kubernetes-apps/ansible/tasks/netchecker.yml index b9047a1e2..aae75d091 100644 --- a/roles/kubernetes-apps/ansible/tasks/netchecker.yml +++ b/roles/kubernetes-apps/ansible/tasks/netchecker.yml @@ -5,7 +5,7 @@ with_items: - {file: netchecker-agent-ds.yml.j2, type: ds, name: netchecker-agent} - {file: netchecker-agent-hostnet-ds.yml.j2, type: ds, name: netchecker-agent-hostnet} - - {file: netchecker-server-pod.yml.j2, type: po, name: netchecker-server} + - {file: netchecker-server-deployment.yml.j2, type: po, name: netchecker-server} - {file: netchecker-server-svc.yml.j2, type: svc, name: netchecker-service} register: manifests when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 new file mode 100644 index 000000000..6c52352fb --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2 @@ -0,0 +1,33 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: netchecker-server +spec: + replicas: 1 + template: + metadata: + name: netchecker-server + labels: + app: netchecker-server + namespace: {{ netcheck_namespace }} + spec: + containers: + - name: netchecker-server + image: "{{ server_img }}" + env: + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + limits: + cpu: {{ netchecker_server_cpu_limit }} + memory: {{ netchecker_server_memory_limit }} + requests: + cpu: {{ netchecker_server_cpu_requests }} + memory: {{ netchecker_server_memory_requests }} + ports: + - containerPort: 8081 + hostPort: 8081 + args: + - "-v=5" + - "-logtostderr" + - "-kubeproxyinit" + - "-endpoint=0.0.0.0:8081" diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml.j2 deleted file mode 100644 index 06aea406a..000000000 --- a/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml.j2 +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: netchecker-server - labels: - app: netchecker-server - namespace: {{ netcheck_namespace }} -spec: - containers: - - name: netchecker-server - image: "{{ server_img }}" - env: - imagePullPolicy: {{ k8s_image_pull_policy }} - resources: - limits: - cpu: {{ netchecker_server_cpu_limit }} - memory: {{ netchecker_server_memory_limit }} - requests: - cpu: {{ netchecker_server_cpu_requests }} - memory: {{ netchecker_server_memory_requests }} - ports: - - containerPort: 8081 - hostPort: 8081 - args: - - "-v=5" - - "-logtostderr" - - "-kubeproxyinit" - - "-endpoint=0.0.0.0:8081" From c23385e77ebf6f382d5fd36555d3d1485f93c1de Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 14 Apr 2017 13:32:41 +0300 Subject: [PATCH 10/47] Update start_vault_temp.yml --- roles/vault/tasks/bootstrap/start_vault_temp.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/vault/tasks/bootstrap/start_vault_temp.yml b/roles/vault/tasks/bootstrap/start_vault_temp.yml index daebf1a6d..161ef92d6 100644 --- a/roles/vault/tasks/bootstrap/start_vault_temp.yml +++ b/roles/vault/tasks/bootstrap/start_vault_temp.yml @@ -13,6 +13,7 @@ -v /etc/vault:/etc/vault {{ vault_image_repo }}:{{ vault_version }} server +#FIXME(mattymo): Crashes on first start with aufs docker storage. See hashicorp/docker-vault#19 - name: bootstrap/start_vault_temp | Start again single node Vault with file backend command: docker start {{ vault_temp_container_name }} From 7656ae2887535f7f470d950e5bba390a5bda9812 Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Fri, 14 Apr 2017 17:33:04 -0400 Subject: [PATCH 11/47] add ability for custom flags --- docs/vars.md | 14 ++++++++++++++ roles/kubernetes/master/defaults/main.yml | 7 +++++++ .../templates/manifests/kube-apiserver.manifest.j2 | 3 +++ .../manifests/kube-controller-manager.manifest.j2 | 3 +++ .../templates/manifests/kube-scheduler.manifest.j2 | 3 +++ roles/kubernetes/node/defaults/main.yml | 3 +++ roles/kubernetes/node/templates/kubelet.j2 | 2 +- 7 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/vars.md b/docs/vars.md index 966b3ffc8..603a614b2 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -98,6 +98,20 @@ Stack](https://github.com/kubernetes-incubator/kargo/blob/master/docs/dns-stack. loaded by preinstall kubernetes processes. For example, ceph and rbd backed volumes. Set this variable to true to let kubelet load kernel modules. +##### Custom flags for Kube Components +For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. Example: +``` +kubelet_custom_flags: + - "--eviction-hard=memory.available<100Mi" + - "--eviction-soft-grace-period=memory.available=30s" + - "--eviction-soft=memory.available<300Mi" +``` +The possible vars are: +* *apiserver_custom_flags* +* *controller_mgr_custom_flags* +* *scheduler_custom_flags* +* *kubelet_custom_flags* + #### User accounts Kargo sets up two Kubernetes accounts by default: ``root`` and ``kube``. Their diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 016df0c64..bd5461239 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -51,3 +51,10 @@ kube_oidc_auth: false # kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem # kube_oidc_username_claim: sub # kube_oidc_groups_claim: groups + +##Variables for custom flags +apiserver_custom_flags: [] + +controller_mgr_custom_flags: [] + +scheduler_custom_flags: [] \ No newline at end of file diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 967f0a9cb..721474466 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -81,6 +81,9 @@ spec: {% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=') %} - --anonymous-auth={{ kube_api_anonymous_auth }} {% endif %} +{% for flag in apiserver_custom_flags %} + - {{ flag }} +{% endfor %} livenessProbe: httpGet: host: 127.0.0.1 diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 477d6a64f..0f66509ad 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -46,6 +46,9 @@ spec: - --configure-cloud-routes=true - --cluster-cidr={{ kube_pods_subnet }} {% endif %} +{% for flag in controller_mgr_custom_flags %} + - {{ flag }} +{% endfor %} livenessProbe: httpGet: host: 127.0.0.1 diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index 7431ddf3d..a549d5296 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -27,6 +27,9 @@ spec: - --leader-elect=true - --master={{ kube_apiserver_endpoint }} - --v={{ kube_log_level }} +{% for flag in scheduler_custom_flags %} + - {{ flag }} +{% endfor %} livenessProbe: httpGet: host: 127.0.0.1 diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 952214179..7f1e6f4a0 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -45,3 +45,6 @@ etcd_config_dir: /etc/ssl/etcd kube_apiserver_node_port_range: "30000-32767" kubelet_load_modules: false + +##Support custom flags to be passed to kubelet +kubelet_custom_flags: [] \ No newline at end of file diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index ba02e5eb9..d2ca95ad4 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -44,7 +44,7 @@ KUBELET_HOSTNAME="--hostname-override={{ ansible_hostname }}" {% set node_labels %}--node-labels=node-role.kubernetes.io/node=true{% endset %} {% endif %} -KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }}" +KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}" {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave", "canal"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %} From c05d14112843f9869f2f278051b5d64e33eb3f19 Mon Sep 17 00:00:00 2001 From: gbolo Date: Sun, 16 Apr 2017 22:03:45 -0400 Subject: [PATCH 12/47] allow admission control plug-ins to be easily customized --- roles/kubernetes/master/defaults/main.yml | 7 +++++++ .../master/templates/manifests/kube-apiserver.manifest.j2 | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 016df0c64..593ffd9cd 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -36,6 +36,13 @@ kube_apiserver_cpu_limit: 800m kube_apiserver_memory_requests: 256M kube_apiserver_cpu_requests: 100m +# Admission control plug-ins +kube_apiserver_admission_control: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - DefaultStorageClass + - ResourceQuota ## Enable/Disable Kube API Server Authentication Methods kube_basic_auth: true diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 967f0a9cb..36bcbc3f6 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -33,7 +33,7 @@ spec: - --etcd-keyfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem - --insecure-bind-address={{ kube_apiserver_insecure_bind_address }} - --apiserver-count={{ kube_apiserver_count }} - - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota + - --admission-control={{ kube_apiserver_admission_control | join(',') }} - --service-cluster-ip-range={{ kube_service_addresses }} - --service-node-port-range={{ kube_apiserver_node_port_range }} - --client-ca-file={{ kube_cert_dir }}/ca.pem From c1192b11540f06d43b0963e54afffe6f5339b432 Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Mon, 17 Apr 2017 11:09:34 -0400 Subject: [PATCH 13/47] update to safeguard against accidentally passing string instead of list --- .../master/templates/manifests/kube-apiserver.manifest.j2 | 8 ++++++-- .../manifests/kube-controller-manager.manifest.j2 | 8 ++++++-- .../master/templates/manifests/kube-scheduler.manifest.j2 | 8 ++++++-- roles/kubernetes/node/templates/kubelet.j2 | 2 +- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 721474466..c0ddf329b 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -81,9 +81,13 @@ spec: {% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=') %} - --anonymous-auth={{ kube_api_anonymous_auth }} {% endif %} -{% for flag in apiserver_custom_flags %} +{% if apiserver_custom_flags is string %} + - {{ apiserver_custom_flags }} +{% else % } +{% for flag in apiserver_custom_flags %} - {{ flag }} -{% endfor %} +{% endfor %} +{% endif %} livenessProbe: httpGet: host: 127.0.0.1 diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 0f66509ad..1bdcc4324 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -46,9 +46,13 @@ spec: - --configure-cloud-routes=true - --cluster-cidr={{ kube_pods_subnet }} {% endif %} -{% for flag in controller_mgr_custom_flags %} +{% if controller_mgr_custom_flags is string %} + - {{ controller_mgr_custom_flags }} +{% else % } +{% for flag in controller_mgr_custom_flags %} - {{ flag }} -{% endfor %} +{% endfor %} +{% endif %} livenessProbe: httpGet: host: 127.0.0.1 diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index a549d5296..d21db5470 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -27,9 +27,13 @@ spec: - --leader-elect=true - --master={{ kube_apiserver_endpoint }} - --v={{ kube_log_level }} -{% for flag in scheduler_custom_flags %} +{% if scheduler_custom_flags is string %} + - {{ scheduler_custom_flags }} +{% else % } +{% for flag in scheduler_custom_flags %} - {{ flag }} -{% endfor %} +{% endfor %} +{% endif %} livenessProbe: httpGet: host: 127.0.0.1 diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index d2ca95ad4..df207a545 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -44,7 +44,7 @@ KUBELET_HOSTNAME="--hostname-override={{ ansible_hostname }}" {% set node_labels %}--node-labels=node-role.kubernetes.io/node=true{% endset %} {% endif %} -KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}" +KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% if kubelet_custom_flags is string %}{{kubelet_custom_flags}}{% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave", "canal"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %} From 0fb9469249f2282ecf2ac307530bc1a93592ab87 Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Mon, 17 Apr 2017 11:11:10 -0400 Subject: [PATCH 14/47] ensure spacing on string of flags --- roles/kubernetes/node/templates/kubelet.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index df207a545..8de1e63e9 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -44,7 +44,7 @@ KUBELET_HOSTNAME="--hostname-override={{ ansible_hostname }}" {% set node_labels %}--node-labels=node-role.kubernetes.io/node=true{% endset %} {% endif %} -KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% if kubelet_custom_flags is string %}{{kubelet_custom_flags}}{% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" +KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave", "canal"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %} From daa728e3cf2cf031c367f509a1e6c85c2e197815 Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Mon, 17 Apr 2017 12:13:39 -0400 Subject: [PATCH 15/47] ensure spacing on string of flags --- .../master/templates/manifests/kube-apiserver.manifest.j2 | 2 +- .../templates/manifests/kube-controller-manager.manifest.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index c0ddf329b..a3b8a6f0a 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -83,7 +83,7 @@ spec: {% endif %} {% if apiserver_custom_flags is string %} - {{ apiserver_custom_flags }} -{% else % } +{% else %} {% for flag in apiserver_custom_flags %} - {{ flag }} {% endfor %} diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 1bdcc4324..b483047db 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -48,7 +48,7 @@ spec: {% endif %} {% if controller_mgr_custom_flags is string %} - {{ controller_mgr_custom_flags }} -{% else % } +{% else %} {% for flag in controller_mgr_custom_flags %} - {{ flag }} {% endfor %} From 1d848dc2119f322645d8ce12589a3124cbeaac08 Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Mon, 17 Apr 2017 12:24:24 -0400 Subject: [PATCH 16/47] remove stray spaces in templating --- .../master/templates/manifests/kube-scheduler.manifest.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index d21db5470..694450ce7 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -29,7 +29,7 @@ spec: - --v={{ kube_log_level }} {% if scheduler_custom_flags is string %} - {{ scheduler_custom_flags }} -{% else % } +{% else %} {% for flag in scheduler_custom_flags %} - {{ flag }} {% endfor %} From 33dfd43d1b9df0833889c5ea6296af7f2fc9b7e0 Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 17 Apr 2017 20:56:52 -0400 Subject: [PATCH 17/47] Fix IPS array variable expansion $IPS only expands to the first ip address in the array: justin@box:~$ declare -a IPS=(10.10.1.3 10.10.1.4 10.10.1.5) justin@box:~$ echo $IPS 10.10.1.3 justin@box:~$ echo ${IPS[@]} 10.10.1.3 10.10.1.4 10.10.1.5 --- docs/getting-started.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index caf4485ae..5c61ef764 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -38,7 +38,7 @@ Example inventory generator usage: ``` cp -r inventory my_inventory declare -a IPS=(10.10.1.3 10.10.1.4 10.10.1.5) -CONFIG_FILE=my_inventory/inventory.cfg python3 contrib/inventory_builder/inventory.py ${IPS} +CONFIG_FILE=my_inventory/inventory.cfg python3 contrib/inventory_builder/inventory.py ${IPS[@]} ``` Starting custom deployment From 8318fac5c2384cf905d397d5b536a60ea9cd3c09 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Mon, 17 Apr 2017 17:14:05 +0300 Subject: [PATCH 18/47] Add minimal k8s upgrade playbook --- .gitlab-ci.yml | 1 + extra_playbooks/inventory | 1 + extra_playbooks/roles | 1 + extra_playbooks/upgrade-only-k8s.yml | 60 ++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+) create mode 120000 extra_playbooks/inventory create mode 120000 extra_playbooks/roles create mode 100644 extra_playbooks/upgrade-only-k8s.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2a0106162..7a6694f24 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -596,6 +596,7 @@ syntax-check: - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root -b --become-user=root cluster.yml -vvv --syntax-check - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root -b --become-user=root upgrade-cluster.yml -vvv --syntax-check - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root -b --become-user=root reset.yml -vvv --syntax-check + - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root -b --become-user=root extra_playbooks/upgrade-only-k8s.yml -vvv --syntax-check except: ['triggers', 'master'] tox-inventory-builder: diff --git a/extra_playbooks/inventory b/extra_playbooks/inventory new file mode 120000 index 000000000..e09e1addd --- /dev/null +++ b/extra_playbooks/inventory @@ -0,0 +1 @@ +../inventory \ No newline at end of file diff --git a/extra_playbooks/roles b/extra_playbooks/roles new file mode 120000 index 000000000..d8c4472ca --- /dev/null +++ b/extra_playbooks/roles @@ -0,0 +1 @@ +../roles \ No newline at end of file diff --git a/extra_playbooks/upgrade-only-k8s.yml b/extra_playbooks/upgrade-only-k8s.yml new file mode 100644 index 000000000..f10259b07 --- /dev/null +++ b/extra_playbooks/upgrade-only-k8s.yml @@ -0,0 +1,60 @@ +### NOTE: This playbook cannot be used to deploy any new nodes to the cluster. +### Additional information: +### * Will not upgrade etcd +### * Will not upgrade network plugins +### * Will not upgrade Docker +### * Currently does not support Vault deployment. +### +### In most cases, you probably want to use upgrade-cluster.yml playbook and +### not this one. + +- hosts: localhost + gather_facts: False + roles: + - { role: kargo-defaults} + - { role: bastion-ssh-config, tags: ["localhost", "bastion"]} + +- hosts: k8s-cluster:etcd:calico-rr + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + gather_facts: false + vars: + # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining + # fail. bootstrap-os fixes this on these systems, so in later plays it can be enabled. + ansible_ssh_pipelining: false + roles: + - { role: kargo-defaults} + - { role: bootstrap-os, tags: bootstrap-os} + +- hosts: k8s-cluster:etcd:calico-rr + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + vars: + ansible_ssh_pipelining: true + gather_facts: true + +- hosts: k8s-cluster:etcd:calico-rr + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + roles: + - { role: kargo-defaults} + - { role: kubernetes/preinstall, tags: preinstall } + +#Handle upgrades to master components first to maintain backwards compat. +- hosts: kube-master + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + serial: 1 + roles: + - { role: kargo-defaults} + - { role: upgrade/pre-upgrade, tags: pre-upgrade } + - { role: kubernetes/node, tags: node } + - { role: kubernetes/master, tags: master } + - { role: upgrade/post-upgrade, tags: post-upgrade } + +#Finally handle worker upgrades, based on given batch size +- hosts: kube-node:!kube-master + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + serial: "{{ serial | default('20%') }}" + roles: + - { role: kargo-defaults} + - { role: upgrade/pre-upgrade, tags: pre-upgrade } + - { role: kubernetes/node, tags: node } + - { role: upgrade/post-upgrade, tags: post-upgrade } + - { role: kargo-defaults} From 12bbb243b254ba58bacd598a82d957a07fa385f3 Mon Sep 17 00:00:00 2001 From: Hans Kristian Flaatten Date: Tue, 18 Apr 2017 14:59:14 +0200 Subject: [PATCH 19/47] Move namespace file to template directory --- roles/kubernetes/master/tasks/main.yml | 6 +++--- .../master/{files/namespace.yml => templates/namespace.j2} | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename roles/kubernetes/master/{files/namespace.yml => templates/namespace.j2} (100%) diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index 2c669c46d..dadef4bf5 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -34,9 +34,9 @@ - meta: flush_handlers -- name: copy kube system namespace manifest - copy: - src: namespace.yml +- name: Write kube system namespace manifest + template: + src: namespace.j2 dest: "{{kube_config_dir}}/{{system_namespace}}-ns.yml" run_once: yes when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes/master/files/namespace.yml b/roles/kubernetes/master/templates/namespace.j2 similarity index 100% rename from roles/kubernetes/master/files/namespace.yml rename to roles/kubernetes/master/templates/namespace.j2 From a1ba2a90e6e669ef3f856d730b6ca8193a5dc2f5 Mon Sep 17 00:00:00 2001 From: Hans Kristian Flaatten Date: Tue, 18 Apr 2017 12:07:03 +0200 Subject: [PATCH 20/47] Remove and ignore .bak files --- .gitignore | 1 + .travis.yml.bak | 161 ------------------------------------------------ 2 files changed, 1 insertion(+), 161 deletions(-) delete mode 100644 .travis.yml.bak diff --git a/.gitignore b/.gitignore index e7bbe0bea..f4c7d990a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ temp .idea .tox .cache +*.bak *.egg-info *.pyc *.pyo diff --git a/.travis.yml.bak b/.travis.yml.bak deleted file mode 100644 index 7b948dcfe..000000000 --- a/.travis.yml.bak +++ /dev/null @@ -1,161 +0,0 @@ -sudo: required - -services: - - docker - -git: - depth: 5 - -env: - global: - GCE_USER=travis - SSH_USER=$GCE_USER - TEST_ID=$TRAVIS_JOB_NUMBER - CONTAINER_ENGINE=docker - PRIVATE_KEY=$GCE_PRIVATE_KEY - GS_ACCESS_KEY_ID=$GS_KEY - GS_SECRET_ACCESS_KEY=$GS_SECRET - ANSIBLE_KEEP_REMOTE_FILES=1 - CLUSTER_MODE=default - BOOTSTRAP_OS=none - matrix: - # Debian Jessie - - >- - KUBE_NETWORK_PLUGIN=canal - CLOUD_IMAGE=debian-8-kubespray - CLOUD_REGION=asia-east1-a - CLUSTER_MODE=ha - - >- - KUBE_NETWORK_PLUGIN=calico - CLOUD_IMAGE=debian-8-kubespray - CLOUD_REGION=europe-west1-c - CLUSTER_MODE=default - - # Centos 7 - - >- - KUBE_NETWORK_PLUGIN=flannel - CLOUD_IMAGE=centos-7 - CLOUD_REGION=asia-northeast1-c - CLUSTER_MODE=default - - >- - KUBE_NETWORK_PLUGIN=calico - CLOUD_IMAGE=centos-7 - CLOUD_REGION=us-central1-b - CLUSTER_MODE=ha - - # Redhat 7 - - >- - KUBE_NETWORK_PLUGIN=weave - CLOUD_IMAGE=rhel-7 - CLOUD_REGION=us-east1-c - CLUSTER_MODE=default - - # CoreOS stable - #- >- - # KUBE_NETWORK_PLUGIN=weave - # CLOUD_IMAGE=coreos-stable - # CLOUD_REGION=europe-west1-b - # CLUSTER_MODE=ha - # BOOTSTRAP_OS=coreos - - >- - KUBE_NETWORK_PLUGIN=canal - CLOUD_IMAGE=coreos-stable - CLOUD_REGION=us-west1-b - CLUSTER_MODE=default - BOOTSTRAP_OS=coreos - - # Extra cases for separated roles - - >- - KUBE_NETWORK_PLUGIN=canal - CLOUD_IMAGE=rhel-7 - CLOUD_REGION=asia-northeast1-b - CLUSTER_MODE=separate - - >- - KUBE_NETWORK_PLUGIN=weave - CLOUD_IMAGE=ubuntu-1604-xenial - CLOUD_REGION=europe-west1-d - CLUSTER_MODE=separate - - >- - KUBE_NETWORK_PLUGIN=calico - CLOUD_IMAGE=coreos-stable - CLOUD_REGION=us-central1-f - CLUSTER_MODE=separate - BOOTSTRAP_OS=coreos - -matrix: - allow_failures: - - env: KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=coreos-stable CLOUD_REGION=europe-west1-b CLUSTER_MODE=ha BOOTSTRAP_OS=coreos - -before_install: - # Install Ansible. - - pip install --user ansible - - pip install --user netaddr - # W/A https://github.com/ansible/ansible-modules-core/issues/5196#issuecomment-253766186 - - pip install --user apache-libcloud==0.20.1 - - pip install --user boto==2.9.0 -U - # Load cached docker images - - if [ -d /var/tmp/releases ]; then find /var/tmp/releases -type f -name "*.tar" | xargs -I {} sh -c "zcat {} | docker load"; fi - -cache: - - directories: - - $HOME/.cache/pip - - $HOME/.local - - /var/tmp/releases - -before_script: - - echo "RUN $TRAVIS_JOB_NUMBER $KUBE_NETWORK_PLUGIN $CONTAINER_ENGINE " - - mkdir -p $HOME/.ssh - - echo $PRIVATE_KEY | base64 -d > $HOME/.ssh/id_rsa - - echo $GCE_PEM_FILE | base64 -d > $HOME/.ssh/gce - - chmod 400 $HOME/.ssh/id_rsa - - chmod 755 $HOME/.local/bin/ansible-playbook - - $HOME/.local/bin/ansible-playbook --version - - cp tests/ansible.cfg . - - export PYPATH=$([ $BOOTSTRAP_OS = none ] && echo /usr/bin/python || echo /opt/bin/python) -# - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}' $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml" - -script: - - > - $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts.cfg -c local $LOG_LEVEL - -e mode=${CLUSTER_MODE} - -e test_id=${TEST_ID} - -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} - -e gce_project_id=${GCE_PROJECT_ID} - -e gce_service_account_email=${GCE_ACCOUNT} - -e gce_pem_file=${HOME}/.ssh/gce - -e cloud_image=${CLOUD_IMAGE} - -e inventory_path=${PWD}/inventory/inventory.ini - -e cloud_region=${CLOUD_REGION} - - # Create cluster with netchecker app deployed - - > - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS - -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} - -e bootstrap_os=${BOOTSTRAP_OS} - -e ansible_python_interpreter=${PYPATH} - -e download_run_once=true - -e download_localhost=true - -e local_release_dir=/var/tmp/releases - -e deploy_netchecker=true - cluster.yml - - # Tests Cases - ## Test Master API - - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/010_check-apiserver.yml $LOG_LEVEL - ## Ping the between 2 pod - - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL - ## Advanced DNS checks - - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/040_check-network-adv.yml $LOG_LEVEL - -after_script: - - > - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local $LOG_LEVEL - -e mode=${CLUSTER_MODE} - -e test_id=${TEST_ID} - -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} - -e gce_project_id=${GCE_PROJECT_ID} - -e gce_service_account_email=${GCE_ACCOUNT} - -e gce_pem_file=${HOME}/.ssh/gce - -e cloud_image=${CLOUD_IMAGE} - -e inventory_path=${PWD}/inventory/inventory.ini - -e cloud_region=${CLOUD_REGION} From 0dc4967e4327e5a27e05c13495fe900a54292985 Mon Sep 17 00:00:00 2001 From: Brad Beam Date: Wed, 19 Apr 2017 16:00:44 +0000 Subject: [PATCH 21/47] Explicitly create cni bin dir If this path doesnt exist, it will cause kubelet to fail to start when using rkt --- roles/kubernetes/node/tasks/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 324e38867..f09845f76 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -7,6 +7,12 @@ - include: pre_upgrade.yml tags: kubelet +- name: Ensure /var/lib/cni exists + file: + path: /var/lib/cni + state: directory + mode: 0755 + - include: install.yml tags: kubelet From be224664893374bdedd5efe1209307ee8e55d983 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Tue, 18 Apr 2017 17:15:22 +0300 Subject: [PATCH 22/47] Add tags to reset playbook and make iptables flush optional Fixes #1229 --- roles/reset/defaults/main.yml | 2 ++ roles/reset/tasks/main.yml | 14 ++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 roles/reset/defaults/main.yml diff --git a/roles/reset/defaults/main.yml b/roles/reset/defaults/main.yml new file mode 100644 index 000000000..7d4dbfdff --- /dev/null +++ b/roles/reset/defaults/main.yml @@ -0,0 +1,2 @@ +--- +flush_iptables: true diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index f5f749647..96984f92b 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -8,6 +8,7 @@ - kubelet - etcd failed_when: false + tags: ['services'] - name: reset | remove services file: @@ -17,6 +18,7 @@ - kubelet - etcd register: services_removed + tags: ['services'] - name: reset | remove docker dropins file: @@ -26,6 +28,7 @@ - docker-dns.conf - docker-options.conf register: docker_dropins_removed + tags: ['docker'] - name: reset | systemctl daemon-reload command: systemctl daemon-reload @@ -33,25 +36,31 @@ - name: reset | remove all containers shell: "{{ docker_bin_dir }}/docker ps -aq | xargs -r docker rm -fv" + tags: ['docker'] - name: reset | restart docker if needed service: name: docker state: restarted when: docker_dropins_removed.changed + tags: ['docker'] - name: reset | gather mounted kubelet dirs shell: mount | grep /var/lib/kubelet | awk '{print $3}' | tac check_mode: no register: mounted_dirs + tags: ['mounts'] - name: reset | unmount kubelet dirs command: umount {{item}} with_items: '{{ mounted_dirs.stdout_lines }}' + tags: ['mounts'] - name: flush iptables iptables: flush: yes + when: flush_iptables|bool + tags: ['iptables'] - name: reset | delete some files and directories file: @@ -74,6 +83,8 @@ - /etc/dhcp/dhclient.d/zdnsupdate.sh - /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate - "{{ bin_dir }}/kubelet" + tags: ['files'] + - name: reset | remove dns settings from dhclient.conf blockinfile: @@ -85,6 +96,7 @@ with_items: - /etc/dhclient.conf - /etc/dhcp/dhclient.conf + tags: ['files', 'dns'] - name: reset | remove host entries from /etc/hosts blockinfile: @@ -92,6 +104,7 @@ state: absent follow: yes marker: "# Ansible inventory hosts {mark}" + tags: ['files', 'dns'] - name: reset | Restart network service: @@ -103,3 +116,4 @@ {%- endif %} state: restarted when: ansible_os_family not in ["CoreOS", "Container Linux by CoreOS"] + tags: ['services', 'network'] From 1268c9b64237dfc11f3d2012c486f75a27cf537c Mon Sep 17 00:00:00 2001 From: Sergii Golovatiuk Date: Thu, 20 Apr 2017 10:26:01 +0200 Subject: [PATCH 23/47] Fix restart kube-controller (#1242) kubernetesUnitPrefix was changed to k8s_* in 1.5. This patch reflects this change in kargo --- roles/kubernetes/preinstall/handlers/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes/preinstall/handlers/main.yml b/roles/kubernetes/preinstall/handlers/main.yml index f7e309e92..35fec7d94 100644 --- a/roles/kubernetes/preinstall/handlers/main.yml +++ b/roles/kubernetes/preinstall/handlers/main.yml @@ -45,5 +45,5 @@ when: inventory_hostname in groups['kube-master'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' - name: Preinstall | restart kube-controller-manager - shell: "docker ps -f name=k8s-controller-manager* -q | xargs --no-run-if-empty docker rm -f" + shell: "docker ps -f name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty docker rm -f" when: inventory_hostname in groups['kube-master'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' and kube_controller_set.stat.exists From 0a687a22ffd8e38ca7983f04e277cb20f43ca0e9 Mon Sep 17 00:00:00 2001 From: Sergii Golovatiuk Date: Thu, 20 Apr 2017 11:07:34 +0200 Subject: [PATCH 24/47] Change DNS policy for kubernetes components According to code apiserver, scheduler, controller-manager, proxy don't use resolution of objects they created. It's not harmful to change policy to have external resolver. Signed-off-by: Sergii Golovatiuk --- .../master/templates/manifests/kube-apiserver.manifest.j2 | 2 +- .../templates/manifests/kube-controller-manager.manifest.j2 | 2 +- .../master/templates/manifests/kube-scheduler.manifest.j2 | 2 +- .../kubernetes/node/templates/manifests/kube-proxy.manifest.j2 | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index ae014f8d3..b0f1a2f53 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -9,7 +9,7 @@ metadata: spec: hostNetwork: true {% if kube_version | version_compare('v1.6', '>=') %} - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: ClusterFirst {% endif %} containers: - name: kube-apiserver diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index b483047db..d3f8a23a5 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -8,7 +8,7 @@ metadata: spec: hostNetwork: true {% if kube_version | version_compare('v1.6', '>=') %} - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: ClusterFirst {% endif %} containers: - name: kube-controller-manager diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index 694450ce7..441f991eb 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -8,7 +8,7 @@ metadata: spec: hostNetwork: true {% if kube_version | version_compare('v1.6', '>=') %} - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: ClusterFirst {% endif %} containers: - name: kube-scheduler diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index 745c671d8..9b7d53857 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -8,7 +8,7 @@ metadata: spec: hostNetwork: true {% if kube_version | version_compare('v1.6', '>=') %} - dnsPolicy: ClusterFirstWithHostNet + dnsPolicy: ClusterFirst {% endif %} containers: - name: kube-proxy From f061ce63b3c33ebb97dc36aca1e2540300778be0 Mon Sep 17 00:00:00 2001 From: Sergii Golovatiuk Date: Thu, 20 Apr 2017 11:24:43 +0200 Subject: [PATCH 25/47] Add aws to default_resolver When VPC is used, external DNS might not be available. This patch change behavior to use metadata service instead of external DNS when upstream_dns_servers is not specified. Signed-off-by: Sergii Golovatiuk --- roles/kubernetes/preinstall/tasks/set_resolv_facts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml index ffea74b40..1f2b82cc1 100644 --- a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml @@ -16,7 +16,7 @@ {{dns_domain}}.{{d}}./{{d}}.{{d}}./com.{{d}}./ {%- endfor %} default_resolver: >- - {%- if cloud_provider is defined and cloud_provider == 'gce' -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%} + {%- if cloud_provider is defined and cloud_provider in [ 'gce', 'aws' ] -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%} - name: check if kubelet is configured stat: From 21b10784f429a9800b505e0b6df2bf49761ba13e Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Thu, 20 Apr 2017 09:32:03 -0400 Subject: [PATCH 26/47] allow for correct aws default resolver --- roles/kubernetes/preinstall/tasks/set_resolv_facts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml index 1f2b82cc1..390d4e562 100644 --- a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml @@ -16,7 +16,7 @@ {{dns_domain}}.{{d}}./{{d}}.{{d}}./com.{{d}}./ {%- endfor %} default_resolver: >- - {%- if cloud_provider is defined and cloud_provider in [ 'gce', 'aws' ] -%}169.254.169.254{%- else -%}8.8.8.8{%- endif -%} + {%- if cloud_provider is defined and cloud_provider in == 'gce' -%}169.254.169.254{%- elif cloud_provider is defined and cloud_provider == 'aws' -%}169.254.169.253{%- else -%}8.8.8.8{%- endif -%} - name: check if kubelet is configured stat: From 04818b9d942a191eccd26c407126f00bb2b79ad2 Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Thu, 20 Apr 2017 09:53:01 -0400 Subject: [PATCH 27/47] fix stray 'in' and break into multiple lines for clarity --- roles/kubernetes/preinstall/tasks/set_resolv_facts.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml index 390d4e562..18728faa7 100644 --- a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml @@ -16,7 +16,13 @@ {{dns_domain}}.{{d}}./{{d}}.{{d}}./com.{{d}}./ {%- endfor %} default_resolver: >- - {%- if cloud_provider is defined and cloud_provider in == 'gce' -%}169.254.169.254{%- elif cloud_provider is defined and cloud_provider == 'aws' -%}169.254.169.253{%- else -%}8.8.8.8{%- endif -%} + {%- if cloud_provider is defined and cloud_provider == 'gce' -%} + 169.254.169.254 + {%- elif cloud_provider is defined and cloud_provider == 'aws' -%} + 169.254.169.253 + {%- else -%} + 8.8.8.8 + {%- endif -%} - name: check if kubelet is configured stat: From 9f5a62dc2dc3ef7afd10c244c3e6e03925d86c5b Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Thu, 20 Apr 2017 11:14:41 -0400 Subject: [PATCH 28/47] add some known tweaks that need to be made for coreos --- docs/coreos.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/coreos.md b/docs/coreos.md index 7c9b2c8a6..546ad0e89 100644 --- a/docs/coreos.md +++ b/docs/coreos.md @@ -11,6 +11,10 @@ Or with Ansible: Before running the cluster playbook you must satisfy the following requirements: -* On each CoreOS nodes a writable directory **/opt/bin** (~400M disk space) +General CoreOS Pre-Installation Notes: +- You should set the bootstrap_os variable to `coreos` +- Ensure that the bin_dir is set to `/opt/bin` +- ansible_python_interpreter should be `/opt/bin/python`. This will be laid down by the bootstrap task. +- The default resolvconf_mode setting of `docker_dns` **does not** work for CoreOS. This is because we do not edit the systemd service file for docker on CoreOS nodes. Instead, just use the `host_resolvconf` mode. It should work out of the box. Then you can proceed to [cluster deployment](#run-deployment) From e85b53a6cb154899f58f6d4587699325adbd417c Mon Sep 17 00:00:00 2001 From: Greg Althaus Date: Thu, 20 Apr 2017 11:17:01 -0500 Subject: [PATCH 29/47] Install required selinux-python bindings in bootstrap on centos. The bootstrap tty fixup needs it. --- roles/bootstrap-os/tasks/bootstrap-centos.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/roles/bootstrap-os/tasks/bootstrap-centos.yml b/roles/bootstrap-os/tasks/bootstrap-centos.yml index b8cf126c1..dfc117b29 100644 --- a/roles/bootstrap-os/tasks/bootstrap-centos.yml +++ b/roles/bootstrap-os/tasks/bootstrap-centos.yml @@ -13,3 +13,15 @@ line: "enabled=0" state: present when: fastestmirror.stat.exists + +- name: Install packages requirements for bootstrap + action: + module: "{{ ansible_pkg_mgr }}" + name: "{{ item }}" + state: latest + register: bs_pkgs_task_result + until: bs_pkgs_task_result|succeeded + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + with_items: "libselinux-python" + From 4ef76e0c75de67ff3ed114b09155e695fcf010f2 Mon Sep 17 00:00:00 2001 From: FengyunPan Date: Fri, 21 Apr 2017 10:51:27 +0800 Subject: [PATCH 30/47] Specify a dir and attach it to helm for HELM_HOME --- roles/kubernetes-apps/helm/defaults/main.yml | 3 +++ roles/kubernetes-apps/helm/tasks/main.yml | 3 +++ roles/kubernetes-apps/helm/templates/helm-container.j2 | 1 + 3 files changed, 7 insertions(+) diff --git a/roles/kubernetes-apps/helm/defaults/main.yml b/roles/kubernetes-apps/helm/defaults/main.yml index ae139556d..b1b2dfca9 100644 --- a/roles/kubernetes-apps/helm/defaults/main.yml +++ b/roles/kubernetes-apps/helm/defaults/main.yml @@ -1 +1,4 @@ helm_enabled: false + +# specify a dir and attach it to helm for HELM_HOME. +helm_home_dir: "/root/.helm" diff --git a/roles/kubernetes-apps/helm/tasks/main.yml b/roles/kubernetes-apps/helm/tasks/main.yml index 551b0375e..3060bcda2 100644 --- a/roles/kubernetes-apps/helm/tasks/main.yml +++ b/roles/kubernetes-apps/helm/tasks/main.yml @@ -1,4 +1,7 @@ --- +- name: Helm | Make sure HELM_HOME directory exists + file: path={{ helm_home_dir }} state=directory + - name: Helm | Set up helm launcher template: src: helm-container.j2 diff --git a/roles/kubernetes-apps/helm/templates/helm-container.j2 b/roles/kubernetes-apps/helm/templates/helm-container.j2 index 598daa73a..68210ea30 100644 --- a/roles/kubernetes-apps/helm/templates/helm-container.j2 +++ b/roles/kubernetes-apps/helm/templates/helm-container.j2 @@ -3,6 +3,7 @@ --net=host \ --name=helm \ -v /etc/ssl:/etc/ssl:ro \ + -v {{ helm_home_dir }}:{{ helm_home_dir }}:rw \ {% for dir in ssl_ca_dirs -%} -v {{ dir }}:{{ dir }}:ro \ {% endfor -%} From 31d8f29ae8bb6cf3480f0e2b870986ac684ae076 Mon Sep 17 00:00:00 2001 From: Aleksey Kasatkin Date: Mon, 24 Apr 2017 16:46:16 +0300 Subject: [PATCH 31/47] add MY_NODE_NAME variable into netchecker-agent environment --- .../ansible/templates/netchecker-agent-ds.yml.j2 | 4 ++++ .../ansible/templates/netchecker-agent-hostnet-ds.j2 | 4 ++++ .../ansible/templates/netchecker-agent-hostnet-ds.yml.j2 | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 index 139498733..df0b8ba90 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2 @@ -20,6 +20,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName args: - "-v=5" - "-alsologtostderr=true" diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.j2 index 13a966c80..10a74da84 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.j2 @@ -24,6 +24,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName args: - "-v=5" - "-alsologtostderr=true" diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 index 13a966c80..10a74da84 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2 @@ -24,6 +24,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName args: - "-v=5" - "-alsologtostderr=true" From 085aeb6a0a09ce1533a823d83e5e945ebe8c5078 Mon Sep 17 00:00:00 2001 From: Sergii Golovatiuk Date: Wed, 26 Apr 2017 14:11:13 +0200 Subject: [PATCH 32/47] Ansible 2.3 support - Fix when clauses in various places - Update requirements.txt - Fix README.md Signed-off-by: Sergii Golovatiuk --- README.md | 4 +- requirements.txt | 15 +--- roles/download/tasks/main.yml | 89 +++++++++++++++---- roles/kernel-upgrade/tasks/reboot.yml | 6 +- .../kubernetes/preinstall/tasks/etchosts.yml | 5 +- roles/kubernetes/preinstall/tasks/main.yml | 68 +++++++++----- .../secrets/tasks/sync_kube_master_certs.yml | 6 +- roles/network_plugin/calico/tasks/main.yml | 2 +- .../tasks/bootstrap/start_vault_temp.yml | 2 +- 9 files changed, 136 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index cfc82a6cd..02bdb72a4 100644 --- a/README.md +++ b/README.md @@ -67,9 +67,9 @@ plugins can be deployed for a given single cluster. Requirements -------------- -* **Ansible v2.2 (or newer) and python-netaddr is installed on the machine +* **Ansible v2.3 (or newer) and python-netaddr is installed on the machine that will run Ansible commands** -* **Jinja 2.8 (or newer) is required to run the Ansible Playbooks** +* **Jinja 2.9 (or newer) is required to run the Ansible Playbooks** * The target servers must have **access to the Internet** in order to pull docker images. * The target servers are configured to allow **IPv4 forwarding**. * **Your ssh key must be copied** to all the servers part of your inventory. diff --git a/requirements.txt b/requirements.txt index ccf58ea3a..6458113ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,3 @@ -ansible==2.2.1.0 +ansible>=2.3.0 netaddr -# Ansible 2.2.1 requires jinja2<2.9, see , -# but without explicit limiting upper jinja2 version here pip ignores -# Ansible requirements and installs latest available jinja2 -# (pip is not very smart here), which is incompatible with with -# Ansible 2.2.1. -# With incompatible jinja2 version "ansible-vault create" (and probably other parts) -# fails with: -# ERROR! Unexpected Exception: The 'jinja2<2.9' distribution was not found -# and is required by ansible -# This upper limit should be removed in 2.2.2 release, see: -# -jinja2>=2.8,<2.9 +jinja2>=2.9.6 diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index 37c72462e..24d1b5bca 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -2,14 +2,18 @@ - name: downloading... debug: msg: "{{ download.url }}" - when: "{{ download.enabled|bool and not download.container|bool }}" + when: + - download.enabled|bool + - not download.container|bool - name: Create dest directories file: path: "{{local_release_dir}}/{{download.dest|dirname}}" state: directory recurse: yes - when: "{{ download.enabled|bool and not download.container|bool }}" + when: + - download.enabled|bool + - not download.container|bool tags: bootstrap-os - name: Download items @@ -23,7 +27,9 @@ until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg" retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: "{{ download.enabled|bool and not download.container|bool }}" + when: + - download.enabled|bool + - not download.container|bool - name: Extract archives unarchive: @@ -32,7 +38,11 @@ owner: "{{ download.owner|default(omit) }}" mode: "{{ download.mode|default(omit) }}" copy: no - when: "{{ download.enabled|bool and not download.container|bool and download.unarchive is defined and download.unarchive == True }}" + when: + - download.enabled|bool + - not download.container|bool + - download.unarchive is defined + - download.unarchive == True - name: Fix permissions file: @@ -40,7 +50,10 @@ path: "{{local_release_dir}}/{{download.dest}}" owner: "{{ download.owner|default(omit) }}" mode: "{{ download.mode|default(omit) }}" - when: "{{ download.enabled|bool and not download.container|bool and (download.unarchive is not defined or download.unarchive == False) }}" + when: + - download.enabled|bool + - not download.container|bool + - (download.unarchive is not defined or download.unarchive == False) - set_fact: download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube-master'][0]}}{% endif %}" @@ -53,13 +66,15 @@ recurse: yes mode: 0755 owner: "{{ansible_ssh_user|default(ansible_user_id)}}" - when: "{{ download.enabled|bool and download.container|bool }}" + when: + - download.enabled|bool + - download.container|bool tags: bootstrap-os # This is required for the download_localhost delegate to work smooth with Container Linux by CoreOS cluster nodes - name: Hack python binary path for localhost raw: sh -c "mkdir -p /opt/bin; ln -sf /usr/bin/python /opt/bin/python" - when: "{{ download_delegate == 'localhost' }}" + when: download_delegate == 'localhost' delegate_to: localhost failed_when: false run_once: true @@ -73,12 +88,18 @@ delegate_to: localhost become: false run_once: true - when: "{{ download_run_once|bool and download.enabled|bool and download.container|bool and download_delegate == 'localhost' }}" + when: + - download_run_once|bool + - download.enabled|bool + - download.container|bool + - download_delegate == 'localhost' tags: localhost - name: Make download decision if pull is required by tag or sha256 include: set_docker_image_facts.yml - when: "{{ download.enabled|bool and download.container|bool }}" + when: + - download.enabled|bool + - download.container|bool delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}" run_once: "{{ download_run_once|bool }}" tags: facts @@ -86,7 +107,9 @@ - name: pulling... debug: msg: "{{ pull_args }}" - when: "{{ download.enabled|bool and download.container|bool }}" + when: + - download.enabled|bool + - download.container|bool #NOTE(bogdando) this brings no docker-py deps for nodes - name: Download containers if pull is required or told to always pull @@ -95,7 +118,10 @@ until: pull_task_result|succeeded retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: "{{ download.enabled|bool and download.container|bool and pull_required|bool|default(download_always_pull) }}" + when: + - download.enabled|bool + - download.container|bool + - pull_required|bool|default(download_always_pull) delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}" run_once: "{{ download_run_once|bool }}" @@ -110,7 +136,10 @@ - name: "Update the 'container_changed' fact" set_fact: container_changed: "{{ pull_required|bool|default(false) or not 'up to date' in pull_task_result.stdout }}" - when: "{{ download.enabled|bool and download.container|bool and pull_required|bool|default(download_always_pull) }}" + when: + - download.enabled|bool + - download.container|bool + - pull_required|bool|default(download_always_pull) delegate_to: "{{ download_delegate if download_run_once|bool else inventory_hostname }}" run_once: "{{ download_run_once|bool }}" tags: facts @@ -120,7 +149,10 @@ path: "{{fname}}" register: img changed_when: false - when: "{{ download.enabled|bool and download.container|bool and download_run_once|bool }}" + when: + - download.enabled|bool + - download.container|bool + - download_run_once|bool delegate_to: "{{ download_delegate }}" become: false run_once: true @@ -131,7 +163,12 @@ delegate_to: "{{ download_delegate }}" register: saved run_once: true - when: (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or download_delegate == "localhost") and download_run_once|bool and download.enabled|bool and download.container|bool and (container_changed|bool or not img.stat.exists) + when: + - (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or download_delegate == "localhost") + - download_run_once|bool + - download.enabled|bool + - download.container|bool + - (container_changed|bool or not img.stat.exists) - name: Download | copy container images to ansible host synchronize: @@ -140,7 +177,14 @@ mode: pull delegate_to: localhost become: false - when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and inventory_hostname == groups['kube-master'][0] and download_delegate != "localhost" and download_run_once|bool and download.enabled|bool and download.container|bool and saved.changed + when: + - not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + - inventory_hostname == groups['kube-master'][0] + - download_delegate != "localhost" + - download_run_once|bool + - download.enabled|bool + - download.container|bool + - saved.changed - name: Download | upload container images to nodes synchronize: @@ -153,10 +197,21 @@ until: get_task|succeeded retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and inventory_hostname != groups['kube-master'][0] or download_delegate == "localhost") and download_run_once|bool and download.enabled|bool and download.container|bool + when: + - (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and + inventory_hostname != groups['kube-master'][0] or + download_delegate == "localhost") + - download_run_once|bool + - download.enabled|bool + - download.container|bool tags: [upload, upgrade] - name: Download | load container images shell: "{{ docker_bin_dir }}/docker load < {{ fname }}" - when: (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and inventory_hostname != groups['kube-master'][0] or download_delegate == "localhost") and download_run_once|bool and download.enabled|bool and download.container|bool + when: + - (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] and + inventory_hostname != groups['kube-master'][0] or download_delegate == "localhost") + - download_run_once|bool + - download.enabled|bool + - download.container|bool tags: [upload, upgrade] diff --git a/roles/kernel-upgrade/tasks/reboot.yml b/roles/kernel-upgrade/tasks/reboot.yml index 5e01dd8fc..87748f3f6 100644 --- a/roles/kernel-upgrade/tasks/reboot.yml +++ b/roles/kernel-upgrade/tasks/reboot.yml @@ -17,7 +17,7 @@ - set_fact: wait_for_delegate: "{{hostvars['bastion']['ansible_ssh_host']}}" - when: "{{ 'bastion' in groups['all'] }}" + when: "'bastion' in groups['all']" - name: wait for bastion to come back wait_for: @@ -27,7 +27,7 @@ timeout: 300 become: false delegate_to: localhost - when: "is_bastion" + when: is_bastion - name: waiting for server to come back (using bastion if necessary) wait_for: @@ -37,4 +37,4 @@ timeout: 300 become: false delegate_to: "{{ wait_for_delegate }}" - when: "not is_bastion" + when: not is_bastion diff --git a/roles/kubernetes/preinstall/tasks/etchosts.yml b/roles/kubernetes/preinstall/tasks/etchosts.yml index 181fbcb0f..df330be08 100644 --- a/roles/kubernetes/preinstall/tasks/etchosts.yml +++ b/roles/kubernetes/preinstall/tasks/etchosts.yml @@ -17,7 +17,10 @@ line: "{{ loadbalancer_apiserver.address }} {{ apiserver_loadbalancer_domain_name| default('lb-apiserver.kubernetes.local') }}" state: present backup: yes - when: loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined and apiserver_loadbalancer_domain_name is defined + when: + - loadbalancer_apiserver is defined + - loadbalancer_apiserver.address is defined + - apiserver_loadbalancer_domain_name is defined - name: Hosts | localhost ipv4 in hosts file lineinfile: diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 3ae785d6e..2f5bff229 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -43,7 +43,7 @@ path: "{{ kube_config_dir }}" state: directory owner: kube - when: "{{ inventory_hostname in groups['k8s-cluster'] }}" + when: inventory_hostname in groups['k8s-cluster'] tags: [kubelet, k8s-secrets, kube-controller-manager, kube-apiserver, bootstrap-os, apps, network, master, node] - name: Create kubernetes script directory @@ -51,7 +51,7 @@ path: "{{ kube_script_dir }}" state: directory owner: kube - when: "{{ inventory_hostname in groups['k8s-cluster'] }}" + when: "inventory_hostname in groups['k8s-cluster']" tags: [k8s-secrets, bootstrap-os] - name: Create kubernetes manifests directory @@ -59,17 +59,21 @@ path: "{{ kube_manifest_dir }}" state: directory owner: kube - when: "{{ inventory_hostname in groups['k8s-cluster'] }}" + when: "inventory_hostname in groups['k8s-cluster']" tags: [kubelet, bootstrap-os, master, node] - name: check cloud_provider value fail: msg: "If set the 'cloud_provider' var must be set either to 'generic', 'gce', 'aws', 'azure', 'openstack' or 'vsphere'" - when: cloud_provider is defined and cloud_provider not in ['generic', 'gce', 'aws', 'azure', 'openstack', 'vsphere'] + when: + - cloud_provider is defined + - cloud_provider not in ['generic', 'gce', 'aws', 'azure', 'openstack', 'vsphere'] tags: [cloud-provider, facts] - include: "{{ cloud_provider }}-credential-check.yml" - when: cloud_provider is defined and cloud_provider in [ 'openstack', 'azure', 'vsphere' ] + when: + - cloud_provider is defined + - cloud_provider in [ 'openstack', 'azure', 'vsphere' ] tags: [cloud-provider, facts] - name: Create cni directories @@ -80,7 +84,9 @@ with_items: - "/etc/cni/net.d" - "/opt/cni/bin" - when: kube_network_plugin in ["calico", "weave", "canal"] and "{{ inventory_hostname in groups['k8s-cluster'] }}" + when: + - kube_network_plugin in ["calico", "weave", "canal"] + - inventory_hostname in groups['k8s-cluster'] tags: [network, calico, weave, canal, bootstrap-os] - name: Update package management cache (YUM) @@ -91,7 +97,9 @@ until: yum_task_result|succeeded retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: ansible_pkg_mgr == 'yum' and not is_atomic + when: + - ansible_pkg_mgr == 'yum' + - not is_atomic tags: bootstrap-os - name: Install latest version of python-apt for Debian distribs @@ -109,14 +117,17 @@ until: dnf_task_result|succeeded retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: ansible_distribution == "Fedora" and - ansible_distribution_major_version > 21 + when: + - ansible_distribution == "Fedora" + - ansible_distribution_major_version > 21 changed_when: False tags: bootstrap-os - name: Install epel-release on RedHat/CentOS shell: rpm -qa | grep epel-release || rpm -ivh {{ epel_rpm_download_url }} - when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic + when: + - ansible_distribution in ["CentOS","RedHat"] + - not is_atomic register: epel_task_result until: epel_task_result|succeeded retries: 4 @@ -149,7 +160,9 @@ selinux: policy: targeted state: permissive - when: ansible_os_family == "RedHat" and slc.stat.exists == True + when: + - ansible_os_family == "RedHat" + - slc.stat.exists == True changed_when: False tags: bootstrap-os @@ -159,7 +172,9 @@ line: "precedence ::ffff:0:0/96 100" state: present backup: yes - when: disable_ipv6_dns and not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + when: + - disable_ipv6_dns + - not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] tags: bootstrap-os - name: set default sysctl file path @@ -176,7 +191,9 @@ - name: Change sysctl file path to link source if linked set_fact: sysctl_file_path: "{{sysctl_file_stat.stat.lnk_source}}" - when: sysctl_file_stat.stat.islnk is defined and sysctl_file_stat.stat.islnk + when: + - sysctl_file_stat.stat.islnk is defined + - sysctl_file_stat.stat.islnk tags: bootstrap-os - name: Enable ip forwarding @@ -193,22 +210,33 @@ dest: "{{ kube_config_dir }}/cloud_config" group: "{{ kube_cert_group }}" mode: 0640 - when: inventory_hostname in groups['k8s-cluster'] and cloud_provider is defined and cloud_provider in [ 'openstack', 'azure', 'vsphere' ] + when: + - inventory_hostname in groups['k8s-cluster'] + - cloud_provider is defined + - cloud_provider in [ 'openstack', 'azure', 'vsphere' ] tags: [cloud-provider] - include: etchosts.yml tags: [bootstrap-os, etchosts] - include: resolvconf.yml - when: dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' + when: + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' tags: [bootstrap-os, resolvconf] - include: dhclient-hooks.yml - when: dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' and not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + when: + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] tags: [bootstrap-os, resolvconf] - include: dhclient-hooks-undo.yml - when: dns_mode != 'none' and resolvconf_mode != 'host_resolvconf' and not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + when: + - dns_mode != 'none' + - resolvconf_mode != 'host_resolvconf' + - not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] tags: [bootstrap-os, resolvconf] - name: Check if we are running inside a Azure VM @@ -218,7 +246,7 @@ tags: bootstrap-os - include: growpart-azure-centos-7.yml - when: azure_check.stat.exists and - ansible_distribution in ["CentOS","RedHat"] + when: + - azure_check.stat.exists + - ansible_distribution in ["CentOS","RedHat"] tags: bootstrap-os - diff --git a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml index 0561d6581..b02120ccb 100644 --- a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml +++ b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml @@ -6,7 +6,7 @@ with_items: "{{ groups['kube-master'] }}" - include: ../../../vault/tasks/shared/sync_file.yml - vars: + vars: sync_file: "{{ item }}" sync_file_dir: "{{ kube_cert_dir }}" sync_file_group: "{{ kube_cert_group }}" @@ -38,7 +38,7 @@ set_fact: kube_api_certs_needed: "{{ item.path }}" with_items: "{{ sync_file_results|d([]) }}" - when: "{{ item.no_srcs }}" + when: item.no_srcs - name: sync_kube_master_certs | Unset sync_file_results after apiserver cert set_fact: @@ -46,7 +46,7 @@ - include: ../../../vault/tasks/shared/sync_file.yml - vars: + vars: sync_file: ca.pem sync_file_dir: "{{ kube_cert_dir }}" sync_file_group: "{{ kube_cert_group }}" diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index 2f3096bf3..eda9c2934 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -56,7 +56,7 @@ retries: 4 delay: "{{ retry_stagger | random + 3 }}" changed_when: false - when: "{{ overwrite_hyperkube_cni|bool }}" + when: overwrite_hyperkube_cni|bool tags: [hyperkube, upgrade] - name: Calico | Set cni directory permissions diff --git a/roles/vault/tasks/bootstrap/start_vault_temp.yml b/roles/vault/tasks/bootstrap/start_vault_temp.yml index 161ef92d6..4a5e6bc5e 100644 --- a/roles/vault/tasks/bootstrap/start_vault_temp.yml +++ b/roles/vault/tasks/bootstrap/start_vault_temp.yml @@ -3,7 +3,7 @@ - name: bootstrap/start_vault_temp | Ensure vault-temp isn't already running shell: if docker rm -f {{ vault_temp_container_name }} 2>&1 1>/dev/null;then echo true;else echo false;fi register: vault_temp_stop_check - changed_when: "{{ 'true' in vault_temp_stop_check.stdout }}" + changed_when: "'true' in vault_temp_stop_check.stdout" - name: bootstrap/start_vault_temp | Start single node Vault with file backend command: > From 73d8ef9329511b72fbb7cc9c2fbe09553e0245c5 Mon Sep 17 00:00:00 2001 From: Sergii Golovatiuk Date: Thu, 27 Apr 2017 12:46:18 +0200 Subject: [PATCH 33/47] Switch CI to ansible 2.3.0 Closes: 1253 Signed-off-by: Sergii Golovatiuk --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a6694f24..be43c4f06 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,7 +18,7 @@ variables: # us-west1-a before_script: - - pip install ansible==2.2.1.0 + - pip install ansible==2.3.0 - pip install netaddr - pip install apache-libcloud==0.20.1 - pip install boto==2.9.0 @@ -74,7 +74,7 @@ before_script: - $HOME/.cache before_script: - docker info - - pip install ansible==2.2.1.0 + - pip install ansible==2.3.0 - pip install netaddr - pip install apache-libcloud==0.20.1 - pip install boto==2.9.0 @@ -137,7 +137,7 @@ before_script: if [ "${UPGRADE_TEST}" != "false" ]; then test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml"; test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml"; - pip install ansible==2.2.1.0; + pip install ansible==2.3.0; git checkout "${CI_BUILD_REF}"; ansible-playbook -i inventory/inventory.ini -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER ${SSH_ARGS} From a443af85c9d12bff4a94d6e68552308b55b90007 Mon Sep 17 00:00:00 2001 From: Aleksandr Didenko Date: Fri, 21 Apr 2017 13:00:00 +0200 Subject: [PATCH 34/47] Add support for different tags for netcheck containers Replace 'netcheck_tag' with 'netcheck_version' and add additional 'netcheck_server_tag' and 'netcheck_agent_tag' config options to provide ability to use different tags for server and agent containers. --- roles/download/defaults/main.yml | 8 +++++--- roles/kubernetes-apps/ansible/defaults/main.yml | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 2f5c8358a..c537614d9 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -59,9 +59,11 @@ hyperkube_image_repo: "quay.io/coreos/hyperkube" hyperkube_image_tag: "{{ kube_version }}_coreos.0" pod_infra_image_repo: "gcr.io/google_containers/pause-amd64" pod_infra_image_tag: "{{ pod_infra_version }}" -netcheck_tag: "v1.0" +netcheck_version: "v1.0" netcheck_agent_img_repo: "quay.io/l23network/k8s-netchecker-agent" +netcheck_agent_tag: "{{ netcheck_version }}" netcheck_server_img_repo: "quay.io/l23network/k8s-netchecker-server" +netcheck_server_tag: "{{ netcheck_version }}" weave_kube_image_repo: "weaveworks/weave-kube" weave_kube_image_tag: "{{ weave_version }}" weave_npc_image_repo: "weaveworks/weave-npc" @@ -101,13 +103,13 @@ downloads: netcheck_server: container: true repo: "{{ netcheck_server_img_repo }}" - tag: "{{ netcheck_tag }}" + tag: "{{ netcheck_server_tag }}" sha256: "{{ netcheck_server_digest_checksum|default(None) }}" enabled: "{{ deploy_netchecker|bool }}" netcheck_agent: container: true repo: "{{ netcheck_agent_img_repo }}" - tag: "{{ netcheck_tag }}" + tag: "{{ netcheck_agent_tag }}" sha256: "{{ netcheck_agent_digest_checksum|default(None) }}" enabled: "{{ deploy_netchecker|bool }}" etcd: diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 6d0562fc9..89bdd4277 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -24,8 +24,8 @@ deploy_netchecker: false netchecker_port: 31081 agent_report_interval: 15 netcheck_namespace: default -agent_img: "{{ netcheck_agent_img_repo }}:{{ netcheck_tag }}" -server_img: "{{ netcheck_server_img_repo }}:{{ netcheck_tag }}" +agent_img: "{{ netcheck_agent_img_repo }}:{{ netcheck_agent_tag }}" +server_img: "{{ netcheck_server_img_repo }}:{{ netcheck_server_tag }}" # Limits for netchecker apps netchecker_agent_cpu_limit: 30m From 1e016f0f6686c545bd1b315dd8bb09784ab6314f Mon Sep 17 00:00:00 2001 From: Charles Farquhar Date: Fri, 28 Apr 2017 12:10:23 -0500 Subject: [PATCH 35/47] Fix link from ansible.md to calico.md This commit fixes a broken link from ansible.md to calico.md. --- docs/ansible.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ansible.md b/docs/ansible.md index eb8a60769..4da6edb48 100644 --- a/docs/ansible.md +++ b/docs/ansible.md @@ -27,7 +27,7 @@ not _kube-node_. There are also two special groups: -* **calico-rr** : explained for [advanced Calico networking cases](docs/calico.md) +* **calico-rr** : explained for [advanced Calico networking cases](calico.md) * **bastion** : configure a bastion host if your nodes are not directly reachable Below is a complete inventory example: From fe7c2709f9108b4a5e03672fd93299f2296f759b Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Fri, 28 Apr 2017 13:40:54 -0400 Subject: [PATCH 36/47] mount os-release to ensure the node's OS is what's seen in k8s api --- roles/kubernetes/node/templates/kubelet-container.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/kubernetes/node/templates/kubelet-container.j2 b/roles/kubernetes/node/templates/kubelet-container.j2 index 75d067cf6..b5b89461a 100644 --- a/roles/kubernetes/node/templates/kubelet-container.j2 +++ b/roles/kubernetes/node/templates/kubelet-container.j2 @@ -25,6 +25,7 @@ -v /var/lib/cni:/var/lib/cni:shared \ -v /var/run:/var/run:rw \ -v {{kube_config_dir}}:{{kube_config_dir}}:ro \ + -v /etc/os-release:/etc/os-release \ {{ hyperkube_image_repo }}:{{ hyperkube_image_tag}} \ ./hyperkube kubelet \ "$@" From 752c5ef5cf247d69dc217ab4e7ed2bf65974926d Mon Sep 17 00:00:00 2001 From: Brad Beam Date: Tue, 25 Apr 2017 12:17:56 -0500 Subject: [PATCH 37/47] Updating calico to v2.1.4 --- roles/download/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 2f5c8358a..9484d00a2 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -22,8 +22,8 @@ kube_version: v1.6.1 etcd_version: v3.0.17 #TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download -calico_version: "v1.1.0" -calico_cni_version: "v1.6.2" +calico_version: "v1.1.3" +calico_cni_version: "v1.7.0" calico_policy_version: "v0.5.4" weave_version: 1.8.2 flannel_version: v0.6.2 From f608e9e4f803ba8f6187e7254f048dbea3103bc3 Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Fri, 28 Apr 2017 17:45:10 -0400 Subject: [PATCH 38/47] add for rkt as well --- roles/kubernetes/node/templates/kubelet.rkt.service.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 index be8a13dbf..0b0543ea5 100644 --- a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 @@ -20,6 +20,7 @@ ExecStartPre=-/bin/mkdir -p /var/lib/kubelet EnvironmentFile={{kube_config_dir}}/kubelet.env # stage1-fly mounts /proc /sys /dev so no need to duplicate the mounts ExecStart=/usr/bin/rkt run \ + --volume os-release,kind=host,source=/etc/os-release \ --volume dns,kind=host,source=/etc/resolv.conf \ --volume etc-kubernetes,kind=host,source={{ kube_config_dir }},readOnly=false \ --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ @@ -39,6 +40,7 @@ ExecStart=/usr/bin/rkt run \ --mount volume=opt-cni,target=/opt/cni \ --mount volume=var-lib-cni,target=/var/lib/cni \ {% endif %} + --mount volume=os-release,target=/etc/os-release \ --mount volume=dns,target=/etc/resolv.conf \ --mount volume=etc-kubernetes,target={{ kube_config_dir }} \ --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ From 755c20f2f99508ba50059a71645d418ac71402b5 Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Mon, 1 May 2017 14:51:40 -0400 Subject: [PATCH 39/47] ensure the /etc/os-release is mounted read only --- roles/kubernetes/node/templates/kubelet-container.j2 | 2 +- roles/kubernetes/node/templates/kubelet.rkt.service.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/kubernetes/node/templates/kubelet-container.j2 b/roles/kubernetes/node/templates/kubelet-container.j2 index b5b89461a..94c7f79a5 100644 --- a/roles/kubernetes/node/templates/kubelet-container.j2 +++ b/roles/kubernetes/node/templates/kubelet-container.j2 @@ -25,7 +25,7 @@ -v /var/lib/cni:/var/lib/cni:shared \ -v /var/run:/var/run:rw \ -v {{kube_config_dir}}:{{kube_config_dir}}:ro \ - -v /etc/os-release:/etc/os-release \ + -v /etc/os-release:/etc/os-release:ro \ {{ hyperkube_image_repo }}:{{ hyperkube_image_tag}} \ ./hyperkube kubelet \ "$@" diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 index 0b0543ea5..5f8351458 100644 --- a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 @@ -20,7 +20,7 @@ ExecStartPre=-/bin/mkdir -p /var/lib/kubelet EnvironmentFile={{kube_config_dir}}/kubelet.env # stage1-fly mounts /proc /sys /dev so no need to duplicate the mounts ExecStart=/usr/bin/rkt run \ - --volume os-release,kind=host,source=/etc/os-release \ + --volume os-release,kind=host,source=/etc/os-release,readOnly=true \ --volume dns,kind=host,source=/etc/resolv.conf \ --volume etc-kubernetes,kind=host,source={{ kube_config_dir }},readOnly=false \ --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ From c71a6bbbc0c84a1a897637d64459b4947c417e8f Mon Sep 17 00:00:00 2001 From: moss2k13 Date: Mon, 8 May 2017 09:27:06 +0200 Subject: [PATCH 40/47] Updated helm installation Added full path for helm --- roles/kubernetes-apps/helm/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/kubernetes-apps/helm/tasks/main.yml b/roles/kubernetes-apps/helm/tasks/main.yml index e90ea2c4a..1d50f8b9b 100644 --- a/roles/kubernetes-apps/helm/tasks/main.yml +++ b/roles/kubernetes-apps/helm/tasks/main.yml @@ -11,7 +11,7 @@ register: helm_container - name: Helm | Install/upgrade helm - command: "helm init --upgrade --tiller-image={{ tiller_image_repo }}:{{ tiller_image_tag }}" + command: "{{ bin_dir }}/helm init --upgrade --tiller-image={{ tiller_image_repo }}:{{ tiller_image_tag }}" when: helm_container.changed - name: Helm | Set up bash completion From 391ec811d54bb65c131be605297cc731999e2f08 Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Fri, 12 May 2017 15:57:24 -0400 Subject: [PATCH 41/47] default to kubedns and set nxdomain in kubedns deployment if that's the dns_mode --- inventory/group_vars/k8s-cluster.yml | 2 +- roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index d304b79a9..606d271d4 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -98,7 +98,7 @@ cluster_name: cluster.local # Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods ndots: 2 # Can be dnsmasq_kubedns, kubedns or none -dns_mode: dnsmasq_kubedns +dns_mode: kubedns # Can be docker_dns, host_resolvconf or none resolvconf_mode: docker_dns # Deploy netchecker app to verify DNS resolve as an HTTP service diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml index 1bae177d3..afb5b61ad 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml +++ b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml @@ -82,6 +82,9 @@ spec: - --server=127.0.0.1#10053 {% if kube_log_level == '4' %} - --log-queries +{% endif %} +{% if dns_mode == 'kubedns' %} + - --local=/{{ bogus_domains }} {% endif %} ports: - containerPort: 53 From bfe64ca38221442c4544a66b1e327262827ebea6 Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Fri, 12 May 2017 17:16:49 -0400 Subject: [PATCH 42/47] remove conditional --- roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml index afb5b61ad..a2150cc70 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml +++ b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml @@ -83,9 +83,7 @@ spec: {% if kube_log_level == '4' %} - --log-queries {% endif %} -{% if dns_mode == 'kubedns' %} - --local=/{{ bogus_domains }} -{% endif %} ports: - containerPort: 53 name: dns From 3637afb3a349f94fb1a279da74f8bcb3fa62a9c0 Mon Sep 17 00:00:00 2001 From: Hui Kang Date: Sat, 13 May 2017 22:34:27 -0400 Subject: [PATCH 43/47] Update the kubernete and docker verion in readme - kubernetes v1.6.1 - docker v1.13.1 Signed-off-by: Hui Kang --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 02bdb72a4..a545608d5 100644 --- a/README.md +++ b/README.md @@ -50,13 +50,13 @@ Note: Upstart/SysV init based OS types are not supported. Versions of supported components -------------------------------- -[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.5.1
+[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.6.1
[etcd](https://github.com/coreos/etcd/releases) v3.0.17
[flanneld](https://github.com/coreos/flannel/releases) v0.6.2
[calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.23.0
[canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
[weave](http://weave.works/) v1.8.2
-[docker](https://www.docker.com/) v1.12.5
+[docker](https://www.docker.com/) v1.13.1
[rkt](https://coreos.com/rkt/docs/latest/) v1.21.0
Note: rkt support as docker alternative is limited to control plane (etcd and From 5ccae0d60d281ae06776c9cc03fed075a2b86a1d Mon Sep 17 00:00:00 2001 From: Spencer Smith Date: Tue, 16 May 2017 10:07:38 -0400 Subject: [PATCH 44/47] issue raw yum command since we don't have facts in bootstrapping --- roles/bootstrap-os/tasks/bootstrap-centos.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/roles/bootstrap-os/tasks/bootstrap-centos.yml b/roles/bootstrap-os/tasks/bootstrap-centos.yml index dfc117b29..c9233dfb1 100644 --- a/roles/bootstrap-os/tasks/bootstrap-centos.yml +++ b/roles/bootstrap-os/tasks/bootstrap-centos.yml @@ -15,13 +15,4 @@ when: fastestmirror.stat.exists - name: Install packages requirements for bootstrap - action: - module: "{{ ansible_pkg_mgr }}" - name: "{{ item }}" - state: latest - register: bs_pkgs_task_result - until: bs_pkgs_task_result|succeeded - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - with_items: "libselinux-python" - + raw: yum -y install libselinux-python From 9f38112cc0e86563022f130b7053045b5d656f2d Mon Sep 17 00:00:00 2001 From: Brad Beam Date: Tue, 16 May 2017 15:28:39 -0500 Subject: [PATCH 45/47] Removing old sysv reference --- roles/kargo-defaults/defaults/main.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/roles/kargo-defaults/defaults/main.yaml b/roles/kargo-defaults/defaults/main.yaml index e1a52f22e..373152a92 100644 --- a/roles/kargo-defaults/defaults/main.yaml +++ b/roles/kargo-defaults/defaults/main.yaml @@ -42,9 +42,6 @@ kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" kube_manifest_dir: "{{ kube_config_dir }}/manifests" system_namespace: kube-system -# Logging directory (sysvinit systems) -kube_log_dir: "/var/log/kubernetes" - # This is where all the cert scripts and certs will be located kube_cert_dir: "{{ kube_config_dir }}/ssl" From db0ff8762c50314c66b46d365b8982a4ccf23e09 Mon Sep 17 00:00:00 2001 From: Brad Beam Date: Tue, 16 May 2017 15:43:29 -0500 Subject: [PATCH 46/47] Fixing typo in kubelet cluster-dns and cluster-domain flags --- roles/kubernetes/node/templates/kubelet.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index 8de1e63e9..d2959b8a6 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -19,13 +19,13 @@ KUBELET_HOSTNAME="--hostname-override={{ ansible_hostname }}" {# DNS settings for kubelet #} {% if dns_mode == 'kubedns' %} -{% set kubelet_args_cluster_dns %}--cluster_dns={{ skydns_server }}{% endset %} +{% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }}{% endset %} {% elif dns_mode == 'dnsmasq_kubedns' %} -{% set kubelet_args_cluster_dns %}--cluster_dns={{ dns_server }}{% endset %} +{% set kubelet_args_cluster_dns %}--cluster-dns={{ dns_server }}{% endset %} {% else %} {% set kubelet_args_cluster_dns %}{% endset %} {% endif %} -{% set kubelet_args_dns %}{{ kubelet_args_cluster_dns }} --cluster_domain={{ dns_domain }} --resolv-conf={{ kube_resolv_conf }}{% endset %} +{% set kubelet_args_dns %}{{ kubelet_args_cluster_dns }} --cluster-domain={{ dns_domain }} --resolv-conf={{ kube_resolv_conf }}{% endset %} {# Location of the apiserver #} {% set kubelet_args_kubeconfig %}--kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --require-kubeconfig{% endset %} From 3bb8fb6b3ebf2c40f449e39a4dc751d003c9d24c Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 12 May 2017 17:32:37 +0300 Subject: [PATCH 47/47] Add host-based kubelet deployment Kubelet gets copied from hyperkube container and run locally. --- roles/kubernetes/node/defaults/main.yml | 5 +++- roles/kubernetes/node/tasks/install_host.yml | 10 +++++++ .../node/templates/kubelet.docker.service.j2 | 5 ++-- .../node/templates/kubelet.host.service.j2 | 30 +++++++++++++++++++ 4 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 roles/kubernetes/node/tasks/install_host.yml create mode 100644 roles/kubernetes/node/templates/kubelet.host.service.j2 diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 7f1e6f4a0..7ef6d01e0 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -1,3 +1,6 @@ +# Valid options: docker (default), rkt, or host +kubelet_deployment_type: docker + # change to 0.0.0.0 to enable insecure access from anywhere (not recommended) kube_apiserver_insecure_bind_address: 127.0.0.1 @@ -47,4 +50,4 @@ kube_apiserver_node_port_range: "30000-32767" kubelet_load_modules: false ##Support custom flags to be passed to kubelet -kubelet_custom_flags: [] \ No newline at end of file +kubelet_custom_flags: [] diff --git a/roles/kubernetes/node/tasks/install_host.yml b/roles/kubernetes/node/tasks/install_host.yml new file mode 100644 index 000000000..e80b20498 --- /dev/null +++ b/roles/kubernetes/node/tasks/install_host.yml @@ -0,0 +1,10 @@ +--- +- name: install | Copy kubelet from hyperkube container + command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -f /hyperkube /systembindir/kubelet" + register: kubelet_task_result + until: kubelet_task_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + tags: [hyperkube, upgrade] + notify: restart kubelet diff --git a/roles/kubernetes/node/templates/kubelet.docker.service.j2 b/roles/kubernetes/node/templates/kubelet.docker.service.j2 index e3bf40878..cf79f6fa4 100644 --- a/roles/kubernetes/node/templates/kubelet.docker.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.docker.service.j2 @@ -23,10 +23,11 @@ ExecStart={{ bin_dir }}/kubelet \ $DOCKER_SOCKET \ $KUBELET_NETWORK_PLUGIN \ $KUBELET_CLOUDPROVIDER -ExecStartPre=-{{ docker_bin_dir }}/docker rm -f kubelet -ExecReload={{ docker_bin_dir }}/docker restart kubelet Restart=always RestartSec=10s +ExecStartPre=-{{ docker_bin_dir }}/docker rm -f kubelet +ExecReload={{ docker_bin_dir }}/docker restart kubelet + [Install] WantedBy=multi-user.target diff --git a/roles/kubernetes/node/templates/kubelet.host.service.j2 b/roles/kubernetes/node/templates/kubelet.host.service.j2 new file mode 100644 index 000000000..71a9da8c3 --- /dev/null +++ b/roles/kubernetes/node/templates/kubelet.host.service.j2 @@ -0,0 +1,30 @@ +[Unit] +Description=Kubernetes Kubelet Server +Documentation=https://github.com/GoogleCloudPlatform/kubernetes +{% if kube_network_plugin is defined and kube_network_plugin == "calico" %} +After=docker.service docker.socket calico-node.service +Wants=docker.socket calico-node.service +{% else %} +After=docker.service +Wants=docker.socket +{% endif %} + +[Service] +EnvironmentFile={{kube_config_dir}}/kubelet.env +ExecStart={{ bin_dir }}/kubelet \ + $KUBE_LOGTOSTDERR \ + $KUBE_LOG_LEVEL \ + $KUBELET_API_SERVER \ + $KUBELET_ADDRESS \ + $KUBELET_PORT \ + $KUBELET_HOSTNAME \ + $KUBE_ALLOW_PRIV \ + $KUBELET_ARGS \ + $DOCKER_SOCKET \ + $KUBELET_NETWORK_PLUGIN \ + $KUBELET_CLOUDPROVIDER +Restart=always +RestartSec=10s + +[Install] +WantedBy=multi-user.target