From 3ad9e9c5eb7c5ebd110803cd399ef01010626340 Mon Sep 17 00:00:00 2001 From: Henry Finucane Date: Tue, 12 Jun 2018 11:30:18 -0700 Subject: [PATCH 01/44] Fix #2261 by supporting Red Hat's limited PATH Red Hat has this theory that binaries in sbin are too dangerous to be on the default path, but we need them anyway. RH7 has /sbin and /usr/sbin as symlinks, so that is no longer important. I'm adding it to the `PATH` instead of making the path to `modinfo` absolute because I am worried about breaking support for other distributions. --- roles/kubernetes/node/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index f7520caf8..086b94563 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -70,6 +70,8 @@ - name: Verify if br_netfilter module exists shell: "modinfo br_netfilter" + environment: + PATH: "{{ ansible_env.PATH}}:/sbin" # Make sure we can workaround RH's conservative path management register: modinfo_br_netfilter failed_when: modinfo_br_netfilter.rc not in [0, 1] changed_when: false From 6650bc6b25449eaff7d13a3b9c00e6271e35a093 Mon Sep 17 00:00:00 2001 From: rguichard Date: Tue, 21 Aug 2018 13:21:25 +0200 Subject: [PATCH 02/44] fix the output of router_id with the right id --- contrib/terraform/openstack/modules/network/outputs.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/terraform/openstack/modules/network/outputs.tf b/contrib/terraform/openstack/modules/network/outputs.tf index e56a792c2..430cc9181 100644 --- a/contrib/terraform/openstack/modules/network/outputs.tf +++ b/contrib/terraform/openstack/modules/network/outputs.tf @@ -1,4 +1,8 @@ output "router_id" { + value = "${openstack_networking_router_v2.k8s.id}" +} + +output "router_internal_port_id" { value = "${openstack_networking_router_interface_v2.k8s.id}" } From fff48d24ea59e642966e1abd78de7d886839abfe Mon Sep 17 00:00:00 2001 From: guenhter Date: Tue, 14 Nov 2017 11:35:41 +0100 Subject: [PATCH 03/44] Replace the raw rsync command with the synchronize module --- roles/kubernetes/node/tasks/install.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index fe4b6c9c8..f3e1dca40 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -7,8 +7,13 @@ - kubeadm - name: install | Copy kubeadm binary from download dir - command: rsync -piu "{{ local_release_dir }}/kubeadm" "{{ bin_dir }}/kubeadm" - changed_when: false + synchronize: + src: "{{ local_release_dir }}/kubeadm" + dest: "{{ bin_dir }}/kubeadm" + compress: no + owner: no + group: no + delegate_to: "{{ inventory_hostname }}" when: kubeadm_enabled tags: - kubeadm From ca6c5e2a6a46cfce1afbd0ce08c51c8addac0593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E8=B6=85?= Date: Wed, 5 Sep 2018 17:41:34 +0800 Subject: [PATCH 04/44] terraform.tfvars.example is not correct, remove. --- contrib/terraform/aws/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/contrib/terraform/aws/README.md b/contrib/terraform/aws/README.md index 709d0633f..e677869d6 100644 --- a/contrib/terraform/aws/README.md +++ b/contrib/terraform/aws/README.md @@ -22,8 +22,6 @@ export TF_VAR_AWS_SECRET_ACCESS_KEY ="xxx" export TF_VAR_AWS_SSH_KEY_NAME="yyy" export TF_VAR_AWS_DEFAULT_REGION="zzz" ``` -- Rename `contrib/terraform/aws/terraform.tfvars.example` to `terraform.tfvars` - - Update `contrib/terraform/aws/terraform.tfvars` with your data. By default, the Terraform scripts use CoreOS as base image. If you want to change this behaviour, see note "Using other distrib than CoreOs" below. - Create an AWS EC2 SSH Key - Run with `terraform apply --var-file="credentials.tfvars"` or `terraform apply` depending if you exported your AWS credentials From d182d4f97901113395ed80d4702a1a062bd9b17b Mon Sep 17 00:00:00 2001 From: Takashi Okamoto Date: Thu, 6 Sep 2018 15:26:32 -0400 Subject: [PATCH 05/44] absolute path for kubectl. --- roles/win_nodes/kubernetes_patch/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/win_nodes/kubernetes_patch/tasks/main.yml b/roles/win_nodes/kubernetes_patch/tasks/main.yml index 8d88818a5..a6af1fd90 100644 --- a/roles/win_nodes/kubernetes_patch/tasks/main.yml +++ b/roles/win_nodes/kubernetes_patch/tasks/main.yml @@ -16,11 +16,11 @@ # Due to https://github.com/kubernetes/kubernetes/issues/58212 we cannot rely on exit code for "kubectl patch" - name: Check current nodeselector for kube-proxy daemonset - shell: kubectl get ds kube-proxy --namespace=kube-system -o jsonpath='{.spec.template.spec.nodeSelector.beta\.kubernetes\.io/os}' + shell: "{{bin_dir}}/kubectl get ds kube-proxy --namespace=kube-system -o jsonpath='{.spec.template.spec.nodeSelector.beta.kubernetes.io/os}'" register: current_kube_proxy_state - name: Apply nodeselector patch for kube-proxy daemonset - shell: kubectl patch ds kube-proxy --namespace=kube-system --type=strategic -p "$(cat nodeselector-os-linux-patch.json)" + shell: "{{bin_dir}}/kubectl patch ds kube-proxy --namespace=kube-system --type=strategic -p \"$(cat nodeselector-os-linux-patch.json)\"" args: chdir: "{{ kubernetes_user_manifests_path }}" register: patch_kube_proxy_state From d91f9e14e6c828a4d9a31d5c5c9393721467d9d3 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Tue, 11 Sep 2018 15:14:10 +0300 Subject: [PATCH 06/44] Put back legacy support for calico ippools and bgp settings --- roles/network_plugin/calico/tasks/check.yml | 1 + roles/network_plugin/calico/tasks/install.yml | 55 +++++++++++++++---- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/roles/network_plugin/calico/tasks/check.yml b/roles/network_plugin/calico/tasks/check.yml index cfacf12a1..89f204c59 100644 --- a/roles/network_plugin/calico/tasks/check.yml +++ b/roles/network_plugin/calico/tasks/check.yml @@ -22,6 +22,7 @@ that: - calico_version_on_server.stdout|version_compare('v2.6.5', '>=') msg: "Your version of calico is not fresh enough for upgrade" + when: calico_upgrade_enabled - name: "Set upgrade flag when version needs to be updated" set_fact: diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index 4e1f1dc5e..dd72ff323 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -1,5 +1,4 @@ --- - - name: Calico | Write Calico cni config template: src: "cni-calico.conflist.j2" @@ -102,6 +101,25 @@ run_once: true delegate_to: "{{ groups['kube-master'][0] }}" when: + - calico_version_on_server.stdout|version_compare('v3.0.0', '>') or calico_upgrade_enabled + - 'calico_conf.stdout == "0"' + +- name: Calico | Configure calico network pool (legacy) + shell: > + echo ' + { "kind": "ipPool", + "spec": {"disabled": false, "ipip": {"enabled": {{ ipip }}, "mode": "{{ ipip_mode|lower }}"}, + "nat-outgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }}}, + "apiVersion": "v1", + "metadata": {"cidr": "{{ kube_pods_subnet }}"} + }' | {{ bin_dir }}/calicoctl apply -f - + environment: + NO_DEFAULT_POOLS: true + run_once: true + delegate_to: "{{ groups['kube-master'][0] }}" + when: + - calico_version_on_server.stdout|version_compare('v3.0.0', '<') + - not calico_upgrade_enabled - 'calico_conf.stdout == "0"' - name: "Determine nodeToNodeMesh needed state" @@ -112,7 +130,6 @@ - inventory_hostname in groups['k8s-cluster'] run_once: yes - - name: Calico | Set global as_num shell: > echo ' @@ -127,6 +144,24 @@ "asNumber": {{ global_as_num }} }} ' | {{ bin_dir }}/calicoctl --skip-exists create -f - run_once: true delegate_to: "{{ groups['kube-master'][0] }}" + when: + - calico_version_on_server.stdout|version_compare('v3.0.0', '>') or calico_upgrade_enabled + +- name: Calico | Set global as_num (legacy) + command: "{{ bin_dir}}/calicoctl config set asNumber {{ global_as_num }}" + run_once: true + when: + - calico_version_on_server.stdout|version_compare('v3.0.0', '<') + - not calico_upgrade_enabled + +- name: Calico | Disable node mesh (legacy) + command: "{{ bin_dir }}/calicoctl config set nodeToNodeMesh off" + run_once: yes + when: + - calico_version_on_server.stdout|version_compare('v3.0.0', '<') + - not calico_upgrade_enabled + - nodeToMeshEnabled|default(True) + run_once: true - name: Calico | Configure peering with router(s) shell: > @@ -164,10 +199,10 @@ delay: "{{ retry_stagger | random + 3 }}" with_items: "{{ peers|default([]) }}" when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '<') - - not calico_upgrade_enabled - - peer_with_router|default(false) - - inventory_hostname in groups['k8s-cluster'] + - calico_version_on_server.stdout|version_compare('v3.0.0', '<') + - not calico_upgrade_enabled + - peer_with_router|default(false) + - inventory_hostname in groups['k8s-cluster'] - name: Calico | Configure peering with route reflectors shell: > @@ -208,10 +243,10 @@ delay: "{{ retry_stagger | random + 3 }}" with_items: "{{ groups['calico-rr'] | default([]) }}" when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '<') - - not calico_upgrade_enabled - - peer_with_calico_rr|default(false) - - hostvars[item]['cluster_id'] == cluster_id + - calico_version_on_server.stdout|version_compare('v3.0.0', '<') + - not calico_upgrade_enabled + - peer_with_calico_rr|default(false) + - hostvars[item]['cluster_id'] == cluster_id - name: Calico | Create calico manifests From 29c7775ea139fcc634b3a50c3244e2067e90361a Mon Sep 17 00:00:00 2001 From: "Zinin D.A" Date: Tue, 11 Sep 2018 20:07:25 +0300 Subject: [PATCH 07/44] Monitor CoreDNS over svc --- .../ansible/templates/coredns-svc.yml.j2 | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 index 193de10eb..7f08d17ca 100644 --- a/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/coredns-svc.yml.j2 @@ -9,14 +9,21 @@ metadata: kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: "CoreDNS" + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "9153" + prometheus.io/scrape: "true" spec: selector: k8s-app: coredns{{ coredns_ordinal_suffix | default('') }} clusterIP: {{ clusterIP }} ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP From cc79125d3e432ba601c0f9cda2e5402ac167f68f Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Wed, 12 Sep 2018 17:03:55 +0300 Subject: [PATCH 08/44] Update install.yml --- roles/network_plugin/calico/tasks/install.yml | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index dd72ff323..a292e3748 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -151,8 +151,8 @@ command: "{{ bin_dir}}/calicoctl config set asNumber {{ global_as_num }}" run_once: true when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '<') - - not calico_upgrade_enabled + - calico_version_on_server.stdout|version_compare('v3.0.0', '<') + - not calico_upgrade_enabled - name: Calico | Disable node mesh (legacy) command: "{{ bin_dir }}/calicoctl config set nodeToNodeMesh off" @@ -161,7 +161,6 @@ - calico_version_on_server.stdout|version_compare('v3.0.0', '<') - not calico_upgrade_enabled - nodeToMeshEnabled|default(True) - run_once: true - name: Calico | Configure peering with router(s) shell: > @@ -199,10 +198,10 @@ delay: "{{ retry_stagger | random + 3 }}" with_items: "{{ peers|default([]) }}" when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '<') - - not calico_upgrade_enabled - - peer_with_router|default(false) - - inventory_hostname in groups['k8s-cluster'] + - calico_version_on_server.stdout|version_compare('v3.0.0', '<') + - not calico_upgrade_enabled + - peer_with_router|default(false) + - inventory_hostname in groups['k8s-cluster'] - name: Calico | Configure peering with route reflectors shell: > @@ -243,10 +242,10 @@ delay: "{{ retry_stagger | random + 3 }}" with_items: "{{ groups['calico-rr'] | default([]) }}" when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '<') - - not calico_upgrade_enabled - - peer_with_calico_rr|default(false) - - hostvars[item]['cluster_id'] == cluster_id + - calico_version_on_server.stdout|version_compare('v3.0.0', '<') + - not calico_upgrade_enabled + - peer_with_calico_rr|default(false) + - hostvars[item]['cluster_id'] == cluster_id - name: Calico | Create calico manifests From 1091e82327b417abfffd34a1298e283da513c184 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Wed, 12 Sep 2018 22:15:46 +0300 Subject: [PATCH 09/44] Update install.yml --- roles/network_plugin/calico/tasks/install.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index a292e3748..2da09a9aa 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -101,7 +101,7 @@ run_once: true delegate_to: "{{ groups['kube-master'][0] }}" when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '>') or calico_upgrade_enabled + - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') - 'calico_conf.stdout == "0"' - name: Calico | Configure calico network pool (legacy) @@ -145,8 +145,7 @@ run_once: true delegate_to: "{{ groups['kube-master'][0] }}" when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '>') or calico_upgrade_enabled - + - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') - name: Calico | Set global as_num (legacy) command: "{{ bin_dir}}/calicoctl config set asNumber {{ global_as_num }}" run_once: true @@ -181,7 +180,7 @@ with_items: - "{{ peers|default([]) }}" when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '>') or calico_upgrade_enabled + - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') - peer_with_router|default(false) - inventory_hostname in groups['k8s-cluster'] @@ -222,7 +221,7 @@ with_items: - "{{ groups['calico-rr'] | default([]) }}" when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '>') or calico_upgrade_enabled + - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') - peer_with_calico_rr|default(false) - inventory_hostname in groups['k8s-cluster'] - hostvars[item]['cluster_id'] == cluster_id From cd3b30d3bf65108f2bacd98db32950c6c733634c Mon Sep 17 00:00:00 2001 From: Sascha Marcel Schmidt Date: Fri, 7 Sep 2018 14:47:34 +0200 Subject: [PATCH 10/44] fix path to configuration --- contrib/metallb/roles/provision/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/metallb/roles/provision/tasks/main.yml b/contrib/metallb/roles/provision/tasks/main.yml index 57e8eb6bb..9192ceb54 100644 --- a/contrib/metallb/roles/provision/tasks/main.yml +++ b/contrib/metallb/roles/provision/tasks/main.yml @@ -9,8 +9,8 @@ - name: "Kubernetes Apps | Install and configure MetalLB" kube: name: "MetalLB" - filename: "{{ kube_config_dir }}/metallb.yml" kubectl: "{{bin_dir}}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item }}" state: "{{ item.changed | ternary('latest','present') }}" with_items: "{{ rendering.results }}" when: From 1df0b67ec12c9ee62eff90e27f81b87a276716c9 Mon Sep 17 00:00:00 2001 From: rabi Date: Wed, 12 Sep 2018 21:05:28 +0530 Subject: [PATCH 11/44] Add volume and volumeMount for crio-socket This commit fixes #3295 --- .../cilium/templates/cilium-ds.yml.j2 | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 index 7fff7ac0e..5fa75f98f 100755 --- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 @@ -159,9 +159,15 @@ spec: mountPath: /host/opt/cni/bin - name: etc-cni-netd mountPath: /host/etc/cni/net.d +{% if container_manager == 'crio' %} + - name: crio-socket + mountPath: /var/run/crio.sock + readOnly: true +{% else %} - name: docker-socket mountPath: /var/run/docker.sock readOnly: true +{% endif %} - name: etcd-config-path mountPath: /var/lib/etcd-config readOnly: true @@ -183,10 +189,17 @@ spec: - name: bpf-maps hostPath: path: /sys/fs/bpf +{% if container_manager == 'crio' %} + # To read crio events from the node + - name: crio-socket + hostPath: + path: /var/run/crio/crio.sock +{% else %} # To read docker events from the node - name: docker-socket hostPath: path: /var/run/docker.sock +{% endif %} # To install cilium cni plugin in the host - name: cni-path hostPath: From 55d76ea3d815d0548f1555b5ff3736cf3d5326b8 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Thu, 13 Sep 2018 12:04:53 +0300 Subject: [PATCH 12/44] Update install.yml --- roles/network_plugin/calico/tasks/install.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index 2da09a9aa..294b09d3b 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -101,7 +101,7 @@ run_once: true delegate_to: "{{ groups['kube-master'][0] }}" when: - - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') + - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') - 'calico_conf.stdout == "0"' - name: Calico | Configure calico network pool (legacy) @@ -180,7 +180,7 @@ with_items: - "{{ peers|default([]) }}" when: - - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') + - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') - peer_with_router|default(false) - inventory_hostname in groups['k8s-cluster'] @@ -221,7 +221,7 @@ with_items: - "{{ groups['calico-rr'] | default([]) }}" when: - - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') + - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') - peer_with_calico_rr|default(false) - inventory_hostname in groups['k8s-cluster'] - hostvars[item]['cluster_id'] == cluster_id From 730866f431707520fc720378a6da910abec407f7 Mon Sep 17 00:00:00 2001 From: Karol Chrapek Date: Thu, 13 Sep 2018 15:50:42 +0200 Subject: [PATCH 13/44] Update upgrades.md --- docs/upgrades.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/upgrades.md b/docs/upgrades.md index 620e07a6b..5443e067f 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -18,6 +18,11 @@ versions. Here are all version vars for each component: * flannel_version * kubedns_version +Please remember that some components upgrade depends on minimum installed version. +In example **calico version 2.6.5 upgrade to 3.1.1 is upgrading etcd store to etcdv3**. +The kubespray stack upgrade would failed when calico version is below 2.6.5. Please check +components' documentation and always upgrade test environment first. + #### Unsafe upgrade example If you wanted to upgrade just kube_version from v1.4.3 to v1.4.6, you could @@ -86,7 +91,7 @@ for impact to user deployed pods. A deployer may want to upgrade specific components in order to minimize risk or save time. This strategy is not covered by CI as of this writing, so it is -not guaranteed to work. +not guaranteed to work. These commands are useful only for upgrading fully-deployed, healthy, existing hosts. This will definitely not work for undeployed or partially deployed From c83350e59709df4e3200b37d462d5ad16e9d670c Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Thu, 13 Sep 2018 18:05:10 +0300 Subject: [PATCH 14/44] refactor to base on calico_version --- roles/network_plugin/calico/tasks/check.yml | 2 +- roles/network_plugin/calico/tasks/install.yml | 23 ++++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/roles/network_plugin/calico/tasks/check.yml b/roles/network_plugin/calico/tasks/check.yml index 89f204c59..7ddca6418 100644 --- a/roles/network_plugin/calico/tasks/check.yml +++ b/roles/network_plugin/calico/tasks/check.yml @@ -10,7 +10,7 @@ run_once: yes - name: "Get current version of calico cluster version" - shell: "{{ bin_dir }}/calicoctl version | grep 'Cluster Version' | awk '{ print $3}'" + shell: "{{ bin_dir }}/calicoctl version | grep 'Cluster Version:' | awk '{ print $3}'" register: calico_version_on_server run_once: yes delegate_to: "{{ groups['kube-master'][0] }}" diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index 294b09d3b..f2d41f9fc 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -101,8 +101,8 @@ run_once: true delegate_to: "{{ groups['kube-master'][0] }}" when: - - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') - 'calico_conf.stdout == "0"' + - calico_version | version_compare("v3.0.0", ">=") - name: Calico | Configure calico network pool (legacy) shell: > @@ -118,9 +118,8 @@ run_once: true delegate_to: "{{ groups['kube-master'][0] }}" when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '<') - - not calico_upgrade_enabled - 'calico_conf.stdout == "0"' + - calico_version | version_compare("v3.0.0", "<") - name: "Determine nodeToNodeMesh needed state" set_fact: @@ -145,20 +144,19 @@ run_once: true delegate_to: "{{ groups['kube-master'][0] }}" when: - - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') + - calico_version | version_compare('v3.0.0', '>=') + - name: Calico | Set global as_num (legacy) command: "{{ bin_dir}}/calicoctl config set asNumber {{ global_as_num }}" run_once: true when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '<') - - not calico_upgrade_enabled + - calico_version | version_compare('v3.0.0', '<') - name: Calico | Disable node mesh (legacy) command: "{{ bin_dir }}/calicoctl config set nodeToNodeMesh off" run_once: yes when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '<') - - not calico_upgrade_enabled + - calico_version | version_compare('v3.0.0', '<') - nodeToMeshEnabled|default(True) - name: Calico | Configure peering with router(s) @@ -180,7 +178,7 @@ with_items: - "{{ peers|default([]) }}" when: - - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') + - calico_version | version_compare('v3.0.0', '>=') - peer_with_router|default(false) - inventory_hostname in groups['k8s-cluster'] @@ -197,8 +195,7 @@ delay: "{{ retry_stagger | random + 3 }}" with_items: "{{ peers|default([]) }}" when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '<') - - not calico_upgrade_enabled + - calico_version | version_compare('v3.0.0', '<') - peer_with_router|default(false) - inventory_hostname in groups['k8s-cluster'] @@ -221,7 +218,7 @@ with_items: - "{{ groups['calico-rr'] | default([]) }}" when: - - calico_upgrade_enabled or calico_version_on_server.stdout|version_compare('v3.0.0', '>') + - calico_version | version_compare('v3.0.0', '>=') - peer_with_calico_rr|default(false) - inventory_hostname in groups['k8s-cluster'] - hostvars[item]['cluster_id'] == cluster_id @@ -241,7 +238,7 @@ delay: "{{ retry_stagger | random + 3 }}" with_items: "{{ groups['calico-rr'] | default([]) }}" when: - - calico_version_on_server.stdout|version_compare('v3.0.0', '<') + - calico_version | version_compare('v3.0.0', '<') - not calico_upgrade_enabled - peer_with_calico_rr|default(false) - hostvars[item]['cluster_id'] == cluster_id From 8becd905b8d03d396945b1b1e921dbf817035369 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Tue, 11 Sep 2018 14:57:19 +0300 Subject: [PATCH 15/44] Fixes for upgrade mode Uses correct flag for draining with a pod selector Verifies minimum kubectl version for compatibility --- roles/upgrade/pre-upgrade/defaults/main.yml | 2 +- roles/upgrade/pre-upgrade/tasks/main.yml | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/roles/upgrade/pre-upgrade/defaults/main.yml b/roles/upgrade/pre-upgrade/defaults/main.yml index 220c7a39e..bb381cbde 100644 --- a/roles/upgrade/pre-upgrade/defaults/main.yml +++ b/roles/upgrade/pre-upgrade/defaults/main.yml @@ -1,5 +1,5 @@ --- drain_grace_period: 300 drain_timeout: 360s -drain_label_selector: "" +drain_pod_selector: "" drain_nodes: true diff --git a/roles/upgrade/pre-upgrade/tasks/main.yml b/roles/upgrade/pre-upgrade/tasks/main.yml index 1b659b8fd..6eaa041c1 100644 --- a/roles/upgrade/pre-upgrade/tasks/main.yml +++ b/roles/upgrade/pre-upgrade/tasks/main.yml @@ -32,6 +32,24 @@ delegate_to: "{{ groups['kube-master'][0] }}" when: needs_cordoning +- name: Check kubectl version + command: kubectl version --client --short + register: kubectl_version + delegate_to: "{{ groups['kube-master'][0] }}" + run_once: yes + when: + - drain_nodes + - needs_cordoning + - 'drain_pod_selector != ""' + +- name: Ensure minimum version for drain label selector if necessary + assert: + that: "kubectl_version.stdout.split(' ')[-1] | version_compare('v1.10.0', '>=')" + when: + - drain_nodes + - needs_cordoning + - 'drain_pod_selector != ""' + - name: Drain node command: >- {{ bin_dir }}/kubectl drain @@ -40,7 +58,7 @@ --grace-period {{ drain_grace_period }} --timeout {{ drain_timeout }} --delete-local-data {{ inventory_hostname }} - {% if drain_label_selector != "" %}--selector '{{ drain_label_selector }}'{% endif %} + {% if drain_pod_selector != "" %}--pod-selector '{{ drain_pod_selector }}'{% endif %} delegate_to: "{{ groups['kube-master'][0] }}" when: - drain_nodes From 166da2ffd05090ea25521300c3f1491e0605fb1c Mon Sep 17 00:00:00 2001 From: Erwan Miran Date: Thu, 13 Sep 2018 17:21:22 +0200 Subject: [PATCH 16/44] Document podsecuritypolicy_enabled and kubernetes_audit --- docs/vars.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/vars.md b/docs/vars.md index 41248b036..1a0756169 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -126,6 +126,17 @@ node_labels: label1_name: label1_value label2_name: label2_value ``` +* *podsecuritypolicy_enabled* - When set to `true`, enables the PodSecurityPolicy admission controller and defines two policies `privileged` (applying to all resources in `kube-system` namespace and kubelet) and `restricted` (applying all other namespaces). + Addons deployed in kube-system namespaces are handled. +* *kubernetes_audit* - When set to `true`, enables Auditing. + The auditing parameters can be tuned via the following variables (which default values are shown below): + * `audit_log_path`: /var/log/audit/kube-apiserver-audit.log + * `audit_log_maxage`: 30 + * `audit_log_maxbackups`: 1 + * `audit_log_maxsize`: 100 + * `audit_policy_file`: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml" + + By default, the `audit_policy_file` contains [default rules](https://github.com/kubernetes-incubator/kubespray/blob/master/roles/kubernetes/master/templates/apiserver-audit-policy.yaml.j2) that can be overriden with the `audit_policy_custom_rules` variable. ##### Custom flags for Kube Components For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. Example: From d94b7fd57cbbada4a3ec88e87be4b17ae03b835f Mon Sep 17 00:00:00 2001 From: Antoine Legrand <2t.antoine@gmail.com> Date: Thu, 13 Sep 2018 16:49:23 +0200 Subject: [PATCH 17/44] Don't download binary if docker is selected --- roles/download/defaults/main.yml | 69 +++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index c27224b14..406bb92db 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -33,6 +33,7 @@ image_arch: "{{host_architecture | default('amd64')}}" kube_version: v1.11.3 kubeadm_version: "{{ kube_version }}" etcd_version: v3.2.18 + # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download calico_version: "v3.1.3" @@ -40,8 +41,10 @@ calico_ctl_version: "v3.1.3" calico_cni_version: "v3.1.3" calico_policy_version: "v3.1.3" calico_rr_version: "v0.6.1" + flannel_version: "v0.10.0" flannel_cni_version: "v0.3.0" + vault_version: 0.10.1 weave_version: "2.4.0" pod_infra_version: 3.1 @@ -55,9 +58,9 @@ etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_ver hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64/hyperkube" # Checksums -etcd_checksum: b729db0732448064271ea6fdcb901773c4fe917763ca07776f22d0e5e0bd4097 -hyperkube_checksum: dac8da16dd6688e52b5dc510f5dd0a20b54350d52fb27ceba2f018ba2c8be692 -kubeadm_checksum: 422a7a32ed9a7b1eaa2a4f9d121674dfbe80eb41e206092c13017d097f75aaec +etcd_binary_checksum: b729db0732448064271ea6fdcb901773c4fe917763ca07776f22d0e5e0bd4097 +hyperkube_binary_checksum: dac8da16dd6688e52b5dc510f5dd0a20b54350d52fb27ceba2f018ba2c8be692 +kubeadm_binary_checksum: 422a7a32ed9a7b1eaa2a4f9d121674dfbe80eb41e206092c13017d097f75aaec vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188 # Containers @@ -174,6 +177,7 @@ downloads: sha256: "{{ netcheck_server_digest_checksum|default(None) }}" groups: - k8s-cluster + netcheck_agent: enabled: "{{ deploy_netchecker }}" container: true @@ -182,20 +186,16 @@ downloads: sha256: "{{ netcheck_agent_digest_checksum|default(None) }}" groups: - k8s-cluster + etcd: + container: "{{ etcd_deployment_type != 'host' }}" + file: "{{ etcd_deployment_type == 'host' }}" enabled: true - container: true - repo: "{{ etcd_image_repo }}" - tag: "{{ etcd_image_tag }}" - sha256: "{{ etcd_digest_checksum|default(None) }}" - groups: - - etcd - etcd_file: - enabled: true - file: true version: "{{ etcd_version }}" dest: "etcd-{{ etcd_version }}-linux-amd64.tar.gz" - sha256: "{{ etcd_checksum }}" + repo: "{{ etcd_image_repo }}" + tag: "{{ etcd_image_tag }}" + sha256: "{{ etcd_binary_checksum if etcd_deployment_type == 'host' else etcd_digest_checksum|d(None) }}" source_url: "{{ etcd_download_url }}" url: "{{ etcd_download_url }}" unarchive: true @@ -203,12 +203,13 @@ downloads: mode: "0755" groups: - etcd + kubeadm: enabled: "{{ kubeadm_enabled }}" file: true version: "{{ kubeadm_version }}" dest: "kubeadm" - sha256: "{{ kubeadm_checksum }}" + sha256: "{{ kubeadm_binary_checksum }}" source_url: "{{ kubeadm_download_url }}" url: "{{ kubeadm_download_url }}" unarchive: false @@ -216,6 +217,7 @@ downloads: mode: "0755" groups: - k8s-cluster + hyperkube: enabled: true container: true @@ -224,12 +226,13 @@ downloads: sha256: "{{ hyperkube_digest_checksum|default(None) }}" groups: - k8s-cluster + hyperkube_file: enabled: true file: true version: "{{ kube_version }}" dest: "hyperkube" - sha256: "{{ hyperkube_checksum }}" + sha256: "{{ hyperkube_binary_checksum }}" source_url: "{{ hyperkube_download_url }}" url: "{{ hyperkube_download_url }}" unarchive: false @@ -237,6 +240,7 @@ downloads: mode: "0755" groups: - k8s-cluster + cilium: enabled: "{{ kube_network_plugin == 'cilium' }}" container: true @@ -245,6 +249,7 @@ downloads: sha256: "{{ cilium_digest_checksum|default(None) }}" groups: - k8s-cluster + flannel: enabled: "{{ kube_network_plugin == 'flannel' or kube_network_plugin == 'canal' }}" container: true @@ -253,6 +258,7 @@ downloads: sha256: "{{ flannel_digest_checksum|default(None) }}" groups: - k8s-cluster + flannel_cni: enabled: "{{ kube_network_plugin == 'flannel' }}" container: true @@ -261,6 +267,7 @@ downloads: sha256: "{{ flannel_cni_digest_checksum|default(None) }}" groups: - k8s-cluster + calicoctl: enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}" container: true @@ -269,6 +276,7 @@ downloads: sha256: "{{ calicoctl_digest_checksum|default(None) }}" groups: - k8s-cluster + calico_node: enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}" container: true @@ -277,6 +285,7 @@ downloads: sha256: "{{ calico_node_digest_checksum|default(None) }}" groups: - k8s-cluster + calico_cni: enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}" container: true @@ -285,6 +294,7 @@ downloads: sha256: "{{ calico_cni_digest_checksum|default(None) }}" groups: - k8s-cluster + calico_policy: enabled: "{{ enable_network_policy or kube_network_plugin == 'canal' }}" container: true @@ -293,6 +303,7 @@ downloads: sha256: "{{ calico_policy_digest_checksum|default(None) }}" groups: - k8s-cluster + calico_rr: enabled: "{{ peer_with_calico_rr is defined and peer_with_calico_rr and kube_network_plugin == 'calico' }}" container: true @@ -301,6 +312,7 @@ downloads: sha256: "{{ calico_rr_digest_checksum|default(None) }}" groups: - calico-rr + weave_kube: enabled: "{{ kube_network_plugin == 'weave' }}" container: true @@ -309,6 +321,7 @@ downloads: sha256: "{{ weave_kube_digest_checksum|default(None) }}" groups: - k8s-cluster + weave_npc: enabled: "{{ kube_network_plugin == 'weave' }}" container: true @@ -317,6 +330,7 @@ downloads: sha256: "{{ weave_npc_digest_checksum|default(None) }}" groups: - k8s-cluster + contiv: enabled: "{{ kube_network_plugin == 'contiv' }}" container: true @@ -325,6 +339,7 @@ downloads: sha256: "{{ contiv_digest_checksum|default(None) }}" groups: - k8s-cluster + contiv_auth_proxy: enabled: "{{ kube_network_plugin == 'contiv' }}" container: true @@ -333,6 +348,7 @@ downloads: sha256: "{{ contiv_auth_proxy_digest_checksum|default(None) }}" groups: - k8s-cluster + contiv_etcd_init: enabled: "{{ kube_network_plugin == 'contiv' }}" container: true @@ -341,6 +357,7 @@ downloads: sha256: "{{ contiv_etcd_init_digest_checksum|default(None) }}" groups: - k8s-cluster + pod_infra: enabled: true container: true @@ -349,6 +366,7 @@ downloads: sha256: "{{ pod_infra_digest_checksum|default(None) }}" groups: - k8s-cluster + install_socat: enabled: "{{ ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] }}" container: true @@ -357,6 +375,7 @@ downloads: sha256: "{{ install_socat_digest_checksum|default(None) }}" groups: - k8s-cluster + nginx: enabled: "{{ loadbalancer_apiserver_localhost }}" container: true @@ -365,6 +384,7 @@ downloads: sha256: "{{ nginx_digest_checksum|default(None) }}" groups: - kube-node + dnsmasq: enabled: "{{ dns_mode == 'dnsmasq_kubedns' }}" container: true @@ -373,6 +393,7 @@ downloads: sha256: "{{ dnsmasq_digest_checksum|default(None) }}" groups: - kube-node + kubedns: enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" container: true @@ -381,6 +402,7 @@ downloads: sha256: "{{ kubedns_digest_checksum|default(None) }}" groups: - kube-node + coredns: enabled: "{{ dns_mode in ['coredns', 'coredns_dual'] }}" container: true @@ -389,6 +411,7 @@ downloads: sha256: "{{ coredns_digest_checksum|default(None) }}" groups: - kube-node + dnsmasq_nanny: enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" container: true @@ -397,6 +420,7 @@ downloads: sha256: "{{ dnsmasq_nanny_digest_checksum|default(None) }}" groups: - kube-node + dnsmasq_sidecar: enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" container: true @@ -405,6 +429,7 @@ downloads: sha256: "{{ dnsmasq_sidecar_digest_checksum|default(None) }}" groups: - kube-node + kubednsautoscaler: enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}" container: true @@ -413,12 +438,14 @@ downloads: sha256: "{{ kubednsautoscaler_digest_checksum|default(None) }}" groups: - kube-node + testbox: enabled: false container: true repo: "{{ test_image_repo }}" tag: "{{ test_image_tag }}" sha256: "{{ testbox_digest_checksum|default(None) }}" + elasticsearch: enabled: "{{ efk_enabled }}" container: true @@ -427,6 +454,7 @@ downloads: sha256: "{{ elasticsearch_digest_checksum|default(None) }}" groups: - kube-node + fluentd: enabled: "{{ efk_enabled }}" container: true @@ -435,6 +463,7 @@ downloads: sha256: "{{ fluentd_digest_checksum|default(None) }}" groups: - kube-node + kibana: enabled: "{{ efk_enabled }}" container: true @@ -443,6 +472,7 @@ downloads: sha256: "{{ kibana_digest_checksum|default(None) }}" groups: - kube-node + helm: enabled: "{{ helm_enabled }}" container: true @@ -451,6 +481,7 @@ downloads: sha256: "{{ helm_digest_checksum|default(None) }}" groups: - kube-node + tiller: enabled: "{{ helm_enabled }}" container: true @@ -459,6 +490,7 @@ downloads: sha256: "{{ tiller_digest_checksum|default(None) }}" groups: - kube-node + vault: enabled: "{{ cert_management == 'vault' }}" container: "{{ vault_deployment_type != 'host' }}" @@ -475,6 +507,7 @@ downloads: version: "{{ vault_version }}" groups: - vault + registry: enabled: "{{ registry_enabled }}" container: true @@ -483,6 +516,7 @@ downloads: sha256: "{{ registry_digest_checksum|default(None) }}" groups: - kube-node + registry_proxy: enabled: "{{ registry_enabled }}" container: true @@ -491,6 +525,7 @@ downloads: sha256: "{{ registry_proxy_digest_checksum|default(None) }}" groups: - kube-node + local_volume_provisioner: enabled: "{{ local_volume_provisioner_enabled }}" container: true @@ -499,6 +534,7 @@ downloads: sha256: "{{ local_volume_provisioner_digest_checksum|default(None) }}" groups: - kube-node + cephfs_provisioner: enabled: "{{ cephfs_provisioner_enabled }}" container: true @@ -507,6 +543,7 @@ downloads: sha256: "{{ cephfs_provisioner_digest_checksum|default(None) }}" groups: - kube-node + ingress_nginx_controller: enabled: "{{ ingress_nginx_enabled }}" container: true @@ -515,6 +552,7 @@ downloads: sha256: "{{ ingress_nginx_controller_digest_checksum|default(None) }}" groups: - kube-node + ingress_nginx_default_backend: enabled: "{{ ingress_nginx_enabled }}" container: true @@ -523,6 +561,7 @@ downloads: sha256: "{{ ingress_nginx_default_backend_digest_checksum|default(None) }}" groups: - kube-node + cert_manager_controller: enabled: "{{ cert_manager_enabled }}" container: true From 934d92f09c01bfd805a3341532ae353a666b04ef Mon Sep 17 00:00:00 2001 From: rongzhang Date: Sat, 15 Sep 2018 15:14:19 +0800 Subject: [PATCH 19/44] Upgrade kubedns to 1.14.11 --- roles/download/defaults/main.yml | 2 +- roles/kubernetes-apps/ansible/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index c27224b14..fb9c6a766 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -112,7 +112,7 @@ nginx_image_tag: 1.13 dnsmasq_version: 2.78 dnsmasq_image_repo: "andyshinn/dnsmasq" dnsmasq_image_tag: "{{ dnsmasq_version }}" -kubedns_version: 1.14.10 +kubedns_version: 1.14.11 kubedns_image_repo: "gcr.io/google_containers/k8s-dns-kube-dns-{{ image_arch }}" kubedns_image_tag: "{{ kubedns_version }}" diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 6a60e09d8..9cf716d0f 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -1,6 +1,6 @@ --- # Versions -kubedns_version: 1.14.10 +kubedns_version: 1.14.11 kubednsautoscaler_version: 1.1.2 # Limits for dnsmasq/kubedns apps From fe35c32c62dcc7ebd26e8a74c62a3e356242c213 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Sat, 15 Sep 2018 16:35:19 +0300 Subject: [PATCH 20/44] Fix tiller override command --- roles/kubernetes-apps/helm/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/kubernetes-apps/helm/tasks/main.yml b/roles/kubernetes-apps/helm/tasks/main.yml index 24614da32..c3b8c26f5 100644 --- a/roles/kubernetes-apps/helm/tasks/main.yml +++ b/roles/kubernetes-apps/helm/tasks/main.yml @@ -68,6 +68,7 @@ {% if tiller_max_history is defined %} --history-max={{ tiller_max_history }}{% endif %} {% if tiller_enable_tls %} --tiller-tls --tiller-tls-verify --tiller-tls-cert={{ tiller_tls_cert }} --tiller-tls-key={{ tiller_tls_key }} --tls-ca-cert={{ tiller_tls_ca_cert }} {% endif %} {% if tiller_secure_release_info %} --override 'spec.template.spec.containers[0].command'='{/tiller,--storage=secret}' {% endif %} + --debug --dry-run | kubectl apply -f - changed_when: false when: From 1d4aa7abcc170358cfbc96ad67951b7ecb97026c Mon Sep 17 00:00:00 2001 From: rongzhang Date: Wed, 12 Sep 2018 13:24:31 +0800 Subject: [PATCH 21/44] Fix upgrade k8s --- roles/kubernetes/node/tasks/install_host.yml | 10 +++++++--- upgrade-cluster.yml | 3 +-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/roles/kubernetes/node/tasks/install_host.yml b/roles/kubernetes/node/tasks/install_host.yml index 3ca923848..485610469 100644 --- a/roles/kubernetes/node/tasks/install_host.yml +++ b/roles/kubernetes/node/tasks/install_host.yml @@ -1,11 +1,16 @@ --- - name: install | Copy kubelet binary from download dir - command: rsync -piu "{{ local_release_dir }}/hyperkube" "{{ bin_dir }}/kubelet" - changed_when: false + synchronize: + src: "{{ local_release_dir }}/hyperkube" + dest: "{{ bin_dir }}/kubelet" + compress: no + owner: no + group: no tags: - hyperkube - upgrade + notify: restart kubelet - name: install | Set kubelet binary permissions file: @@ -15,7 +20,6 @@ tags: - hyperkube - upgrade - notify: restart kubelet - name: install | Copy socat wrapper for Container Linux command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/opt/bin {{ install_socat_image_repo }}:{{ install_socat_image_tag }}" diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml index cbb59e93e..dd705439e 100644 --- a/upgrade-cluster.yml +++ b/upgrade-cluster.yml @@ -100,9 +100,8 @@ - { role: kubespray-defaults} - { role: upgrade/pre-upgrade, tags: pre-upgrade } - { role: kubernetes/node, tags: node } - - { role: upgrade/post-upgrade, tags: post-upgrade } - { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" } - - { role: kubespray-defaults} + - { role: upgrade/post-upgrade, tags: post-upgrade } environment: "{{proxy_env}}" - hosts: kube-master[0] From 84c4c7dc82db27e2134a75643cca05de04ecac6d Mon Sep 17 00:00:00 2001 From: rongzhang Date: Sun, 16 Sep 2018 10:50:56 +0800 Subject: [PATCH 22/44] Use synchronize module --- roles/etcd/tasks/install_host.yml | 30 +++++++++++--------- roles/kubernetes/master/tasks/main.yml | 9 +++++- roles/kubernetes/node/tasks/install | 0 roles/kubernetes/node/tasks/install.yml | 1 + roles/kubernetes/node/tasks/install_ | 0 roles/kubernetes/node/tasks/install_host.yml | 2 ++ 6 files changed, 28 insertions(+), 14 deletions(-) create mode 100644 roles/kubernetes/node/tasks/install create mode 100644 roles/kubernetes/node/tasks/install_ diff --git a/roles/etcd/tasks/install_host.yml b/roles/etcd/tasks/install_host.yml index 0dc226e66..fe50a7b1a 100644 --- a/roles/etcd/tasks/install_host.yml +++ b/roles/etcd/tasks/install_host.yml @@ -1,21 +1,25 @@ --- -- name: install | Copy etcd binary from download dir - shell: | - rsync -piu "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcd" "{{ bin_dir }}/etcd" - rsync -piu "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcdctl" "{{ bin_dir }}/etcdctl" +- name: install | Copy etcd and etcdctl binary from download dir + synchronize: + src: "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-amd64/{{ item }}" + dest: "{{ bin_dir }}/{{ item }}" + compress: no + perms: yes + owner: no + group: no changed_when: false + delegate_to: "{{ inventory_hostname }}" + with_items: + - "etcd" + - "etcdctl" when: etcd_cluster_setup -- name: install | Set etcd binary permissions +- name: install | Set etcd and etcdctl binary permissions file: - path: "{{ bin_dir }}/etcd" + path: "{{ bin_dir }}/{{ item }}" mode: "0755" state: file + with_items: + - "etcd" + - "etcdctl" when: etcd_cluster_setup - -- name: install | Set etcdctl binary permissions - file: - path: "{{ bin_dir }}/etcdctl" - mode: "0755" - state: file - when: etcd_cluster_setup \ No newline at end of file diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index 93da9760b..a249e4164 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -10,8 +10,15 @@ when: kube_encrypt_secret_data - name: install | Copy kubectl binary from download dir - command: rsync -piu "{{ local_release_dir }}/hyperkube" "{{ bin_dir }}/kubectl" + synchronize: + src: "{{ local_release_dir }}/hyperkube" + dest: "{{ bin_dir }}/kubectl" + compress: no + perms: yes + owner: no + group: no changed_when: false + delegate_to: "{{ inventory_hostname }}" tags: - hyperkube - kubectl diff --git a/roles/kubernetes/node/tasks/install b/roles/kubernetes/node/tasks/install new file mode 100644 index 000000000..e69de29bb diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index f3e1dca40..ceeaa442b 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -11,6 +11,7 @@ src: "{{ local_release_dir }}/kubeadm" dest: "{{ bin_dir }}/kubeadm" compress: no + perms: yes owner: no group: no delegate_to: "{{ inventory_hostname }}" diff --git a/roles/kubernetes/node/tasks/install_ b/roles/kubernetes/node/tasks/install_ new file mode 100644 index 000000000..e69de29bb diff --git a/roles/kubernetes/node/tasks/install_host.yml b/roles/kubernetes/node/tasks/install_host.yml index 485610469..3ec1f1800 100644 --- a/roles/kubernetes/node/tasks/install_host.yml +++ b/roles/kubernetes/node/tasks/install_host.yml @@ -5,8 +5,10 @@ src: "{{ local_release_dir }}/hyperkube" dest: "{{ bin_dir }}/kubelet" compress: no + perms: yes owner: no group: no + delegate_to: "{{ inventory_hostname }}" tags: - hyperkube - upgrade From f34a6699efab9e78aa090332cbb9a85bde652991 Mon Sep 17 00:00:00 2001 From: Wong Hoi Sing Edison Date: Mon, 17 Sep 2018 16:58:04 +0800 Subject: [PATCH 23/44] cert-manager: Upgrade to 0.5.0 Upstream Changes: - cert-manager 0.5.0 (https://github.com/jetstack/cert-manager/releases/tag/v0.5.0) Our Changes: - Templates sync with upstream manifests --- README.md | 2 +- roles/download/defaults/main.yml | 2 +- .../cert_manager/templates/00-namespace.yml.j2 | 1 + .../templates/clusterrole-cert-manager.yml.j2 | 9 ++------- .../templates/clusterrolebinding-cert-manager.yml.j2 | 2 +- .../cert_manager/templates/crd-certificate.yml.j2 | 4 +++- .../cert_manager/templates/crd-clusterissuer.yml.j2 | 4 +++- .../cert_manager/templates/crd-issuer.yml.j2 | 4 +++- .../cert_manager/templates/deploy-cert-manager.yml.j2 | 2 +- .../cert_manager/templates/sa-cert-manager.yml.j2 | 2 +- 10 files changed, 17 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 59686019f..04992cc66 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ Supported Components - [weave](https://github.com/weaveworks/weave) v2.4.0 - Application - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11 - - [cert-manager](https://github.com/jetstack/cert-manager) v0.4.1 + - [cert-manager](https://github.com/jetstack/cert-manager) v0.5.0 - [coredns](https://github.com/coredns/coredns) v1.2.2 - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.19.0 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 5c1ca3f4e..9f01cf08b 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -164,7 +164,7 @@ ingress_nginx_controller_image_repo: "quay.io/kubernetes-ingress-controller/ngin ingress_nginx_controller_image_tag: "0.19.0" ingress_nginx_default_backend_image_repo: "gcr.io/google_containers/defaultbackend" ingress_nginx_default_backend_image_tag: "1.4" -cert_manager_version: "v0.4.1" +cert_manager_version: "v0.5.0" cert_manager_controller_image_repo: "quay.io/jetstack/cert-manager-controller" cert_manager_controller_image_tag: "{{ cert_manager_version }}" diff --git a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/00-namespace.yml.j2 b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/00-namespace.yml.j2 index 7cf3a282d..fef90aed6 100644 --- a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/00-namespace.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/00-namespace.yml.j2 @@ -5,3 +5,4 @@ metadata: name: {{ cert_manager_namespace }} labels: name: {{ cert_manager_namespace }} + certmanager.k8s.io/disable-validation: "true" diff --git a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/clusterrole-cert-manager.yml.j2 b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/clusterrole-cert-manager.yml.j2 index 0ce11fb9b..b8b6251fa 100644 --- a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/clusterrole-cert-manager.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/clusterrole-cert-manager.yml.j2 @@ -5,7 +5,7 @@ metadata: name: cert-manager labels: app: cert-manager - chart: cert-manager-v0.4.1 + chart: cert-manager-v0.5.0 release: cert-manager heritage: Tiller rules: @@ -13,12 +13,7 @@ rules: resources: ["certificates", "issuers", "clusterissuers"] verbs: ["*"] - apiGroups: [""] - # TODO: remove endpoints once 0.4 is released. We include it here in case - # users use the 'master' version of the Helm chart with a 0.2.x release of - # cert-manager that still performs leader election with Endpoint resources. - # We advise users don't do this, but some will anyway and this will reduce - # friction. - resources: ["endpoints", "configmaps", "secrets", "events", "services", "pods"] + resources: ["configmaps", "secrets", "events", "services", "pods"] verbs: ["*"] - apiGroups: ["extensions"] resources: ["ingresses"] diff --git a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/clusterrolebinding-cert-manager.yml.j2 b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/clusterrolebinding-cert-manager.yml.j2 index 7dd567fd9..95cdeb525 100644 --- a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/clusterrolebinding-cert-manager.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/clusterrolebinding-cert-manager.yml.j2 @@ -5,7 +5,7 @@ metadata: name: cert-manager labels: app: cert-manager - chart: cert-manager-v0.4.1 + chart: cert-manager-v0.5.0 release: cert-manager heritage: Tiller roleRef: diff --git a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-certificate.yml.j2 b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-certificate.yml.j2 index a1663c64d..2d9a5c1f9 100644 --- a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-certificate.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-certificate.yml.j2 @@ -3,9 +3,11 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: certificates.certmanager.k8s.io + annotations: + "helm.sh/hook": crd-install labels: app: cert-manager - chart: cert-manager-v0.4.1 + chart: cert-manager-v0.5.0 release: cert-manager heritage: Tiller spec: diff --git a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-clusterissuer.yml.j2 b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-clusterissuer.yml.j2 index 869d4d260..53d65e4bc 100644 --- a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-clusterissuer.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-clusterissuer.yml.j2 @@ -3,9 +3,11 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clusterissuers.certmanager.k8s.io + annotations: + "helm.sh/hook": crd-install labels: app: cert-manager - chart: cert-manager-v0.4.1 + chart: cert-manager-v0.5.0 release: cert-manager heritage: Tiller spec: diff --git a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-issuer.yml.j2 b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-issuer.yml.j2 index 1946b81bf..7a19c7ede 100644 --- a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-issuer.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/crd-issuer.yml.j2 @@ -3,9 +3,11 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: issuers.certmanager.k8s.io + annotations: + "helm.sh/hook": crd-install labels: app: cert-manager - chart: cert-manager-v0.4.1 + chart: cert-manager-v0.5.0 release: cert-manager heritage: Tiller spec: diff --git a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/deploy-cert-manager.yml.j2 b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/deploy-cert-manager.yml.j2 index 2bcf5c701..1fedf42a2 100644 --- a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/deploy-cert-manager.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/deploy-cert-manager.yml.j2 @@ -6,7 +6,7 @@ metadata: namespace: {{ cert_manager_namespace }} labels: app: cert-manager - chart: cert-manager-v0.4.1 + chart: cert-manager-v0.5.0 release: cert-manager heritage: Tiller spec: diff --git a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/sa-cert-manager.yml.j2 b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/sa-cert-manager.yml.j2 index c5270e88b..f73fd0c34 100644 --- a/roles/kubernetes-apps/ingress_controller/cert_manager/templates/sa-cert-manager.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/cert_manager/templates/sa-cert-manager.yml.j2 @@ -6,6 +6,6 @@ metadata: namespace: {{ cert_manager_namespace }} labels: app: cert-manager - chart: cert-manager-v0.4.1 + chart: cert-manager-v0.5.0 release: cert-manager heritage: Tiller From a544e54578791c7891ee4dcc2dc4f8f52ea17944 Mon Sep 17 00:00:00 2001 From: Wong Hoi Sing Edison Date: Mon, 17 Sep 2018 17:09:19 +0800 Subject: [PATCH 24/44] weave: Upgrade to 2.4.1 Upstream Changes: - weave 2.4.1 (https://github.com/weaveworks/weave/releases/tag/v2.4.1) Our Changes: - Templates sync with upstream manifests --- README.md | 2 +- roles/download/defaults/main.yml | 6 +++--- roles/network_plugin/weave/templates/weave-net.yml.j2 | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 59686019f..d019215bf 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ Supported Components - [cilium](https://github.com/cilium/cilium) v1.2.0 - [contiv](https://github.com/contiv/install) v1.1.7 - [flanneld](https://github.com/coreos/flannel) v0.10.0 - - [weave](https://github.com/weaveworks/weave) v2.4.0 + - [weave](https://github.com/weaveworks/weave) v2.4.1 - Application - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11 - [cert-manager](https://github.com/jetstack/cert-manager) v0.4.1 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 5c1ca3f4e..1ad39f37a 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -46,7 +46,7 @@ flannel_version: "v0.10.0" flannel_cni_version: "v0.3.0" vault_version: 0.10.1 -weave_version: "2.4.0" +weave_version: "2.4.1" pod_infra_version: 3.1 contiv_version: 1.1.7 cilium_version: "v1.2.0" @@ -98,9 +98,9 @@ netcheck_agent_img_repo: "mirantis/k8s-netchecker-agent" netcheck_agent_tag: "{{ netcheck_version }}" netcheck_server_img_repo: "mirantis/k8s-netchecker-server" netcheck_server_tag: "{{ netcheck_version }}" -weave_kube_image_repo: "weaveworks/weave-kube" +weave_kube_image_repo: "docker.io/weaveworks/weave-kube" weave_kube_image_tag: "{{ weave_version }}" -weave_npc_image_repo: "weaveworks/weave-npc" +weave_npc_image_repo: "docker.io/weaveworks/weave-npc" weave_npc_image_tag: "{{ weave_version }}" contiv_image_repo: "contiv/netplugin" contiv_image_tag: "{{ contiv_version }}" diff --git a/roles/network_plugin/weave/templates/weave-net.yml.j2 b/roles/network_plugin/weave/templates/weave-net.yml.j2 index 09e5fbb7c..60e7b6325 100644 --- a/roles/network_plugin/weave/templates/weave-net.yml.j2 +++ b/roles/network_plugin/weave/templates/weave-net.yml.j2 @@ -42,13 +42,13 @@ items: - patch - update - apiGroups: - - policy + - policy resourceNames: - - privileged + - privileged resources: - - podsecuritypolicies + - podsecuritypolicies verbs: - - use + - use - apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: From bcaf2f9ea3a000ad6f0acda218f6346f0660e5af Mon Sep 17 00:00:00 2001 From: Erwan Miran Date: Mon, 17 Sep 2018 16:45:05 +0200 Subject: [PATCH 25/44] contiv 1.2.1 --- docs/contiv.md | 4 +- inventory/sample/group_vars/all/all.yml | 7 ++ roles/download/defaults/main.yml | 6 +- .../network_plugin/contiv/tasks/configure.yml | 40 +++++++++ .../network_plugin/contiv/tasks/main.yml | 1 - roles/kubespray-defaults/defaults/main.yaml | 5 ++ roles/network_plugin/contiv/defaults/main.yml | 14 +++ roles/network_plugin/contiv/tasks/main.yml | 51 ++++++++--- roles/network_plugin/contiv/tasks/reset.yml | 9 ++ .../contiv/templates/contiv-api-proxy.yml.j2 | 5 +- .../contiv/templates/contiv-config.yml.j2 | 18 ++-- .../contiv/templates/contiv-etcd-proxy.yml.j2 | 2 + .../contiv-netmaster-clusterrole.yml.j2 | 1 + .../contiv/templates/contiv-netmaster.yml.j2 | 27 ++++-- .../contiv/templates/contiv-netplugin.yml.j2 | 85 ++++++++++++------- roles/reset/tasks/main.yml | 7 +- 16 files changed, 220 insertions(+), 62 deletions(-) create mode 100644 roles/network_plugin/contiv/tasks/reset.yml diff --git a/docs/contiv.md b/docs/contiv.md index 1366a2dfd..29a8ebbc7 100644 --- a/docs/contiv.md +++ b/docs/contiv.md @@ -54,16 +54,18 @@ The default configuration uses VXLAN to create an overlay. Two networks are crea You can change the default network configuration by overriding the `contiv_networks` variable. -The default forward mode is set to routing: +The default forward mode is set to routing and the default network mode is vxlan: ```yaml contiv_fwd_mode: routing +contiv_net_mode: vxlan ``` The following is an example of how you can use VLAN instead of VXLAN: ```yaml contiv_fwd_mode: bridge +contiv_net_mode: vlan contiv_vlan_interface: eth0 contiv_networks: - name: default-net diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml index faf65eb1a..eff115f9f 100644 --- a/inventory/sample/group_vars/all/all.yml +++ b/inventory/sample/group_vars/all/all.yml @@ -43,6 +43,13 @@ bin_dir: /usr/local/bin ## The subnets of each nodes will be distributed by the datacenter router #peer_with_router: false +## With contiv, L3 BGP mode is possible by setting contiv_fwd_mode to "routing". +## In this case, you may need to peer with an uplink +## NB: The hostvars must contain a key "contiv" of which value is a dict containing "router_ip", "as"(defaults to contiv_global_as), "neighbor_as" (defaults to contiv_global_neighbor_as), "neighbor" +#contiv_peer_with_uplink_leaf: false +#contiv_global_as: "65002" +#contiv_global_neighbor_as: "500" + ## Upstream dns servers used by dnsmasq #upstream_dns_servers: # - 8.8.8.8 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 896fe8ba3..4000cffd1 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -48,7 +48,7 @@ flannel_cni_version: "v0.3.0" vault_version: 0.10.1 weave_version: "2.4.1" pod_infra_version: 3.1 -contiv_version: 1.1.7 +contiv_version: 1.2.1 cilium_version: "v1.2.0" # Download URLs @@ -104,10 +104,14 @@ weave_npc_image_repo: "docker.io/weaveworks/weave-npc" weave_npc_image_tag: "{{ weave_version }}" contiv_image_repo: "contiv/netplugin" contiv_image_tag: "{{ contiv_version }}" +contiv_init_image_repo: "contiv/netplugin-init" +contiv_init_image_tag: "latest" contiv_auth_proxy_image_repo: "contiv/auth_proxy" contiv_auth_proxy_image_tag: "{{ contiv_version }}" contiv_etcd_init_image_repo: "ferest/etcd-initer" contiv_etcd_init_image_tag: latest +contiv_ovs_image_repo: "contiv/ovs" +contiv_ovs_image_tag: "latest" cilium_image_repo: "docker.io/cilium/cilium" cilium_image_tag: "{{ cilium_version }}" nginx_image_repo: nginx diff --git a/roles/kubernetes-apps/network_plugin/contiv/tasks/configure.yml b/roles/kubernetes-apps/network_plugin/contiv/tasks/configure.yml index 35eeeacfc..a080aa4f0 100644 --- a/roles/kubernetes-apps/network_plugin/contiv/tasks/configure.yml +++ b/roles/kubernetes-apps/network_plugin/contiv/tasks/configure.yml @@ -33,6 +33,46 @@ when: "contiv_global_config.networkInfraType != contiv_fabric_mode" run_once: true +- name: Contiv | Set peer hostname + set_fact: + contiv_peer_hostname: >- + {%- if override_system_hostname|default(true) -%} + {{ contiv_peer_hostname|default({})|combine({item: hostvars[item]['inventory_hostname']}) }} + {%- else -%} + {{ contiv_peer_hostname|default({})|combine({item: hostvars[item]['ansible_fqdn']}) }} + {%- endif -%} + with_items: "{{ groups['k8s-cluster'] }}" + run_once: true + when: + - contiv_fwd_mode == 'routing' + - contiv_peer_with_uplink_leaf + +- name: Contiv | Get BGP configuration + command: | + {{ bin_dir }}/netctl --netmaster "http://127.0.0.1:{{ contiv_netmaster_port }}" \ + bgp ls --json + register: bgp_config + run_once: true + changed_when: false + when: + - contiv_fwd_mode == 'routing' + - contiv_peer_with_uplink_leaf + +- name: Contiv | Configure peering with router(s) + command: | + {{ bin_dir }}/netctl --netmaster "http://127.0.0.1:{{ contiv_netmaster_port }}" \ + bgp create {{ item.value }} \ + --router-ip="{{ hostvars[item.key]['contiv']['router_ip'] }}" \ + --as="{{ hostvars[item.key]['contiv']['as'] | default(contiv_global_as) }}" \ + --neighbor-as="{{ hostvars[item.key]['contiv']['neighbor_as'] | default(contiv_global_neighbor_as) }}" \ + --neighbor="{{ hostvars[item.key]['contiv']['neighbor'] }}" + run_once: true + with_dict: "{{ contiv_peer_hostname }}" + when: + - contiv_fwd_mode == 'routing' + - contiv_peer_with_uplink_leaf + - bgp_config.stdout|from_json|length == 0 or not item.value in bgp_config.stdout|from_json|map(attribute='key')|list + - name: Contiv | Get existing networks command: | {{ bin_dir }}/netctl --netmaster "http://127.0.0.1:{{ contiv_netmaster_port }}" \ diff --git a/roles/kubernetes-apps/network_plugin/contiv/tasks/main.yml b/roles/kubernetes-apps/network_plugin/contiv/tasks/main.yml index 5289296dc..1bca92329 100644 --- a/roles/kubernetes-apps/network_plugin/contiv/tasks/main.yml +++ b/roles/kubernetes-apps/network_plugin/contiv/tasks/main.yml @@ -9,7 +9,6 @@ filename: "{{ contiv_config_dir }}/{{ item.item.file }}" state: "{{ item.changed | ternary('latest','present') }}" with_items: "{{ contiv_manifests_results.results }}" - delegate_to: "{{ groups['kube-master'][0] }}" run_once: true - import_tasks: configure.yml diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index e405c7a3f..d3e563935 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -303,6 +303,11 @@ weave_mode_seed: false weave_seed: uninitialized weave_peers: uninitialized +# Contiv L3 BGP Mode +contiv_peer_with_uplink_leaf: false +contiv_global_as: "65002" +contiv_global_neighbor_as: "500" + ## Set no_proxy to all assigned cluster IPs and hostnames no_proxy: >- {%- if http_proxy is defined or https_proxy is defined %} diff --git a/roles/network_plugin/contiv/defaults/main.yml b/roles/network_plugin/contiv/defaults/main.yml index b6e237df5..622d0fd8d 100644 --- a/roles/network_plugin/contiv/defaults/main.yml +++ b/roles/network_plugin/contiv/defaults/main.yml @@ -6,8 +6,10 @@ contiv_etcd_data_dir: "/var/lib/etcd/contiv-data" contiv_netmaster_port: 9999 contiv_cni_version: 0.1.0 +# No need to download it by default, but must be defined contiv_etcd_image_repo: "{{ etcd_image_repo }}" contiv_etcd_image_tag: "{{ etcd_image_tag }}" + contiv_etcd_listen_port: 6666 contiv_etcd_peer_port: 6667 contiv_etcd_endpoints: |- @@ -26,9 +28,21 @@ contiv_fwd_mode: routing # Fabric mode: aci, aci-opflex or default contiv_fabric_mode: default +# Defaut netmode: vxlan or vlan +contiv_net_mode: vxlan + # Dataplane interface contiv_vlan_interface: "" +# Default loglevels are INFO +contiv_netmaster_loglevel: "WARN" +contiv_netplugin_loglevel: "WARN" +contiv_ovsdb_server_loglevel: "warn" +contiv_ovs_vswitchd_loglevel: "warn" + +# VxLAN port +contiv_vxlan_port: 4789 + # Default network configuration contiv_networks: - name: contivh1 diff --git a/roles/network_plugin/contiv/tasks/main.yml b/roles/network_plugin/contiv/tasks/main.yml index bc9dcd3c0..9f8258785 100644 --- a/roles/network_plugin/contiv/tasks/main.yml +++ b/roles/network_plugin/contiv/tasks/main.yml @@ -16,8 +16,25 @@ with_items: - "{{ contiv_etcd_conf_dir }}" - "{{ contiv_etcd_data_dir }}" + when: inventory_hostname in groups['kube-master'] -- set_fact: +- name: Contiv | Workaround https://github.com/contiv/netplugin/issues/1152 + set_fact: + kube_apiserver_endpoint_for_contiv: |- + {% if not is_kube_master and loadbalancer_apiserver_localhost -%} + https://localhost:{{ nginx_kube_apiserver_port|default(kube_apiserver_port) }} + {%- elif loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined -%} + https://{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }} + {%- if loadbalancer_apiserver.port|string != "443" -%} + :{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }} + {%- endif -%} + {%- else -%} + https://{{ first_kube_master }}:{{ kube_apiserver_port }} + {%- endif %} + when: inventory_hostname in groups['kube-master'] + +- name: Contiv | Set necessary facts + set_fact: contiv_config_dir: "{{ contiv_config_dir }}" contiv_enable_api_proxy: "{{ contiv_enable_api_proxy }}" contiv_fabric_mode: "{{ contiv_fabric_mode }}" @@ -26,22 +43,26 @@ contiv_networks: "{{ contiv_networks }}" contiv_manifests: - {name: contiv-config, file: contiv-config.yml, type: configmap} + - {name: contiv-etcd, file: contiv-etcd.yml, type: daemonset} + - {name: contiv-etcd-proxy, file: contiv-etcd-proxy.yml, type: daemonset} + - {name: contiv-ovs, file: contiv-ovs.yml, type: daemonset} - {name: contiv-netmaster, file: contiv-netmaster-clusterrolebinding.yml, type: clusterrolebinding} - {name: contiv-netmaster, file: contiv-netmaster-clusterrole.yml, type: clusterrole} - {name: contiv-netmaster, file: contiv-netmaster-serviceaccount.yml, type: serviceaccount} + - {name: contiv-netmaster, file: contiv-netmaster.yml, type: daemonset} - {name: contiv-netplugin, file: contiv-netplugin-clusterrolebinding.yml, type: clusterrolebinding} - {name: contiv-netplugin, file: contiv-netplugin-clusterrole.yml, type: clusterrole} - {name: contiv-netplugin, file: contiv-netplugin-serviceaccount.yml, type: serviceaccount} - - {name: contiv-etcd, file: contiv-etcd.yml, type: daemonset} - - {name: contiv-etcd-proxy, file: contiv-etcd-proxy.yml, type: daemonset} - {name: contiv-netplugin, file: contiv-netplugin.yml, type: daemonset} - - {name: contiv-netmaster, file: contiv-netmaster.yml, type: daemonset} + when: inventory_hostname in groups['kube-master'] - set_fact: contiv_manifests: |- {% set _ = contiv_manifests.append({"name": "contiv-api-proxy", "file": "contiv-api-proxy.yml", "type": "daemonset"}) %} {{ contiv_manifests }} - when: contiv_enable_api_proxy + when: + - contiv_enable_api_proxy + - inventory_hostname in groups['kube-master'] - name: Contiv | Create /var/contiv file: @@ -55,21 +76,23 @@ mode: 0755 owner: root group: root + when: inventory_hostname in groups['kube-master'] - name: Contiv | Install all Kubernetes resources template: src: "{{ item.file }}.j2" dest: "{{ contiv_config_dir }}/{{ item.file }}" with_items: "{{ contiv_manifests }}" - delegate_to: "{{ groups['kube-master'][0] }}" - run_once: true register: contiv_manifests_results + when: inventory_hostname in groups['kube-master'] - name: Contiv | Generate contiv-api-proxy certificates script: generate-certificate.sh args: creates: /var/contiv/auth_proxy_key.pem - when: "contiv_enable_api_proxy and contiv_generate_certificate" + when: + - contiv_enable_api_proxy + - contiv_generate_certificate delegate_to: "{{ groups['kube-master'][0] }}" run_once: true @@ -81,7 +104,9 @@ with_items: - auth_proxy_key.pem - auth_proxy_cert.pem - when: "contiv_enable_api_proxy and contiv_generate_certificate" + when: + - contiv_enable_api_proxy + - contiv_generate_certificate delegate_to: "{{ groups['kube-master'][0] }}" run_once: true @@ -92,9 +117,11 @@ with_items: - auth_proxy_key.pem - auth_proxy_cert.pem - when: "inventory_hostname != groups['kube-master'][0] - and inventory_hostname in groups['kube-master'] - and contiv_enable_api_proxy and contiv_generate_certificate" + when: + - inventory_hostname != groups['kube-master'][0] + - inventory_hostname in groups['kube-master'] + - contiv_enable_api_proxy + - contiv_generate_certificate - name: Contiv | Copy cni plugins from hyperkube command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/bash -c '/bin/cp -fa /opt/cni/bin/* /cnibindir/'" diff --git a/roles/network_plugin/contiv/tasks/reset.yml b/roles/network_plugin/contiv/tasks/reset.yml new file mode 100644 index 000000000..3cf293610 --- /dev/null +++ b/roles/network_plugin/contiv/tasks/reset.yml @@ -0,0 +1,9 @@ +--- +- name: reset | check contiv vxlan_sys network device + stat: + path: "/sys/class/net/vxlan_sys_{{ contiv_vxlan_port | default('4789') }}" + register: contiv_vxlan_sys + +- name: reset | remove the vxlan_sys network device created by contiv + command: "ip link del vxlan_sys_{{ contiv_vxlan_port | default('4789') }}" + when: contiv_vxlan_sys.stat.exists diff --git a/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 b/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 index cea0efe51..16b8a9713 100644 --- a/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 @@ -35,16 +35,19 @@ spec: - --listen-address=0.0.0.0:{{ contiv_api_proxy_port }} - --tls-key-file=/var/contiv/auth_proxy_key.pem - --tls-certificate=/var/contiv/auth_proxy_cert.pem + - --data-store-driver=$(STORE_DRIVER) - --data-store-address=$(CONTIV_ETCD) - --netmaster-address=127.0.0.1:{{ contiv_netmaster_port }} env: - name: NO_NETMASTER_STARTUP_CHECK value: "0" + - name: STORE_DRIVER + value: etcd - name: CONTIV_ETCD valueFrom: configMapKeyRef: name: contiv-config - key: cluster_store + key: contiv_etcd securityContext: privileged: false volumeMounts: diff --git a/roles/network_plugin/contiv/templates/contiv-config.yml.j2 b/roles/network_plugin/contiv/templates/contiv-config.yml.j2 index 249d9d88e..18b7748eb 100644 --- a/roles/network_plugin/contiv/templates/contiv-config.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-config.yml.j2 @@ -7,20 +7,22 @@ metadata: name: contiv-config namespace: kube-system data: - # The location of your cluster store. This is set to the - # avdertise-client value below from the contiv-etcd service. - # Change it to an external etcd/consul instance if required. - cluster_store: "etcd://127.0.0.1:{{ contiv_etcd_listen_port }}" - # The CNI network configuration to install on each node. - cni_config: |- + contiv_netmaster_loglevel: {{ contiv_netmaster_loglevel }} + contiv_netplugin_loglevel: {{ contiv_netplugin_loglevel }} + contiv_ovsdb_server_extra_flags: "--verbose={{ contiv_ovsdb_server_loglevel }}" + contiv_ovs_vswitchd_extra_flags: "--verbose={{ contiv_ovs_vswitchd_loglevel }}" + contiv_fwdmode: {{ contiv_fwd_mode }} + contiv_netmode: {{ contiv_net_mode }} + contiv_etcd: "http://127.0.0.1:{{ contiv_etcd_listen_port }}" + contiv_cni_config: |- { "cniVersion": "{{ contiv_cni_version }}", "name": "contiv-net", "type": "contivk8s" } - config: |- + contiv_k8s_config: |- { - "K8S_API_SERVER": "{{ kube_apiserver_endpoint }}", + "K8S_API_SERVER": "{{ kube_apiserver_endpoint_for_contiv }}", "K8S_CA": "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", "K8S_KEY": "", "K8S_CERT": "", diff --git a/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 b/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 index 75946d821..a4adedd46 100644 --- a/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 @@ -19,6 +19,8 @@ spec: spec: hostNetwork: true hostPID: true + nodeSelector: + node-role.kubernetes.io/node: "true" containers: - name: contiv-etcd-proxy image: {{ contiv_etcd_image_repo }}:{{ contiv_etcd_image_tag }} diff --git a/roles/network_plugin/contiv/templates/contiv-netmaster-clusterrole.yml.j2 b/roles/network_plugin/contiv/templates/contiv-netmaster-clusterrole.yml.j2 index 92b4f588d..4c179e6c4 100644 --- a/roles/network_plugin/contiv/templates/contiv-netmaster-clusterrole.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-netmaster-clusterrole.yml.j2 @@ -13,6 +13,7 @@ rules: - namespaces - networkpolicies verbs: + - get - watch - list - update diff --git a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 index 787fe5c27..be0f23360 100644 --- a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 @@ -1,4 +1,4 @@ -# This manifest deploys the Contiv API Server on Kubernetes. +--- kind: DaemonSet apiVersion: extensions/v1beta1 metadata: @@ -31,20 +31,31 @@ spec: containers: - name: contiv-netmaster image: {{ contiv_image_repo }}:{{ contiv_image_tag }} - args: - - -m - - -pkubernetes env: - - name: CONTIV_ETCD + - name: CONTIV_ROLE + value: netmaster + - name: CONTIV_NETMASTER_MODE + value: kubernetes + - name: CONTIV_NETMASTER_ETCD_ENDPOINTS valueFrom: configMapKeyRef: name: contiv-config - key: cluster_store - - name: CONTIV_CONFIG + key: contiv_etcd + - name: CONTIV_NETMASTER_FORWARD_MODE valueFrom: configMapKeyRef: name: contiv-config - key: config + key: contiv_fwdmode + - name: CONTIV_NETMASTER_NET_MODE + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_netmode + - name: CONTIV_NETMASTER_LOG_LEVEL + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_netmaster_loglevel securityContext: privileged: true volumeMounts: diff --git a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 index b7927f51c..755e9b204 100644 --- a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 @@ -1,3 +1,4 @@ +--- # This manifest installs contiv-netplugin container, as well # as the Contiv CNI plugins and network config on # each master and worker node in a Kubernetes cluster. @@ -27,73 +28,99 @@ spec: - key: node-role.kubernetes.io/master effect: NoSchedule serviceAccountName: contiv-netplugin + initContainers: + - name: contiv-netplugin-init + image: {{ contiv_init_image_repo }}:{{ contiv_init_image_tag }} + env: + - name: CONTIV_ROLE + value: netplugin + - name: CONTIV_MODE + value: kubernetes + - name: CONTIV_K8S_CONFIG + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_k8s_config + - name: CONTIV_CNI_CONFIG + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_cni_config + volumeMounts: + - mountPath: /var/contiv + name: var-contiv + readOnly: false + - mountPath: /etc/cni/net.d/ + name: etc-cni-dir + readOnly: false + - name: contiv-cni + image: {{ contiv_image_repo }}:{{ contiv_version }} + command: ["cp", "/contiv/bin/contivk8s", "/opt/cni/bin/contivk8s"] + volumeMounts: + - mountPath: /opt/cni/bin + name: cni-bin-dir + readOnly: false containers: # Runs netplugin container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: contiv-netplugin image: {{ contiv_image_repo }}:{{ contiv_image_tag }} - args: - - -pkubernetes - - -x env: - name: VLAN_IF value: {{ contiv_vlan_interface }} - - name: VTEP_IP + - name: CONTIV_NETPLUGIN_VLAN_UPLINKS + value: {{ contiv_vlan_interface }} + - name: CONTIV_NETPLUGIN_VXLAN_PORT + value: "{{ contiv_vxlan_port }}" + - name: CONTIV_ROLE + value: netplugin + - name: CONTIV_NETPLUGIN_MODE + value: kubernetes + - name: CONTIV_NETPLUGIN_VTEP_IP valueFrom: fieldRef: fieldPath: status.podIP - - name: CONTIV_ETCD + - name: CONTIV_NETPLUGIN_ETCD_ENDPOINTS valueFrom: configMapKeyRef: name: contiv-config - key: cluster_store - - name: CONTIV_CNI_CONFIG + key: contiv_etcd + - name: CONTIV_NETPLUGIN_FORWARD_MODE valueFrom: configMapKeyRef: name: contiv-config - key: cni_config - - name: CONTIV_CONFIG + key: contiv_fwdmode + - name: CONTIV_NETPLUGIN_NET_MODE valueFrom: configMapKeyRef: name: contiv-config - key: config + key: contiv_netmode + - name: CONTIV_NETPLUGIN_LOG_LEVEL + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_netplugin_loglevel + resources: + requests: + cpu: 250m securityContext: privileged: true volumeMounts: - - mountPath: /etc/openvswitch - name: etc-openvswitch - readOnly: false - - mountPath: /lib/modules - name: lib-modules - readOnly: false - mountPath: /var/run name: var-run readOnly: false - mountPath: /var/contiv name: var-contiv readOnly: false - - mountPath: /opt/cni/bin - name: cni-bin-dir - readOnly: false - - mountPath: /etc/cni/net.d/ - name: etc-cni-dir - readOnly: false volumes: # Used by contiv-netplugin - - name: etc-openvswitch - hostPath: - path: /etc/openvswitch - - name: lib-modules - hostPath: - path: /lib/modules - name: var-run hostPath: path: /var/run - name: var-contiv hostPath: path: /var/contiv - # Used to install CNI. - name: cni-bin-dir hostPath: path: /opt/cni/bin diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index 88dec8d7a..0fe73408b 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -150,6 +150,11 @@ - "{{ bin_dir }}/weave" - /var/lib/rkt - /etc/vault + - /etc/contiv + - /var/contiv + - /run/contiv + - /etc/openvswitch + - /run/openvswitch ignore_errors: yes tags: - files @@ -181,7 +186,7 @@ - name: reset | include file with reset tasks specific to the network_plugin if exists include_tasks: "{{ (role_path + '/../network_plugin/' + kube_network_plugin + '/tasks/reset.yml') | realpath }}" when: - - kube_network_plugin in ['flannel', 'cilium'] + - kube_network_plugin in ['flannel', 'cilium', 'contiv'] tags: - network From afa2a5f1c45dea27a91485e75fe6c0cb7b48b17c Mon Sep 17 00:00:00 2001 From: Erwan Miran Date: Mon, 17 Sep 2018 16:46:19 +0200 Subject: [PATCH 26/44] enhanced reset for contiv --- .../contiv/files/contiv-cleanup.sh | 10 +++ .../network_plugin/contiv/tasks/pre-reset.yml | 66 +++++++++++++++ .../contiv/templates/contiv-cleanup.yml.j2 | 57 +++++++++++++ .../contiv/templates/contiv-ovs.yml.j2 | 80 +++++++++++++++++++ roles/reset/tasks/main.yml | 7 ++ 5 files changed, 220 insertions(+) create mode 100644 roles/network_plugin/contiv/files/contiv-cleanup.sh create mode 100644 roles/network_plugin/contiv/tasks/pre-reset.yml create mode 100644 roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 create mode 100644 roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 diff --git a/roles/network_plugin/contiv/files/contiv-cleanup.sh b/roles/network_plugin/contiv/files/contiv-cleanup.sh new file mode 100644 index 000000000..2aa1a7796 --- /dev/null +++ b/roles/network_plugin/contiv/files/contiv-cleanup.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -e +echo "Starting cleanup" +ovs-vsctl list-br | grep contiv | xargs -I % ovs-vsctl del-br % +for p in $(ifconfig | grep vport | awk '{print $1}'); +do + ip link delete $p type veth +done +touch /tmp/cleanup.done +sleep 60 diff --git a/roles/network_plugin/contiv/tasks/pre-reset.yml b/roles/network_plugin/contiv/tasks/pre-reset.yml new file mode 100644 index 000000000..a811d5921 --- /dev/null +++ b/roles/network_plugin/contiv/tasks/pre-reset.yml @@ -0,0 +1,66 @@ +--- +- name: reset | Check that kubectl is still here + stat: + path: "{{ bin_dir }}/kubectl" + register: contiv_kubectl + +- name: reset | Delete contiv netplugin and netmaster daemonsets + kube: + name: "{{ item }}" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "ds" + state: absent + with_items: + - contiv-netplugin + - contiv-netmaster + register: contiv_cleanup_deletion + tags: + - network + when: + - contiv_kubectl.stat.exists + - inventory_hostname == groups['kube-master'][0] + +- name: reset | Copy contiv temporary cleanup script + copy: + src: ../files/contiv-cleanup.sh # Not in role_path so we must trick... + dest: /opt/cni/bin/cleanup + owner: root + group: root + mode: 0750 + when: + - contiv_kubectl.stat.exists + +- name: reset | Lay down contiv cleanup template + template: + src: ../templates/contiv-cleanup.yml.j2 # Not in role_path so we must trick... + dest: "{{ kube_config_dir }}/contiv-cleanup.yml" # kube_config_dir is used here as contiv_config_dir is not necessarily set at reset + register: contiv_cleanup_manifest + when: + - contiv_kubectl.stat.exists + - inventory_hostname == groups['kube-master'][0] + +- name: reset | Start contiv cleanup resources + kube: + name: "contiv-cleanup" + namespace: "kube-system" + kubectl: "{{ bin_dir }}/kubectl" + resource: "ds" + state: latest + filename: "{{ kube_config_dir }}/contiv-cleanup.yml" + when: + - contiv_kubectl.stat.exists + - inventory_hostname == groups['kube-master'][0] + ignore_errors: true + +- name: reset | Wait until contiv cleanup is done + command: "{{ bin_dir }}/kubectl -n kube-system get ds contiv-cleanup -o jsonpath='{.status.numberReady}'" + register: cleanup_done_all_nodes + until: cleanup_done_all_nodes.stdout|int == groups['k8s-cluster']|length + retries: 5 + delay: 5 + ignore_errors: true + changed_when: false + when: + - contiv_kubectl.stat.exists + - inventory_hostname == groups['kube-master'][0] diff --git a/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 b/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 new file mode 100644 index 000000000..99cbecb7d --- /dev/null +++ b/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 @@ -0,0 +1,57 @@ +--- +kind: DaemonSet +apiVersion: extensions/v1beta1 +metadata: + name: contiv-cleanup + namespace: kube-system + labels: + k8s-app: contiv-cleanup +spec: + selector: + matchLabels: + k8s-app: contiv-cleanup + template: + metadata: + labels: + k8s-app: contiv-cleanup + spec: + hostNetwork: true + hostPID: true + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: contiv-netplugin + containers: + - name: contiv-ovs-cleanup + image: {{ contiv_ovs_image_repo }}:{{ contiv_ovs_image_tag }} + command: ["/opt/cni/bin/cleanup"] + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/openvswitch + name: etc-openvswitch + readOnly: false + - mountPath: /var/run + name: var-run + readOnly: false + - mountPath: /opt/cni/bin + name: cni-bin-dir + readOnly: false + readinessProbe: + exec: + command: + - cat + - /tmp/cleanup.done + initialDelaySeconds: 3 + periodSeconds: 3 + successThreshold: 1 + volumes: + - name: etc-openvswitch + hostPath: + path: /etc/openvswitch + - name: var-run + hostPath: + path: /var/run + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin diff --git a/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 b/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 new file mode 100644 index 000000000..825ab3042 --- /dev/null +++ b/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 @@ -0,0 +1,80 @@ +--- +apiVersion: apps/v1 +# This manifest deploys the contiv-ovs pod. +kind: DaemonSet +apiVersion: extensions/v1beta1 +metadata: + name: contiv-ovs + namespace: kube-system + labels: + k8s-app: contiv-ovs +spec: + selector: + matchLabels: + k8s-app: contiv-ovs + template: + metadata: + labels: + k8s-app: contiv-ovs + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + hostNetwork: true + hostPID: true + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + containers: + # Runs ovs containers on each Kubernetes node. + - name: contiv-ovsdb-server + image: {{ contiv_ovs_image_repo }}:{{ contiv_ovs_image_tag }} + command: ["/scripts/start-ovsdb-server.sh"] + securityContext: + privileged: false + # Won't work until https://github.com/contiv/ovs-docker/pull/4 is merged and image is built again + env: + - name: OVSDBSERVER_EXTRA_FLAGS + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_ovsdb_server_extra_flags + volumeMounts: + - mountPath: /etc/openvswitch + name: etc-openvswitch + readOnly: false + - mountPath: /var/run + name: var-run + readOnly: false + - name: contiv-ovs-vswitchd + image: {{ contiv_ovs_image_repo }}:{{ contiv_ovs_image_tag }} + command: ["/scripts/start-ovs-vswitchd.sh"] + securityContext: + privileged: true + # Won't work until https://github.com/contiv/ovs-docker/pull/4 is merged and image is built again + env: + - name: OVSVSWITCHD_EXTRA_FLAGS + valueFrom: + configMapKeyRef: + name: contiv-config + key: contiv_ovs_vswitchd_extra_flags + volumeMounts: + - mountPath: /etc/openvswitch + name: etc-openvswitch + readOnly: false + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /var/run + name: var-run + readOnly: false + volumes: + # Used by contiv-ovs + - name: etc-openvswitch + hostPath: + path: /etc/openvswitch + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run + hostPath: + path: /var/run diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index 0fe73408b..38945c64b 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -1,5 +1,12 @@ --- +- name: reset | include file with pre-reset tasks specific to the network_plugin if exists + include_tasks: "{{ (role_path + '/../network_plugin/' + kube_network_plugin + '/tasks/pre-reset.yml') | realpath }}" + when: + - kube_network_plugin in ['contiv'] + tags: + - network + - name: reset | stop services service: name: "{{ item }}" From d3adf09bdec4402d80042b7ccd3f4c34852f2e63 Mon Sep 17 00:00:00 2001 From: Kevin Schuck Date: Mon, 17 Sep 2018 15:22:28 -0500 Subject: [PATCH 27/44] Fixes BGPPeer resource for calico >= 3.0.0 --- roles/network_plugin/calico/tasks/install.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index f2d41f9fc..86a304a4f 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -163,14 +163,13 @@ shell: > echo '{ "apiVersion": "projectcalico.org/v3", - "kind": "bgpPeer", + "kind": "BGPPeer", "metadata": { "name": "{{ inventory_hostname }}-bgp" }, "spec": { "asNumber": "{{ item.as }}", "node": "{{ inventory_hostname }}", - "scope": "node", "peerIP": "{{ item.router_id }}" }}' | {{ bin_dir }}/calicoctl create --skip-exists -f - retries: 4 @@ -203,13 +202,12 @@ shell: > echo '{ "apiVersion": "projectcalico.org/v3", - "kind": "bgpPeer", + "kind": "BGPPeer", "metadata": { "name": "{{ inventory_hostname }}" }, "spec": { "asNumber": "{{ local_as | default(global_as_num)}}", - "scope": "node", "node": "{{ inventory_hostname }}", "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}" }}' | {{ bin_dir }}/calicoctl create --skip-exists -f - From 77e08ba204a45f34461c97f086c5fd1c77693086 Mon Sep 17 00:00:00 2001 From: rongzhang Date: Mon, 17 Sep 2018 01:40:20 +0800 Subject: [PATCH 28/44] Support dynamic kubelet config https://kubernetes.io/blog/2018/07/11/dynamic-kubelet-configuration/ --- inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml | 8 ++++++++ .../master/templates/kubeadm-config.v1alpha2.yaml.j2 | 4 ++++ roles/kubernetes/node/tasks/main.yml | 7 +++++++ roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 | 3 +++ roles/kubespray-defaults/defaults/main.yaml | 8 ++++++++ tests/files/gce_centos-weave-kubeadm.yml | 1 + 6 files changed, 31 insertions(+) diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index 6096f52a3..bee09d4a6 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -154,6 +154,14 @@ k8s_image_pull_policy: IfNotPresent # audit log for kubernetes kubernetes_audit: false +# dynamic kubelet configuration +dynamic_kubelet_configuration: false + +# define kubelet config dir for dynamic kubelet +#kubelet_config_dir: +default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir" +dynamic_kubelet_configuration_dir: "{{ kubelet_config_dir | default(default_kubelet_config_dir) }}" + # pod security policy (RBAC must be enabled either by having 'RBAC' in authorization_modes or kubeadm enabled) podsecuritypolicy_enabled: false diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index 447c382b6..4c729e9ac 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -150,3 +150,7 @@ nodeRegistration: {% if container_manager == 'crio' %} criSocket: /var/run/crio/crio.sock {% endif %} +{% if dynamic_kubelet_configuration %} +featureGates: + DynamicKubeletConfig: true +{% endif %} diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 69302915f..03d08bb1a 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -32,6 +32,13 @@ tags: - kubelet +- name: Make sure dynamic kubelet configuration directory is writeable + file: + path: "{{ dynamic_kubelet_configuration_dir }}" + mode: 0600 + state: directory + when: dynamic_kubelet_configuration + - name: Write kubelet config file (kubeadm) template: src: kubelet.kubeadm.env.j2 diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index 7597fd9ae..0424efdf9 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -48,6 +48,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% else %} --fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \ {% endif %} +{% if dynamic_kubelet_configuration %} +--dynamic-config-dir={{ dynamic_kubelet_configuration_dir }} \ +{% endif %} --runtime-cgroups={{ kubelet_runtime_cgroups }} --kubelet-cgroups={{ kubelet_kubelet_cgroups }} \ {% endset %} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index e405c7a3f..24fef7125 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -136,6 +136,14 @@ kube_apiserver_port: 6443 kube_apiserver_insecure_bind_address: 127.0.0.1 kube_apiserver_insecure_port: 8080 +# dynamic kubelet configuration +dynamic_kubelet_configuration: false + +# define kubelet config dir for dynamic kubelet +#kubelet_config_dir: +default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir" +dynamic_kubelet_configuration_dir: "{{ kubelet_config_dir | default(default_kubelet_config_dir) }}" + # Aggregator kube_api_aggregator_routing: false diff --git a/tests/files/gce_centos-weave-kubeadm.yml b/tests/files/gce_centos-weave-kubeadm.yml index 199fa437c..24183eb6e 100644 --- a/tests/files/gce_centos-weave-kubeadm.yml +++ b/tests/files/gce_centos-weave-kubeadm.yml @@ -10,5 +10,6 @@ kube_network_plugin: weave kubeadm_enabled: true deploy_netchecker: true kubernetes_audit: true +dynamic_kubelet_configuration: true kubedns_min_replicas: 1 cloud_provider: gce From 09a1bcb30b7d1a5f79cb64f43307f456e6dbd178 Mon Sep 17 00:00:00 2001 From: rongzhang Date: Tue, 18 Sep 2018 00:02:30 +0800 Subject: [PATCH 29/44] Keep list of k8s checksums for hyperkube and kubeadm Keep a list of checksums for kubeadm and hyperkube downloads. Makes it easier to switch version --- roles/download/defaults/main.yml | 33 ++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 896fe8ba3..82c939b41 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -58,10 +58,39 @@ etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_ver hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64/hyperkube" # Checksums +hyperkube_checksums: + v1.11.3: dac8da16dd6688e52b5dc510f5dd0a20b54350d52fb27ceba2f018ba2c8be692 + v1.11.2: d727f8cae3fc26b1add9b4ff0d4d9b99605544ff7fb3baeecdca394362adbfb8 + v1.11.1: 019ce1ecf4c6a70c06a7f4ef107443351458b4d9e6b9ce4a436bfbfbef93feea + v1.11.0: 7e191c164dc2c942abd37e4b50846e0be31ca959afffeff6b034beacbc2a106a + v1.10.8: f8a68514a6c858089f44ec93b2ffb2d764ea67d3b02b19112348f73ffcfe4386 + v1.10.7: 13e25eb39467014fd169f38b7cd6bec8ff55525b8001c7abba85957e6470b6cc + v1.10.6: 0daa34fa58470e5f20def10d3dd544922c28c558719d3338ad8c524154c91257 + v1.10.5: 1a53456f9d33a7c07adb1636f20f1d0b92b8e7647063a70d0ce134a238e680fe + v1.10.4: 16e36693c15494036d930139a749ec1bc492b7fefa2c3adc1abbe8f38178ae7c + v1.10.3: e807753dc309635902a56069ee06fc390944ef034b72c53b2e1e51d0c9ead8a3 + v1.10.2: 3843fb594a18c4a64d77736bab72000ec4b8c4ddf178e20ec3249f709e9ed9c1 + v1.10.1: 6e0642ad6bae68dc81b8d1c9efa18e265e17e23da1895862823cafac08c0344c + v1.10.0: b5575b2fb4266754c1675b8cd5d9b6cac70f3fee7a05c4e80da3a9e83e58c57e +kubeadm_checksums: + v1.11.3: 422a7a32ed9a7b1eaa2a4f9d121674dfbe80eb41e206092c13017d097f75aaec + v1.11.2: 6b17720a65b8ff46efe92a5544f149c39a221910d89939838d75581d4e6924c0 + v1.11.1: 425ec24b95f7217ee06d1588aba22f206a5829f8c6a5352c2862368552361fe6 + v1.11.0: 0000478fc59a24ec1727de744188d13c4d702a644954132efa9d9954371b3553 + v1.10.8: 42660875dd94c93267bd2f567c67d692b362bd143d7502967a62c5474b2b25b8 + v1.10.7: cdeb07fd3705e973800c4aa0b8a510d5dba1de8e1039428cfebdaf3d93e332b6 + v1.10.6: e1d49a6b33b384f681468add2e9ee08552069ae0d6b0ad59e1c943ddbaeac3fa + v1.10.5: f231d4bcc9f2ed15597272e5359e380cc760c0b57a1f7cb97ce2bbab5df774e0 + v1.10.4: 7e1169bbbeed973ab402941672dec957638dea5952a1e8bc89a37d5e709cc4b4 + v1.10.3: b2a6f0764b89a4a13a3da4471af943ce98efeb29e2913c9e7880fe27f4f43a5f + v1.10.2: 394d7d340214c91d669186cf4f2110d8eb840ca965399b4d8b22d0545a60e377 + v1.10.1: 012e48fb92b1c22543b12ab2db7d780777972043287404c98cca4d2c6ec964ec + v1.10.0: ebbac985834289037b544523c3e2f39bb44bea938aca9d9e88ef7e880fb8472f + etcd_binary_checksum: b729db0732448064271ea6fdcb901773c4fe917763ca07776f22d0e5e0bd4097 -hyperkube_binary_checksum: dac8da16dd6688e52b5dc510f5dd0a20b54350d52fb27ceba2f018ba2c8be692 -kubeadm_binary_checksum: 422a7a32ed9a7b1eaa2a4f9d121674dfbe80eb41e206092c13017d097f75aaec vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188 +hyperkube_binary_checksum: "{{ hyperkube_checksums[kube_version] }}" +kubeadm_binary_checksum: "{{ kubeadm_checksums[kubeadm_version] }}" # Containers # In some cases, we need a way to set --registry-mirror or --insecure-registry for docker, From 93429bc661a95c3f8d96e5267006eb895f007736 Mon Sep 17 00:00:00 2001 From: Sergey Bondarev Date: Tue, 18 Sep 2018 14:29:53 +0300 Subject: [PATCH 30/44] create separate options files for network plugins remove plugin options from common files --- inventory/sample/group_vars/all/all.yml | 7 --- .../group_vars/k8s-cluster/k8s-cluster.yml | 16 ----- .../group_vars/k8s-cluster/k8s-net-calico.yml | 15 +++++ .../group_vars/k8s-cluster/k8s-net-canal.yml | 11 ++++ .../group_vars/k8s-cluster/k8s-net-cilium.yml | 1 + .../group_vars/k8s-cluster/k8s-net-contiv.yml | 20 +++++++ .../k8s-cluster/k8s-net-flannel.yml | 16 +++++ .../group_vars/k8s-cluster/k8s-net-weave.yml | 58 +++++++++++++++++++ 8 files changed, 121 insertions(+), 23 deletions(-) create mode 100644 inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml create mode 100644 inventory/sample/group_vars/k8s-cluster/k8s-net-canal.yml create mode 100644 inventory/sample/group_vars/k8s-cluster/k8s-net-cilium.yml create mode 100644 inventory/sample/group_vars/k8s-cluster/k8s-net-contiv.yml create mode 100644 inventory/sample/group_vars/k8s-cluster/k8s-net-flannel.yml create mode 100644 inventory/sample/group_vars/k8s-cluster/k8s-net-weave.yml diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml index eff115f9f..faf65eb1a 100644 --- a/inventory/sample/group_vars/all/all.yml +++ b/inventory/sample/group_vars/all/all.yml @@ -43,13 +43,6 @@ bin_dir: /usr/local/bin ## The subnets of each nodes will be distributed by the datacenter router #peer_with_router: false -## With contiv, L3 BGP mode is possible by setting contiv_fwd_mode to "routing". -## In this case, you may need to peer with an uplink -## NB: The hostvars must contain a key "contiv" of which value is a dict containing "router_ip", "as"(defaults to contiv_global_as), "neighbor_as" (defaults to contiv_global_neighbor_as), "neighbor" -#contiv_peer_with_uplink_leaf: false -#contiv_global_as: "65002" -#contiv_global_neighbor_as: "500" - ## Upstream dns servers used by dnsmasq #upstream_dns_servers: # - 8.8.8.8 diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index bee09d4a6..a779ff7c0 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -70,22 +70,6 @@ kube_users: # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico -# Weave deployment -# weave_password: ~ -# weave_checkpoint_disable: false -# weave_conn_limit: 100 -# weave_hairpin_mode: true -# weave_ipalloc_range: {{ kube_pods_subnet }} -# weave_expect_npc: {{ enable_network_policy }} -# weave_kube_peers: ~ -# weave_ipalloc_init: ~ -# weave_expose_ip: ~ -# weave_metrics_addr: ~ -# weave_status_addr: ~ -# weave_mtu: 1376 -# weave_no_masq_local: true -# weave_extra_args: ~ - # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml b/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml new file mode 100644 index 000000000..a033bed0e --- /dev/null +++ b/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml @@ -0,0 +1,15 @@ +# see roles/network_plugin/calico/defaults/main.yml + +# Enables Internet connectivity from containers +# nat_outgoing: true + +# add default ippool name +# calico_pool_name: "default-pool" + +# Global as_num (/calico/bgp/v1/global/as_num) +# global_as_num: "64512" + +# You can set MTU value here. If left undefined or empty, it will +# not be specified in calico CNI config, so Calico will use built-in +# defaults. The value should be a number, not a string. +# calico_mtu: 1500 diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-net-canal.yml b/inventory/sample/group_vars/k8s-cluster/k8s-net-canal.yml new file mode 100644 index 000000000..5c78b5d70 --- /dev/null +++ b/inventory/sample/group_vars/k8s-cluster/k8s-net-canal.yml @@ -0,0 +1,11 @@ +# see roles/network_plugin/canal/defaults/main.yml + +# The interface used by canal for host <-> host communication. +# If left blank, then the interface is chosing using the node's +# default route. +# canal_iface: "" + +# Whether or not to masquerade traffic to destinations not within +# the pod network. +# canal_masquerade: "true" + diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-net-cilium.yml b/inventory/sample/group_vars/k8s-cluster/k8s-net-cilium.yml new file mode 100644 index 000000000..95730eb03 --- /dev/null +++ b/inventory/sample/group_vars/k8s-cluster/k8s-net-cilium.yml @@ -0,0 +1 @@ +# see roles/network_plugin/cilium/defaults/main.yml diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-net-contiv.yml b/inventory/sample/group_vars/k8s-cluster/k8s-net-contiv.yml new file mode 100644 index 000000000..3ca66714d --- /dev/null +++ b/inventory/sample/group_vars/k8s-cluster/k8s-net-contiv.yml @@ -0,0 +1,20 @@ +# see roles/network_plugin/contiv/defaults/main.yml + +# Forwarding mode: bridge or routing +# contiv_fwd_mode: routing + +## With contiv, L3 BGP mode is possible by setting contiv_fwd_mode to "routing". +## In this case, you may need to peer with an uplink +## NB: The hostvars must contain a key "contiv" of which value is a dict containing "router_ip", "as"(defaults to contiv_global_as), "neighbor_as" (defaults to contiv_global_neighbor_as), "neighbor" +#contiv_peer_with_uplink_leaf: false +#contiv_global_as: "65002" +#contiv_global_neighbor_as: "500" + +# Fabric mode: aci, aci-opflex or default +# contiv_fabric_mode: default + +# Defaut netmode: vxlan or vlan +# contiv_net_mode: vxlan + +# Dataplane interface +# contiv_vlan_interface: "" diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-net-flannel.yml b/inventory/sample/group_vars/k8s-cluster/k8s-net-flannel.yml new file mode 100644 index 000000000..be1245a42 --- /dev/null +++ b/inventory/sample/group_vars/k8s-cluster/k8s-net-flannel.yml @@ -0,0 +1,16 @@ +# see roles/network_plugin/flannel/defaults/main.yml + +## interface that should be used for flannel operations +## This is actually an inventory cluster-level item +# flannel_interface: + +## Select interface that should be used for flannel operations by regexp on Name or IP +## This is actually an inventory cluster-level item +## example: select interface with ip from net 10.0.0.0/23 +## single quote and escape backslashes +# flannel_interface_regexp: '10\\.0\\.[0-2]\\.\\d{1,3}' + +# You can choose what type of flannel backend to use: 'vxlan' or 'host-gw' +# for experimental backend +# please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md +# flannel_backend_type: "vxlan" diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-net-weave.yml b/inventory/sample/group_vars/k8s-cluster/k8s-net-weave.yml new file mode 100644 index 000000000..f8c8bba33 --- /dev/null +++ b/inventory/sample/group_vars/k8s-cluster/k8s-net-weave.yml @@ -0,0 +1,58 @@ +# see roles/network_plugin/weave/defaults/main.yml + +# Weave's network password for encryption, if null then no network encryption. +# weave_password: ~ + +# If set to 1, disable checking for new Weave Net versions (default is blank, +# i.e. check is enabled) +# weave_checkpoint_disable: false + +# Soft limit on the number of connections between peers. Defaults to 100. +# weave_conn_limit: 100 + +# Weave Net defaults to enabling hairpin on the bridge side of the veth pair +# for containers attached. If you need to disable hairpin, e.g. your kernel is +# one of those that can panic if hairpin is enabled, then you can disable it by +# setting `HAIRPIN_MODE=false`. +# weave_hairpin_mode: true + +# The range of IP addresses used by Weave Net and the subnet they are placed in +# (CIDR format; default 10.32.0.0/12) +# weave_ipalloc_range: "{{ kube_pods_subnet }}" + +# Set to 0 to disable Network Policy Controller (default is on) +# weave_expect_npc: "{{ enable_network_policy }}" + +# List of addresses of peers in the Kubernetes cluster (default is to fetch the +# list from the api-server) +# weave_kube_peers: ~ + +# Set the initialization mode of the IP Address Manager (defaults to consensus +# amongst the KUBE_PEERS) +# weave_ipalloc_init: ~ + +# Set the IP address used as a gateway from the Weave network to the host +# network - this is useful if you are configuring the addon as a static pod. +# weave_expose_ip: ~ + +# Address and port that the Weave Net daemon will serve Prometheus-style +# metrics on (defaults to 0.0.0.0:6782) +# weave_metrics_addr: ~ + +# Address and port that the Weave Net daemon will serve status requests on +# (defaults to disabled) +# weave_status_addr: ~ + +# Weave Net defaults to 1376 bytes, but you can set a smaller size if your +# underlying network has a tighter limit, or set a larger size for better +# performance if your network supports jumbo frames (e.g. 8916) +# weave_mtu: 1376 + +# Set to 1 to preserve the client source IP address when accessing Service +# annotated with `service.spec.externalTrafficPolicy=Local`. The feature works +# only with Weave IPAM (default). +# weave_no_masq_local: true + +# Extra variables that passing to launch.sh, useful for enabling seed mode, see +# https://www.weave.works/docs/net/latest/tasks/ipam/ipam/ +# weave_extra_args: ~ From fb1678d42532a42e0e32128246b8ed7331adef15 Mon Sep 17 00:00:00 2001 From: Kevin Schuck Date: Tue, 18 Sep 2018 10:48:30 -0500 Subject: [PATCH 31/44] Ensures BGPPeer resource names are unique --- roles/network_plugin/calico/tasks/install.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index 86a304a4f..5cce41388 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -165,7 +165,7 @@ "apiVersion": "projectcalico.org/v3", "kind": "BGPPeer", "metadata": { - "name": "{{ inventory_hostname }}-bgp" + "name": "{{ inventory_hostname }}-{{ item.router_id }}" }, "spec": { "asNumber": "{{ item.as }}", @@ -204,7 +204,7 @@ "apiVersion": "projectcalico.org/v3", "kind": "BGPPeer", "metadata": { - "name": "{{ inventory_hostname }}" + "name": "{{ inventory_hostname }}-{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}" }, "spec": { "asNumber": "{{ local_as | default(global_as_num)}}", From e766dd55828763140ceb61d7faf5fbdb942baf04 Mon Sep 17 00:00:00 2001 From: Sergey Bondarev Date: Tue, 18 Sep 2018 21:30:49 +0300 Subject: [PATCH 32/44] move calico options from all.yml to k8s-cluster/k8s-net-calico.yml --- inventory/sample/group_vars/all/all.yml | 5 ----- inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml index faf65eb1a..1d1b562d7 100644 --- a/inventory/sample/group_vars/all/all.yml +++ b/inventory/sample/group_vars/all/all.yml @@ -38,11 +38,6 @@ bin_dir: /usr/local/bin ## modules. #kubelet_load_modules: false -## With calico it is possible to distributed routes with border routers of the datacenter. -## Warning : enabling router peering will disable calico's default behavior ('node mesh'). -## The subnets of each nodes will be distributed by the datacenter router -#peer_with_router: false - ## Upstream dns servers used by dnsmasq #upstream_dns_servers: # - 8.8.8.8 diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml b/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml index a033bed0e..045bd21e0 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml @@ -1,5 +1,10 @@ # see roles/network_plugin/calico/defaults/main.yml +## With calico it is possible to distributed routes with border routers of the datacenter. +## Warning : enabling router peering will disable calico's default behavior ('node mesh'). +## The subnets of each nodes will be distributed by the datacenter router +#peer_with_router: false + # Enables Internet connectivity from containers # nat_outgoing: true From 0121bce9e565b05562a6e4a8851393246e145574 Mon Sep 17 00:00:00 2001 From: Karol Chrapek Date: Tue, 18 Sep 2018 22:13:15 +0200 Subject: [PATCH 33/44] Instead of doc update, change the verify step --- docs/upgrades.md | 7 +------ .../preinstall/tasks/0020-verify-settings.yml | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/upgrades.md b/docs/upgrades.md index 5443e067f..8e152230b 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -18,11 +18,6 @@ versions. Here are all version vars for each component: * flannel_version * kubedns_version -Please remember that some components upgrade depends on minimum installed version. -In example **calico version 2.6.5 upgrade to 3.1.1 is upgrading etcd store to etcdv3**. -The kubespray stack upgrade would failed when calico version is below 2.6.5. Please check -components' documentation and always upgrade test environment first. - #### Unsafe upgrade example If you wanted to upgrade just kube_version from v1.4.3 to v1.4.6, you could @@ -40,7 +35,7 @@ ansible-playbook cluster.yml -i inventory/sample/hosts.ini -e kube_version=v1.4. #### Graceful upgrade -Kubespray also supports cordon, drain and uncordoning of nodes when performing +Kubespray also supports cordon, drain and uncordoning of nodes when performing a cluster upgrade. There is a separate playbook used for this purpose. It is important to note that upgrade-cluster.yml can only be used for upgrading an existing cluster. That means there must be at least 1 kube-master already diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index 8c3cec92e..22ac00934 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -127,3 +127,21 @@ tags: - cloud-provider - facts + +- name: "Get current version of calico cluster version" + shell: "{{ bin_dir }}/calicoctl version | grep 'Cluster Version' | awk '{ print $3}'" + register: calico_version_on_server + run_once: yes + delegate_to: "{{ groups['kube-master'][0] }}" + + +- name: "Check that calico version is enought for upgrade" + assert: + that: + - calico_version_on_server.stdout|version_compare('v2.6.5', '>=') + msg: "Your version of calico is not fresh enough for upgrade. Minimum version v2.6.5" + when: + - 'calico_version_on_server.stdout is defined' + - 'calico_version_on_server.stdout != ""' + - inventory_hostname == groups['kube-master'][0] + run_once: yes From 26d7380c2e0ec19a0c1cd7ec5ba296b03832505a Mon Sep 17 00:00:00 2001 From: Andreas Kruger Date: Wed, 19 Sep 2018 10:01:45 +0200 Subject: [PATCH 34/44] Sync manifests from non-kubeadm to kubeadm deploy --- .../master/templates/kubeadm-config.v1alpha1.yaml.j2 | 10 ++++++++++ .../master/templates/kubeadm-config.v1alpha2.yaml.j2 | 10 ++++++++++ .../templates/manifests/kube-apiserver.manifest.j2 | 8 ++++---- .../manifests/kube-controller-manager.manifest.j2 | 2 +- .../templates/manifests/kube-scheduler.manifest.j2 | 2 +- roles/kubespray-defaults/defaults/main.yaml | 3 +++ 6 files changed, 29 insertions(+), 6 deletions(-) diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 index 416d23e83..adc71cd09 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 @@ -68,9 +68,17 @@ apiServerExtraArgs: {% endif %} service-node-port-range: {{ kube_apiserver_node_port_range }} kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" + profiling: "{{ kube_profiling }}" + repair-malformed-updates: "false" +{% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=') %} + anonymous-auth: "{{ kube_api_anonymous_auth }}" +{% endif %} {% if kube_basic_auth|default(true) %} basic-auth-file: {{ kube_users_dir }}/known_users.csv {% endif %} +{% if kube_token_auth|default(true) %} + token-auth-file: {{ kube_token_dir }}/known_tokens.csv +{% endif %} {% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %} oidc-issuer-url: {{ kube_oidc_url }} oidc-client-id: {{ kube_oidc_client_id }} @@ -102,6 +110,7 @@ controllerManagerExtraArgs: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} node-monitor-period: {{ kube_controller_node_monitor_period }} pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }} + profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} @@ -115,6 +124,7 @@ controllerManagerExtraVolumes: {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" {% endfor %} schedulerExtraArgs: + profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index 4c729e9ac..411a7ac85 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -60,9 +60,17 @@ apiServerExtraArgs: {% endif %} service-node-port-range: {{ kube_apiserver_node_port_range }} kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" + profiling: "{{ kube_profiling }}" + repair-malformed-updates: "false" +{% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=') %} + anonymous-auth: "{{ kube_api_anonymous_auth }}" +{% endif %} {% if kube_basic_auth|default(true) %} basic-auth-file: {{ kube_users_dir }}/known_users.csv {% endif %} +{% if kube_token_auth|default(true) %} + token-auth-file: {{ kube_token_dir }}/known_tokens.csv +{% endif %} {% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %} oidc-issuer-url: {{ kube_oidc_url }} oidc-client-id: {{ kube_oidc_client_id }} @@ -101,6 +109,7 @@ controllerManagerExtraArgs: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} node-monitor-period: {{ kube_controller_node_monitor_period }} pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }} + profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} @@ -126,6 +135,7 @@ apiServerExtraVolumes: {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" {% endfor %} schedulerExtraArgs: + profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 928b16c75..e1023d088 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -33,7 +33,7 @@ spec: - --audit-log-maxage={{ audit_log_maxage }} - --audit-log-maxbackup={{ audit_log_maxbackups }} - --audit-log-maxsize={{ audit_log_maxsize }} - - --audit-policy-file={{ audit_policy_file }} + - --audit-policy-file={{ audit_policy_file }} {% endif %} - --advertise-address={{ ip | default(ansible_default_ipv4.address) }} - --etcd-servers={{ etcd_access_addresses }} @@ -58,16 +58,16 @@ spec: - --admission-control={{ kube_apiserver_admission_control | join(',') }} {% else %} {% if kube_apiserver_enable_admission_plugins|length > 0 %} - - --enable-admission-plugins={{ kube_apiserver_enable_admission_plugins | join(',') }} + - --enable-admission-plugins={{ kube_apiserver_enable_admission_plugins | join(',') }} {% endif %} {% if kube_apiserver_disable_admission_plugins|length > 0 %} - - --disable-admission-plugins={{ kube_apiserver_disable_admission_plugins | join(',') }} + - --disable-admission-plugins={{ kube_apiserver_disable_admission_plugins | join(',') }} {% endif %} {% endif %} - --service-cluster-ip-range={{ kube_service_addresses }} - --service-node-port-range={{ kube_apiserver_node_port_range }} - --client-ca-file={{ kube_cert_dir }}/ca.pem - - --profiling=false + - --profiling={{ kube_profiling }} - --repair-malformed-updates=false - --kubelet-client-certificate={{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem - --kubelet-client-key={{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 0557c4498..848a65a59 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -37,7 +37,7 @@ spec: - --node-monitor-grace-period={{ kube_controller_node_monitor_grace_period }} - --node-monitor-period={{ kube_controller_node_monitor_period }} - --pod-eviction-timeout={{ kube_controller_pod_eviction_timeout }} - - --profiling=false + - --profiling={{ kube_profiling }} - --terminated-pod-gc-threshold=12500 - --v={{ kube_log_level }} {% if rbac_enabled %} diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index fee223eec..813731fa2 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -32,7 +32,7 @@ spec: - --use-legacy-policy-config - --policy-config-file={{ kube_config_dir }}/kube-scheduler-policy.yaml {% endif %} - - --profiling=false + - --profiling={{ kube_profiling }} - --v={{ kube_log_level }} {% if kube_feature_gates %} - --feature-gates={{ kube_feature_gates|join(',') }} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index b41134323..a81eb85f0 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -147,6 +147,9 @@ dynamic_kubelet_configuration_dir: "{{ kubelet_config_dir | default(default_kube # Aggregator kube_api_aggregator_routing: false +# Profiling +kube_profiling: false + # Container for runtime container_manager: docker From 0dd82293f1a813a25f1df68af92b9b0085522eca Mon Sep 17 00:00:00 2001 From: Rui Cao Date: Wed, 19 Sep 2018 16:47:58 +0800 Subject: [PATCH 35/44] Fix some typos Signed-off-by: Rui Cao --- docs/kubernetes-reliability.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/kubernetes-reliability.md b/docs/kubernetes-reliability.md index e338272d3..3e812d8f6 100644 --- a/docs/kubernetes-reliability.md +++ b/docs/kubernetes-reliability.md @@ -69,7 +69,7 @@ minute which may require large etcd containers or even dedicated nodes for etcd. > If we calculate the number of tries, the division will give 5, but in reality > it will be from 3 to 5 with `nodeStatusUpdateRetry` attempts of each try. The -> total number of attemtps will vary from 15 to 25 due to latency of all +> total number of attempts will vary from 15 to 25 due to latency of all > components. ## Medium Update and Average Reaction @@ -92,7 +92,7 @@ etcd updates per minute. Let's set `-–node-status-update-frequency` to **1m**. `--node-monitor-grace-period` will set to **5m** and `--pod-eviction-timeout` to **1m**. In this scenario, every kubelet will try to update the status every -minute. There will be 5 * 5 = 25 attempts before unhealty status. After 5m, +minute. There will be 5 * 5 = 25 attempts before unhealthy status. After 5m, Kubernetes controller manager will set unhealthy status. This means that pods will be evicted after 1m after being marked unhealthy. (6m in total). From 8d1c0c469c3d6b31a6b32bbae1eeb31e0514e885 Mon Sep 17 00:00:00 2001 From: Andreas Kruger Date: Wed, 19 Sep 2018 10:58:46 +0200 Subject: [PATCH 36/44] Added missing enable-aggregator-routing option --- .../kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 | 1 + .../kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 | 1 + 2 files changed, 2 insertions(+) diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 index adc71cd09..2a69f8b4e 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 @@ -70,6 +70,7 @@ apiServerExtraArgs: kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" profiling: "{{ kube_profiling }}" repair-malformed-updates: "false" + enable-aggregator-routing: "{{ kube_api_aggregator_routing }}" {% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=') %} anonymous-auth: "{{ kube_api_anonymous_auth }}" {% endif %} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index 411a7ac85..9482ed083 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -62,6 +62,7 @@ apiServerExtraArgs: kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" profiling: "{{ kube_profiling }}" repair-malformed-updates: "false" + enable-aggregator-routing: "{{ kube_api_aggregator_routing }}" {% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=') %} anonymous-auth: "{{ kube_api_anonymous_auth }}" {% endif %} From 8e37841a2ebbcc3c46cc845291636923d675fd48 Mon Sep 17 00:00:00 2001 From: Andreas Kruger Date: Wed, 19 Sep 2018 11:01:30 +0200 Subject: [PATCH 37/44] Add audit support to v1alpha1 of Kubeadm --- .../master/templates/kubeadm-config.v1alpha1.yaml.j2 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 index 2a69f8b4e..d8d0a0af8 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 @@ -101,6 +101,13 @@ apiServerExtraArgs: runtime-config: {{ kube_api_runtime_config | join(',') }} {% endif %} allow-privileged: "true" +{% if kubernetes_audit %} + audit-log-path: "{{ audit_log_path }}" + audit-log-maxage: "{{ audit_log_maxage }}" + audit-log-maxbackup: "{{ audit_log_maxbackups }}" + audit-log-maxsize: "{{ audit_log_maxsize }}" + audit-policy-file: {{ audit_policy_file }} +{% endif %} {% for key in kube_kubeadm_apiserver_extra_args %} {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}" {% endfor %} From 1c999b2a615be8cc04f37f7c35ad1d88f0f9781c Mon Sep 17 00:00:00 2001 From: Andreas Kruger Date: Wed, 19 Sep 2018 11:24:19 +0200 Subject: [PATCH 38/44] Move kube_kubeadm_controller_extra_args to controllerManagerExtraArgs section. It was placed in controllerManagerExtraVolumes --- .../master/templates/kubeadm-config.v1alpha1.yaml.j2 | 6 +++--- .../master/templates/kubeadm-config.v1alpha2.yaml.j2 | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 index d8d0a0af8..52f9c88f3 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 @@ -122,15 +122,15 @@ controllerManagerExtraArgs: {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} +{% for key in kube_kubeadm_controller_extra_args %} + {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" +{% endfor %} {% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %} controllerManagerExtraVolumes: - name: openstackcacert hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" {% endif %} -{% for key in kube_kubeadm_controller_extra_args %} - {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" -{% endfor %} schedulerExtraArgs: profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index 9482ed083..f7d7e9bc2 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -114,6 +114,9 @@ controllerManagerExtraArgs: {% if kube_feature_gates %} feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} +{% for key in kube_kubeadm_controller_extra_args %} + {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" +{% endfor %} {% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %} controllerManagerExtraVolumes: - name: openstackcacert @@ -132,9 +135,6 @@ apiServerExtraVolumes: writable: true {% endif %} {% endif %} -{% for key in kube_kubeadm_controller_extra_args %} - {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" -{% endfor %} schedulerExtraArgs: profiling: "{{ kube_profiling }}" {% if kube_feature_gates %} From 940d2fdbb1763fdad97a435b14bcb72f7e980cdf Mon Sep 17 00:00:00 2001 From: Andreas Kruger Date: Wed, 19 Sep 2018 11:54:34 +0200 Subject: [PATCH 39/44] Add missing enforce-node-allocatable to kubelet for kubeadm deployments --- roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index 0424efdf9..72b1e4884 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -26,6 +26,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% if kubelet_authorization_mode_webhook %} --authorization-mode=Webhook \ {% endif %} +--enforce-node-allocatable={{ kubelet_enforce_node_allocatable }} \ --client-ca-file={{ kube_cert_dir }}/ca.crt \ --pod-manifest-path={{ kube_manifest_dir }} \ --cadvisor-port={{ kube_cadvisor_port }} \ From c004896a4012d58dbe108db661d4eff0f407f5b1 Mon Sep 17 00:00:00 2001 From: Rui Cao Date: Wed, 19 Sep 2018 18:22:08 +0800 Subject: [PATCH 40/44] Fix some typos Signed-off-by: Rui Cao --- docs/calico.md | 8 ++++---- docs/weave.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/calico.md b/docs/calico.md index f6adaa662..3ebdc2877 100644 --- a/docs/calico.md +++ b/docs/calico.md @@ -21,7 +21,7 @@ The **calicoctl** command allows to check the status of the network workloads. calicoctl node status ``` -or for versions prior *v1.0.0*: +or for versions prior to *v1.0.0*: ``` calicoctl status @@ -33,7 +33,7 @@ calicoctl status calicoctl get ippool -o wide ``` -or for versions prior *v1.0.0*: +or for versions prior to *v1.0.0*: ``` calicoctl pool show @@ -73,7 +73,7 @@ In some cases you may want to route the pods subnet and so NAT is not needed on For instance if you have a cluster spread on different locations and you want your pods to talk each other no matter where they are located. The following variables need to be set: `peer_with_router` to enable the peering with the datacenter's border router (default value: false). -you'll need to edit the inventory and add a and a hostvar `local_as` by node. +you'll need to edit the inventory and add a hostvar `local_as` by node. ``` node1 ansible_ssh_host=95.54.0.12 local_as=xxxxxx @@ -156,7 +156,7 @@ The inventory above will deploy the following topology assuming that calico's ##### Optional : Define default endpoint to host action -By default Calico blocks traffic from endpoints to the host itself by using an iptables DROP action. When using it in kubernetes the action has to be changed to RETURN (default in kubespray) or ACCEPT (see https://github.com/projectcalico/felix/issues/660 and https://github.com/projectcalico/calicoctl/issues/1389). Otherwise all network packets from pods (with hostNetwork=False) to services endpoints (with hostNetwork=True) withing the same node are dropped. +By default Calico blocks traffic from endpoints to the host itself by using an iptables DROP action. When using it in kubernetes the action has to be changed to RETURN (default in kubespray) or ACCEPT (see https://github.com/projectcalico/felix/issues/660 and https://github.com/projectcalico/calicoctl/issues/1389). Otherwise all network packets from pods (with hostNetwork=False) to services endpoints (with hostNetwork=True) within the same node are dropped. To re-define default action please set the following variable in your inventory: diff --git a/docs/weave.md b/docs/weave.md index 9fcb14f80..969c6ea1e 100644 --- a/docs/weave.md +++ b/docs/weave.md @@ -9,7 +9,7 @@ Weave uses [**consensus**](https://www.weave.works/docs/net/latest/ipam/##consen Weave encryption is supported for all communication -* To use Weave encryption, specify a strong password (if no password, no encrytion) +* To use Weave encryption, specify a strong password (if no password, no encryption) ``` # In file ./inventory/sample/group_vars/k8s-cluster.yml From e0ddabc46318822fadf315be74bee1a4330ddcbc Mon Sep 17 00:00:00 2001 From: Andreas Kruger Date: Wed, 19 Sep 2018 12:55:30 +0200 Subject: [PATCH 41/44] Add support for kubelet_node_custom_flags --- docs/vars.md | 3 ++- roles/kubernetes/node/defaults/main.yml | 3 +++ roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 | 2 +- roles/kubernetes/node/templates/kubelet.standard.env.j2 | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/vars.md b/docs/vars.md index 1a0756169..c78708351 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -139,7 +139,7 @@ node_labels: By default, the `audit_policy_file` contains [default rules](https://github.com/kubernetes-incubator/kubespray/blob/master/roles/kubernetes/master/templates/apiserver-audit-policy.yaml.j2) that can be overriden with the `audit_policy_custom_rules` variable. ##### Custom flags for Kube Components -For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. Example: +For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. The `kubelet_node_custom_flags` apply kubelet settings only to nodes and not masters. Example: ``` kubelet_custom_flags: - "--eviction-hard=memory.available<100Mi" @@ -151,6 +151,7 @@ The possible vars are: * *controller_mgr_custom_flags* * *scheduler_custom_flags* * *kubelet_custom_flags* +* *kubelet_node_custom_flags* #### User accounts diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 143a41928..c28ad02d8 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -86,6 +86,9 @@ kubelet_max_pods: 110 ## Support custom flags to be passed to kubelet kubelet_custom_flags: [] +## Support custom flags to be passed to kubelet only on nodes, not masters +kubelet_node_custom_flags: [] + # This setting is used for rkt based kubelet for deploying hyperkube # from a docker based registry ( controls --insecure and docker:// ) ## Empty vaule for quay.io containers diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index 72b1e4884..385102f48 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -94,7 +94,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} {% set all_node_labels = role_node_labels + inventory_node_labels %} -KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" +KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %} diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2 index a1c514c8c..9e1b55c25 100644 --- a/roles/kubernetes/node/templates/kubelet.standard.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2 @@ -108,7 +108,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} {% set all_node_labels = role_node_labels + inventory_node_labels %} -KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" +KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kube_reserved }} --node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %} From c058e7a5ec200eab440fdca6f9837d7ae21e32b9 Mon Sep 17 00:00:00 2001 From: Andreas Kruger Date: Wed, 19 Sep 2018 13:15:14 +0200 Subject: [PATCH 42/44] Remove audit again from Kubeadm 1.10.x. Write mounts not supported untill 1.11 --- .../master/templates/kubeadm-config.v1alpha1.yaml.j2 | 7 ------- 1 file changed, 7 deletions(-) diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 index 52f9c88f3..d2a0d2db2 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 @@ -101,13 +101,6 @@ apiServerExtraArgs: runtime-config: {{ kube_api_runtime_config | join(',') }} {% endif %} allow-privileged: "true" -{% if kubernetes_audit %} - audit-log-path: "{{ audit_log_path }}" - audit-log-maxage: "{{ audit_log_maxage }}" - audit-log-maxbackup: "{{ audit_log_maxbackups }}" - audit-log-maxsize: "{{ audit_log_maxsize }}" - audit-policy-file: {{ audit_policy_file }} -{% endif %} {% for key in kube_kubeadm_apiserver_extra_args %} {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}" {% endfor %} From 963c3479a9036c402bd3936884437946a05e3565 Mon Sep 17 00:00:00 2001 From: Erwan Miran Date: Wed, 19 Sep 2018 14:18:51 +0200 Subject: [PATCH 43/44] Offline environment documentation --- docs/downloads.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/downloads.md b/docs/downloads.md index 50d9ef107..a7c445c51 100644 --- a/docs/downloads.md +++ b/docs/downloads.md @@ -40,3 +40,14 @@ The full list of available vars may be found in the download's ansible role defa Those also allow to specify custom urls and local repositories for binaries and container images as well. See also the DNS stack docs for the related intranet configuration, so the hosts can resolve those urls and repos. + +## Offline environment + +In case your servers don't have access to internet (for example when deploying on premises with security constraints), you'll have, first, to setup the appropriate proxies/caches/mirrors and/or internal repositories and registries and, then, adapt the following variables to fit your environment before deploying: + +* At least `foo_image_repo` and `foo_download_url` as described before (i.e. in case of use of proxies to registries and binaries repositories, checksums and versions do not necessarily need to be changed). + NB: Regarding `foo_image_repo`, when using insecure registries/proxies, you will certainly have to append them to the `docker_insecure_registries` variable in group_vars/all/docker.yml +* Depending on the `container_manager` + * When `container_manager=docker`, `docker_foo_repo_base_url`, `docker_foo_repo_gpgkey`, `dockerproject_bar_repo_base_url` and `dockerproject_bar_repo_gpgkey` (where `foo` is the distribution and `bar` is system package manager) + * When `container_manager=crio`, `crio_rhel_repo_base_url` +* When using Helm, `helm_stable_repo_url` From 131d5654983faacd13b547e3a77689423b7b0b2b Mon Sep 17 00:00:00 2001 From: Romain GUICHARD Date: Wed, 19 Sep 2018 14:50:38 +0200 Subject: [PATCH 44/44] fix openstack cli syntax (#3353) * fix openstack cli syntax * 'allowed-address' is also a dash, not an underscore * multiple allowed-address multiple allowed-address must be in separate parameters --- docs/openstack.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/openstack.md b/docs/openstack.md index ca1e89ae1..ef2d1dcf8 100644 --- a/docs/openstack.md +++ b/docs/openstack.md @@ -39,7 +39,7 @@ Then you can use the instance ids to find the connected [neutron](https://wiki.o Given the port ids on the left, you can set the two `allowed_address`(es) in OpenStack. Note that you have to allow both `kube_service_addresses` (default `10.233.0.0/18`) and `kube_pods_subnet` (default `10.233.64.0/18`.) # allow kube_service_addresses and kube_pods_subnet network - openstack port set 5662a4e0-e646-47f0-bf88-d80fbd2d99ef --allowed_address ip_address=10.233.0.0/18,ip_address=10.233.64.0/18 - openstack port set e5ae2045-a1e1-4e99-9aac-4353889449a7 --allowed_address ip_address=10.233.0.0/18,ip_address=10.233.64.0/18 + openstack port set 5662a4e0-e646-47f0-bf88-d80fbd2d99ef --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18 + openstack port set e5ae2045-a1e1-4e99-9aac-4353889449a7 --allowed-address ip-address=10.233.0.0/18 --allowed-address ip-address=10.233.64.0/18 Now you can finally run the playbook.