From 538cb3b1bd2ff92567517b1339b33483478315db Mon Sep 17 00:00:00 2001 From: Wong Hoi Sing Edison Date: Sun, 22 Jul 2018 08:14:38 +0800 Subject: [PATCH] weave: Upgrade to 2.4.0 Upstream Changes: - weave 2.4.0 (https://github.com/weaveworks/weave/releases/tag/v2.4.0) - Support `externalTrafficPolicy: Local` (https://github.com/weaveworks/weave/issues/2924) - Make the ipset list size bigger (https://github.com/weaveworks/weave/pull/3305) - Break out of kube rm-peers loop if nothing changes (https://github.com/weaveworks/weave/pull/3317) Our Changes: - Revamp weave-net.yml.j2 with upstream changes - Add more variables for customization - Replace WEAVE_PASSWORD with k8s secret - Remove hard-corded seed mode support, in favor of variables customization --- README.md | 2 +- inventory/sample/group_vars/k8s-cluster.yml | 34 ++++--- roles/download/defaults/main.yml | 2 +- .../network_plugin/weave/tasks/main.yml | 6 +- roles/network_plugin/weave/defaults/main.yml | 73 ++++++++++----- roles/network_plugin/weave/tasks/main.yml | 17 ++-- roles/network_plugin/weave/tasks/seed.yml | 56 ------------ .../weave/templates/weave-net.yml.j2 | 89 ++++++++++++------- 8 files changed, 136 insertions(+), 143 deletions(-) delete mode 100644 roles/network_plugin/weave/tasks/seed.yml diff --git a/README.md b/README.md index 3c1c713af..c0a9e6a20 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Supported Components - [cilium](https://github.com/cilium/cilium) v1.0.0-rc8 - [contiv](https://github.com/contiv/install) v1.1.7 - [flanneld](https://github.com/coreos/flannel) v0.10.0 - - [weave](https://github.com/weaveworks/weave) v2.3.0 + - [weave](https://github.com/weaveworks/weave) v2.4.0 - Application - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v1.1.0-k8s1.10 - [cert-manager](https://github.com/jetstack/cert-manager) v0.4.0 diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index cc77d5008..139f47257 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -67,25 +67,21 @@ kube_users: # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico -# weave's network password for encryption -# if null then no network encryption -# you can use --extra-vars to pass the password in command line -weave_password: EnterPasswordHere - -# Weave uses consensus mode by default -# Enabling seed mode allow to dynamically add or remove hosts -# https://www.weave.works/docs/net/latest/ipam/ -weave_mode_seed: false - -# This two variable are automatically changed by the weave's role, do not manually change these values -# To reset values : -# weave_seed: uninitialized -# weave_peers: uninitialized -weave_seed: uninitialized -weave_peers: uninitialized - -# Set the MTU of Weave (default 1376, Jumbo Frames: 8916) -weave_mtu: 1376 +# Weave deployment +# weave_password: ~ +# weave_checkpoint_disable: false +# weave_conn_limit: 100 +# weave_hairpin_mode: true +# weave_ipalloc_range: {{ kube_pods_subnet }} +# weave_expect_npc: {{ enable_network_policy }} +# weave_kube_peers: ~ +# weave_ipalloc_init: ~ +# weave_expose_ip: ~ +# weave_metrics_addr: ~ +# weave_status_addr: ~ +# weave_mtu: 1376 +# weave_no_masq_local: true +# weave_extra_args: ~ # Enable kubernetes network policies enable_network_policy: false diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index be0d6800b..0fbc77a1d 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -38,7 +38,7 @@ flannel_version: "v0.10.0" flannel_cni_version: "v0.3.0" istio_version: "0.2.6" vault_version: 0.10.1 -weave_version: 2.3.0 +weave_version: "2.4.0" pod_infra_version: 3.0 contiv_version: 1.1.7 cilium_version: "v1.0.0-rc8" diff --git a/roles/kubernetes-apps/network_plugin/weave/tasks/main.yml b/roles/kubernetes-apps/network_plugin/weave/tasks/main.yml index 53ad953b5..44babf343 100644 --- a/roles/kubernetes-apps/network_plugin/weave/tasks/main.yml +++ b/roles/kubernetes-apps/network_plugin/weave/tasks/main.yml @@ -1,4 +1,5 @@ --- + - name: Weave | Start Resources kube: name: "weave-net" @@ -9,13 +10,12 @@ state: "latest" when: inventory_hostname == groups['kube-master'][0] -- name: "Weave | wait for weave to become available" +- name: Weave | Wait for Weave to become available uri: url: http://127.0.0.1:6784/status return_content: yes register: weave_status retries: 180 delay: 5 - until: "{{ weave_status.status == 200 and - 'Status: ready' in weave_status.content }}" + until: "{{ weave_status.status == 200 and 'Status: ready' in weave_status.content }}" when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/network_plugin/weave/defaults/main.yml b/roles/network_plugin/weave/defaults/main.yml index ab955ebef..ee636e56f 100644 --- a/roles/network_plugin/weave/defaults/main.yml +++ b/roles/network_plugin/weave/defaults/main.yml @@ -1,29 +1,58 @@ --- -# Limits -weave_memory_limits: 400M -weave_cpu_limits: 300m -weave_memory_requests: 64M -weave_cpu_requests: 10m -# This two variable are automatically changed by the weave's role, do not manually change these values -# To reset values : -# weave_seed: unset -# weave_peers: unset -weave_seed: uninitialized -weave_peers: uninitialized +# Weave's network password for encryption, if null then no network encryption. +weave_password: ~ -# weave's network password for encryption -# if null then no network encryption -# you can use --extra-vars to pass the password in command line -weave_password: EnterPasswordHere +# If set to 1, disable checking for new Weave Net versions (default is blank, +# i.e. check is enabled) +weave_checkpoint_disable: false -# Weave uses consensus mode by default -# Enabling seed mode allow to dynamically add or remove hosts -# https://www.weave.works/docs/net/latest/ipam/ -weave_mode_seed: false +# Soft limit on the number of connections between peers. Defaults to 100. +weave_conn_limit: 100 -# Set the MTU of Weave (default 1376, Jumbo Frames: 8916) +# Weave Net defaults to enabling hairpin on the bridge side of the veth pair +# for containers attached. If you need to disable hairpin, e.g. your kernel is +# one of those that can panic if hairpin is enabled, then you can disable it by +# setting `HAIRPIN_MODE=false`. +weave_hairpin_mode: true + +# The range of IP addresses used by Weave Net and the subnet they are placed in +# (CIDR format; default 10.32.0.0/12) +weave_ipalloc_range: "{{ kube_pods_subnet }}" + +# Set to 0 to disable Network Policy Controller (default is on) +weave_expect_npc: "{{ enable_network_policy }}" + +# List of addresses of peers in the Kubernetes cluster (default is to fetch the +# list from the api-server) +weave_kube_peers: ~ + +# Set the initialization mode of the IP Address Manager (defaults to consensus +# amongst the KUBE_PEERS) +weave_ipalloc_init: ~ + +# Set the IP address used as a gateway from the Weave network to the host +# network - this is useful if you are configuring the addon as a static pod. +weave_expose_ip: ~ + +# Address and port that the Weave Net daemon will serve Prometheus-style +# metrics on (defaults to 0.0.0.0:6782) +weave_metrics_addr: ~ + +# Address and port that the Weave Net daemon will serve status requests on +# (defaults to disabled) +weave_status_addr: ~ + +# Weave Net defaults to 1376 bytes, but you can set a smaller size if your +# underlying network has a tighter limit, or set a larger size for better +# performance if your network supports jumbo frames (e.g. 8916) weave_mtu: 1376 -# this variable is use in seed mode -weave_ip_current_cluster: "{% for host in groups['k8s-cluster'] %}{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}{% if not loop.last %} {% endif %}{% endfor %}" +# Set to 1 to preserve the client source IP address when accessing Service +# annotated with `service.spec.externalTrafficPolicy=Local`. The feature works +# only with Weave IPAM (default). +weave_no_masq_local: true + +# Extra variables that passing to launch.sh, useful for enabling seed mode, see +# https://www.weave.works/docs/net/latest/tasks/ipam/ipam/ +weave_extra_args: ~ diff --git a/roles/network_plugin/weave/tasks/main.yml b/roles/network_plugin/weave/tasks/main.yml index fb34b1c2f..318b6a369 100644 --- a/roles/network_plugin/weave/tasks/main.yml +++ b/roles/network_plugin/weave/tasks/main.yml @@ -1,12 +1,4 @@ --- -- import_tasks: seed.yml - when: weave_mode_seed - -- name: template weavenet conflist - template: - src: 00-weave.conflist.j2 - dest: /etc/cni/net.d/00-weave.conflist - owner: kube - name: Weave | Copy cni plugins from hyperkube command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -rf /opt/cni/bin/. /cnibindir/" @@ -19,9 +11,12 @@ - hyperkube - upgrade -- name: Weave | Create weave-net manifest +- name: Weave | Create manifest template: src: weave-net.yml.j2 dest: "{{ kube_config_dir }}/weave-net.yml" - mode: 0640 - register: weave_manifest + +- name: Weave | Fix nodePort for Weave + template: + src: 00-weave.conflist.j2 + dest: /etc/cni/net.d/00-weave.conflist diff --git a/roles/network_plugin/weave/tasks/seed.yml b/roles/network_plugin/weave/tasks/seed.yml deleted file mode 100644 index 2765267e5..000000000 --- a/roles/network_plugin/weave/tasks/seed.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -- name: Weave seed | Set seed if first time - set_fact: - seed: '{% for host in groups["k8s-cluster"] %}{{ hostvars[host]["ansible_default_ipv4"]["macaddress"] }}{% if not loop.last %},{% endif %}{% endfor %}' - when: "weave_seed == 'uninitialized'" - run_once: true - tags: - - confweave - -- name: Weave seed | Set seed if not first time - set_fact: - seed: '{{ weave_seed }}' - when: "weave_seed != 'uninitialized'" - run_once: true - tags: - - confweave - -- name: Weave seed | Set peers if fist time - set_fact: - peers: '{{ weave_ip_current_cluster }}' - when: "weave_peers == 'uninitialized'" - run_once: true - tags: - - confweave - -- name: Weave seed | Set peers if existing peers - set_fact: - peers: '{{ weave_peers }}{% for ip in weave_ip_current_cluster.split(" ") %}{% if ip not in weave_peers.split(" ") %} {{ ip }}{% endif %}{% endfor %}' - when: "weave_peers != 'uninitialized'" - run_once: true - tags: - - confweave - -- name: Weave seed | Save seed - lineinfile: - dest: "{{ inventory_dir }}/group_vars/k8s-cluster.yml" - state: present - regexp: '^weave_seed:' - line: 'weave_seed: {{ seed }}' - become: no - delegate_to: 127.0.0.1 - run_once: true - tags: - - confweave - -- name: Weave seed | Save peers - lineinfile: - dest: "{{ inventory_dir }}/group_vars/k8s-cluster.yml" - state: present - regexp: '^weave_peers:' - line: 'weave_peers: {{ peers }}' - become: no - delegate_to: 127.0.0.1 - run_once: true - tags: - - confweave diff --git a/roles/network_plugin/weave/templates/weave-net.yml.j2 b/roles/network_plugin/weave/templates/weave-net.yml.j2 index 9a7da7377..1995b6677 100644 --- a/roles/network_plugin/weave/templates/weave-net.yml.j2 +++ b/roles/network_plugin/weave/templates/weave-net.yml.j2 @@ -15,7 +15,6 @@ items: name: weave-net labels: name: weave-net - namespace: kube-system rules: - apiGroups: - '' @@ -35,13 +34,19 @@ items: - get - list - watch + - apiGroups: + - '' + resources: + - nodes/status + verbs: + - patch + - update - apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: weave-net labels: name: weave-net - namespace: kube-system roleRef: kind: ClusterRole name: weave-net @@ -94,7 +99,6 @@ items: name: weave-net labels: name: weave-net - version: v{{ weave_version }} namespace: kube-system spec: minReadySeconds: 5 @@ -106,31 +110,56 @@ items: containers: - name: weave command: -{% if weave_mode_seed == true %} - - /bin/sh - - -c - - export EXTRA_ARGS=--name=$(cat /sys/class/net/{{ ansible_default_ipv4['interface'] }}/address) && /home/weave/launch.sh -{% else %} - /home/weave/launch.sh -{% endif %} env: - name: HOSTNAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - - name: WEAVE_MTU - value: "{{ weave_mtu }}" - - name: IPALLOC_RANGE - value: {{ kube_pods_subnet }} -{% if weave_mode_seed == true %} - - name: KUBE_PEERS - value: {{ peers }} - - name: IPALLOC_INIT - value: seed={{ seed }} -{% endif %} - name: WEAVE_PASSWORD - value: {{ weave_password }} + valueFrom: + secretKeyRef: + name: weave-net + key: WEAVE_PASSWORD + - name: CHECKPOINT_DISABLE + value: "{{ weave_checkpoint_disable | bool | int }}" + - name: CONN_LIMIT + value: "{{ weave_conn_limit | int }}" + - name: HAIRPIN_MODE + value: "{{ weave_hairpin_mode | bool }}" + - name: IPALLOC_RANGE + value: "{{ weave_ipalloc_range }}" + - name: EXPECT_NPC + value: "{{ weave_expect_npc | bool | int }}" +{% if weave_kube_peers %} + - name: KUBE_PEERS + value: "{{ weave_kube_peers }}" +{% endif %} +{% if weave_ipalloc_init %} + - name: IPALLOC_INIT + value: "{{ weave_ipalloc_init }}" +{% endif %} +{% if weave_expose_ip %} + - name: WEAVE_EXPOSE_IP + value: "{{ weave_expose_ip }}" +{% endif %} +{% if weave_metrics_addr %} + - name: WEAVE_METRICS_ADDR + value: "{{ weave_metrics_addr }}" +{% endif %} +{% if weave_status_addr %} + - name: WEAVE_STATUS_ADDR + value: "{{ weave_status_addr }}" +{% endif %} + - name: WEAVE_MTU + value: "{{ weave_mtu | int }}" + - name: NO_MASQ_LOCAL + value: "{{ weave_no_masq_local | bool | int }}" +{% if weave_extra_args %} + - name: EXTRA_ARGS + value: "{{ weave_extra_args }}" +{% endif %} image: {{ weave_kube_image_repo }}:{{ weave_kube_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} livenessProbe: @@ -141,11 +170,7 @@ items: initialDelaySeconds: 30 resources: requests: - cpu: {{ weave_cpu_requests }} - memory: {{ weave_memory_requests }} - limits: - cpu: {{ weave_cpu_limits }} - memory: {{ weave_memory_limits }} + cpu: 10m securityContext: privileged: true volumeMounts: @@ -175,11 +200,7 @@ items: imagePullPolicy: {{ k8s_image_pull_policy }} resources: requests: - cpu: {{ weave_cpu_requests }} - memory: {{ weave_memory_requests }} - limits: - cpu: {{ weave_cpu_limits }} - memory: {{ weave_memory_limits }} + cpu: 10m securityContext: privileged: true volumeMounts: @@ -216,7 +237,15 @@ items: - name: xtables-lock hostPath: path: /run/xtables.lock + type: FileOrCreate updateStrategy: rollingUpdate: maxUnavailable: {{ serial | default('20%') }} type: RollingUpdate + - apiVersion: v1 + kind: Secret + metadata: + name: weave-net + namespace: kube-system + data: + WEAVE_PASSWORD: "{{ weave_password | default("") | b64encode }}"