From 58f48500b1adac3f18466fa1c5cf8aa9d9838150 Mon Sep 17 00:00:00 2001 From: Florian Ruynat Date: Wed, 15 Apr 2020 08:48:02 +0200 Subject: [PATCH] Update Flannel manifests, install script and version (0.12) + fix tests scripts (#5937) * Add CI_TEST_VARS to tests * Update flannel to 0.12.0 (with new manifests) and disable tx/rx offloading in networking test --- README.md | 2 +- docs/flannel.md | 6 ++ roles/download/defaults/main.yml | 2 +- .../network_plugin/flannel/defaults/main.yml | 3 - roles/network_plugin/flannel/tasks/main.yml | 16 ++++ .../flannel/templates/cni-flannel-rbac.yml.j2 | 65 +++++++++++-- .../flannel/templates/cni-flannel.yml.j2 | 92 +++++++++---------- tests/files/packet_centos7-flannel-addons.yml | 3 - tests/scripts/testcases_run.sh | 10 +- tests/testcases/040_check-network-adv.yml | 6 ++ 10 files changed, 136 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index 57f4b1139..97cf78593 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ Note: Upstart/SysV init based OS types are not supported. - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) - [cilium](https://github.com/cilium/cilium) v1.5.5 - [contiv](https://github.com/contiv/install) v1.2.1 - - [flanneld](https://github.com/coreos/flannel) v0.11.0 + - [flanneld](https://github.com/coreos/flannel) v0.12.0 - [kube-router](https://github.com/cloudnativelabs/kube-router) v0.4.0 - [multus](https://github.com/intel/multus-cni) v3.4.1 - [weave](https://github.com/weaveworks/weave) v2.5.2 diff --git a/docs/flannel.md b/docs/flannel.md index bea14d79c..d4a9b9713 100644 --- a/docs/flannel.md +++ b/docs/flannel.md @@ -1,5 +1,11 @@ # Flannel +Flannel is a network fabric for containers, designed for Kubernetes + +**Warning:** You may encounter this [bug](https://github.com/coreos/flannel/pull/1282) with `VXLAN` backend, while waiting on a newer Flannel version the current workaround (`ethtool --offload flannel.1 rx off tx off`) is showcase in kubespray [networking test](tests/testcases/040_check-network-adv.yml:31). + +## Verifying flannel install + * Flannel configuration file should have been created there ```ShellSession diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 5d5cf8a60..f78fb662e 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -72,7 +72,7 @@ calico_policy_version: "v3.13.2" calico_typha_version: "v3.13.2" typha_enabled: false -flannel_version: "v0.11.0" +flannel_version: "v0.12.0" flannel_cni_version: "v0.3.0" cni_version: "v0.8.5" diff --git a/roles/network_plugin/flannel/defaults/main.yml b/roles/network_plugin/flannel/defaults/main.yml index f7f773fdc..09b18fd3b 100644 --- a/roles/network_plugin/flannel/defaults/main.yml +++ b/roles/network_plugin/flannel/defaults/main.yml @@ -23,6 +23,3 @@ flannel_memory_limit: 500M flannel_cpu_limit: 300m flannel_memory_requests: 64M flannel_cpu_requests: 150m - -# Legacy directory, will be removed if found. -flannel_cert_dir: /etc/flannel/certs diff --git a/roles/network_plugin/flannel/tasks/main.yml b/roles/network_plugin/flannel/tasks/main.yml index d5a725baf..d187a8e19 100644 --- a/roles/network_plugin/flannel/tasks/main.yml +++ b/roles/network_plugin/flannel/tasks/main.yml @@ -9,3 +9,19 @@ register: flannel_node_manifests when: - inventory_hostname in groups['kube-master'] + +- name: Flannel | Set CNI directory permissions + file: + path: /opt/cni/bin + state: directory + owner: kube + recurse: true + mode: 0755 + register: cni_bin_dir + +- name: Flannel | Copy CNI plugins + unarchive: + src: "{{ local_release_dir }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz" + dest: "/opt/cni/bin" + mode: 0755 + remote_src: yes diff --git a/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 index ceb2c09e0..ce4980ccb 100644 --- a/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 +++ b/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 @@ -3,13 +3,66 @@ apiVersion: v1 kind: ServiceAccount metadata: name: flannel - namespace: "kube-system" + namespace: kube-system +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: psp.flannel.unprivileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default + seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default +{% if podsecuritypolicy_enabled and apparmor_enabled %} + apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default + apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default +{% endif %} +spec: + privileged: false + volumes: + - configMap + - secret + - emptyDir + - hostPath + allowedHostPaths: + - pathPrefix: "/etc/cni/net.d" + - pathPrefix: "/etc/kube-flannel" + - pathPrefix: "/run/flannel" + readOnlyRootFilesystem: false + # Users and groups + runAsUser: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + fsGroup: + rule: RunAsAny + # Privilege Escalation + allowPrivilegeEscalation: false + defaultAllowPrivilegeEscalation: false + # Capabilities + allowedCapabilities: ['NET_ADMIN'] + defaultAddCapabilities: [] + requiredDropCapabilities: [] + # Host namespaces + hostPID: false + hostIPC: false + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + # SELinux + seLinux: + # SELinux is unused in CaaSP + rule: 'RunAsAny' --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: flannel rules: + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['psp.flannel.unprivileged'] - apiGroups: - "" resources: @@ -29,14 +82,6 @@ rules: - nodes/status verbs: - patch - - apiGroups: - - policy - resourceNames: - - privileged - resources: - - podsecuritypolicies - verbs: - - use --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -49,4 +94,4 @@ roleRef: subjects: - kind: ServiceAccount name: flannel - namespace: "kube-system" \ No newline at end of file + namespace: kube-system diff --git a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 index 9afabd151..135be5c94 100644 --- a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 +++ b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 @@ -3,28 +3,27 @@ kind: ConfigMap apiVersion: v1 metadata: name: kube-flannel-cfg - namespace: "kube-system" + namespace: kube-system labels: tier: node app: flannel data: cni-conf.json: | { - "name":"cni0", - "cniVersion":"0.3.1", - "plugins":[ + "name": "cbr0", + "cniVersion": "0.3.1", + "plugins": [ { - "type":"flannel", - "delegate":{ - "forceAddress":true, + "type": "flannel", + "delegate": { "hairpinMode": true, - "isDefaultGateway":true + "isDefaultGateway": true } }, { - "type":"portmap", - "capabilities":{ - "portMappings":true + "type": "portmap", + "capabilities": { + "portMappings": true } } ] @@ -41,25 +40,22 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: kube-flannel - namespace: "kube-system" + namespace: kube-system labels: tier: node - k8s-app: flannel + app: flannel spec: selector: matchLabels: - tier: node - k8s-app: flannel + app: flannel template: metadata: labels: tier: node - k8s-app: flannel + app: flannel spec: priorityClassName: system-node-critical serviceAccountName: flannel - nodeSelector: - beta.kubernetes.io/os: linux containers: - name: kube-flannel image: {{ flannel_image_repo }}:{{ flannel_image_tag }} @@ -73,7 +69,9 @@ spec: memory: {{ flannel_memory_requests }} command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"{% if flannel_interface is defined %}, "--iface={{ flannel_interface }}"{% endif %}{% if flannel_interface_regexp is defined %}, "--iface-regex={{ flannel_interface_regexp }}"{% endif %} ] securityContext: - privileged: true + privileged: false + capabilities: + add: ["NET_ADMIN"] env: - name: POD_NAME valueFrom: @@ -83,45 +81,47 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP volumeMounts: - name: run - mountPath: /run + mountPath: /run/flannel + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - linux + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + initContainers: + - name: install-cni + image: {{ flannel_cni_image_repo }}:{{ flannel_cni_image_tag }} + command: + - cp + args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conflist + volumeMounts: - name: cni mountPath: /etc/cni/net.d - name: flannel-cfg mountPath: /etc/kube-flannel/ - - name: install-cni - image: {{ flannel_cni_image_repo }}:{{ flannel_cni_image_tag }} - command: ["/install-cni.sh"] - env: - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: kube-flannel-cfg - key: cni-conf.json - - name: CNI_CONF_NAME - value: "10-flannel.conflist" - volumeMounts: - - name: cni - mountPath: /host/etc/cni/net.d - - name: host-cni-bin - mountPath: /host/opt/cni/bin/ hostNetwork: true dnsPolicy: ClusterFirstWithHostNet tolerations: - - operator: Exists - # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - - key: CriticalAddonsOnly - operator: "Exists" + - operator: Exists + effect: NoSchedule volumes: - name: run hostPath: - path: /run + path: /run/flannel - name: cni hostPath: path: /etc/cni/net.d diff --git a/tests/files/packet_centos7-flannel-addons.yml b/tests/files/packet_centos7-flannel-addons.yml index 60ccb69c0..a90e5eaf5 100644 --- a/tests/files/packet_centos7-flannel-addons.yml +++ b/tests/files/packet_centos7-flannel-addons.yml @@ -28,6 +28,3 @@ enable_nodelocaldns: false kube_oidc_url: https://accounts.google.com/.well-known/openid-configuration kube_oidc_client_id: kubespray-example - -# Temp set k8s ver to 1.16.8 -kube_version: v1.16.8 \ No newline at end of file diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh index 85d28c2f1..e406efc3f 100755 --- a/tests/scripts/testcases_run.sh +++ b/tests/scripts/testcases_run.sh @@ -52,19 +52,19 @@ fi # Tests Cases ## Test Master API -ansible-playbook --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $ANSIBLE_LOG_LEVEL +ansible-playbook --limit "all:!fake_hosts" -e @${CI_TEST_VARS} tests/testcases/010_check-apiserver.yml $ANSIBLE_LOG_LEVEL ## Test that all pods are Running -ansible-playbook --limit "all:!fake_hosts" tests/testcases/015_check-pods-running.yml $ANSIBLE_LOG_LEVEL +ansible-playbook --limit "all:!fake_hosts" -e @${CI_TEST_VARS} tests/testcases/015_check-pods-running.yml $ANSIBLE_LOG_LEVEL ## Test that all nodes are Ready -ansible-playbook --limit "all:!fake_hosts" tests/testcases/020_check-nodes-ready.yml $ANSIBLE_LOG_LEVEL +ansible-playbook --limit "all:!fake_hosts" -e @${CI_TEST_VARS} tests/testcases/020_check-nodes-ready.yml $ANSIBLE_LOG_LEVEL ## Test pod creation and ping between them -ansible-playbook --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $ANSIBLE_LOG_LEVEL +ansible-playbook --limit "all:!fake_hosts" -e @${CI_TEST_VARS} tests/testcases/030_check-network.yml $ANSIBLE_LOG_LEVEL ## Advanced DNS checks -ansible-playbook --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml $ANSIBLE_LOG_LEVEL +ansible-playbook --limit "all:!fake_hosts" -e @${CI_TEST_VARS} tests/testcases/040_check-network-adv.yml $ANSIBLE_LOG_LEVEL ## Kubernetes conformance tests ansible-playbook -i ${ANSIBLE_INVENTORY} -e @${CI_TEST_VARS} --limit "all:!fake_hosts" tests/testcases/100_check-k8s-conformance.yml $ANSIBLE_LOG_LEVEL diff --git a/tests/testcases/040_check-network-adv.yml b/tests/testcases/040_check-network-adv.yml index de2197e09..9818ef38d 100644 --- a/tests/testcases/040_check-network-adv.yml +++ b/tests/testcases/040_check-network-adv.yml @@ -14,6 +14,12 @@ netchecker_port: 31081 tasks: + - name: Flannel | Disable tx and rx offloading on VXLAN interfaces (see https://github.com/coreos/flannel/pull/1282) + shell: "ethtool --offload flannel.1 rx off tx off" + ignore_errors: true + when: + - kube_network_plugin == 'flannel' + - name: Force binaries directory for Container Linux by CoreOS and Flatcar set_fact: bin_dir: "/opt/bin"