From c929b5e82e46465f73febb222f24da16de58e845 Mon Sep 17 00:00:00 2001 From: Florian Ruynat Date: Wed, 15 Apr 2020 12:10:03 +0200 Subject: [PATCH] Upgrade kube-ovn to v1.1.0 and move test from centos7 to centos8 (#5852) --- .gitlab-ci/packet.yml | 2 +- docs/kube-ovn.md | 6 + roles/download/defaults/main.yml | 48 +---- .../network_plugin/kube-ovn/defaults/main.yml | 7 +- .../templates/cni-kube-ovn-crd.yml.j2 | 19 +- .../kube-ovn/templates/cni-kube-ovn.yml.j2 | 195 ++++++++++++++++-- .../kube-ovn/templates/cni-ovn.yml.j2 | 71 +++++-- ...be-ovn.yml => packet_centos8-kube-ovn.yml} | 7 +- 8 files changed, 271 insertions(+), 84 deletions(-) rename tests/files/{packet_centos7-kube-ovn.yml => packet_centos8-kube-ovn.yml} (55%) diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml index 3f83db566..ca1da70fc 100644 --- a/.gitlab-ci/packet.yml +++ b/.gitlab-ci/packet.yml @@ -108,7 +108,7 @@ packet_centos7-calico-ha-once-localhost: services: - docker:18.09.9-dind -packet_centos7-kube-ovn: +packet_centos8-kube-ovn: stage: deploy-part2 extends: .packet when: on_success diff --git a/docs/kube-ovn.md b/docs/kube-ovn.md index d811b13d1..375c7a4d5 100644 --- a/docs/kube-ovn.md +++ b/docs/kube-ovn.md @@ -4,6 +4,12 @@ Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It off For more information please check [Kube-OVN documentation](https://github.com/alauda/kube-ovn) +**Warning:** Kernel version (`cat /proc/version`) needs to be different than `3.10.0-862` or kube-ovn won't start and will print this message: + +```bash +kernel version 3.10.0-862 has a nat related bug that will affect ovs function, please update to a version greater than 3.10.0-898 +``` + ## How to use it Enable kube-ovn in `group_vars/k8s-cluster/k8s-cluster.yml` diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index f78fb662e..202378bec 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -63,6 +63,9 @@ docker_image_repo: "docker.io" # quay image repo define quay_image_repo: "quay.io" +# alauda.cn image repo (for kube-ovn...) +alauda_image_repo: "index.alauda.cn" + # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download calico_version: "v3.13.2" @@ -81,7 +84,7 @@ weave_version: 2.6.2 pod_infra_version: 3.1 contiv_version: 1.2.1 cilium_version: "v1.7.2" -kube_ovn_version: "v0.6.0" +kube_ovn_version: "v1.1.0" kube_router_version: "v0.4.0" multus_version: "v3.4.1" @@ -472,14 +475,8 @@ cilium_init_image_repo: "{{ docker_image_repo }}/cilium/cilium-init" cilium_init_image_tag: "2019-04-05" cilium_operator_image_repo: "{{ docker_image_repo }}/cilium/operator" cilium_operator_image_tag: "{{ cilium_version }}" -kube_ovn_db_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-db" -kube_ovn_node_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-node" -kube_ovn_cni_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-cni" -kube_ovn_controller_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-controller" -kube_ovn_db_image_tag: "{{ kube_ovn_version }}" -kube_ovn_node_image_tag: "{{ kube_ovn_version }}" -kube_ovn_controller_image_tag: "{{ kube_ovn_version }}" -kube_ovn_cni_image_tag: "{{ kube_ovn_version }}" +kube_ovn_container_image_repo: "{{ alauda_image_repo }}/alaudak8s/kube-ovn" +kube_ovn_container_image_tag: "{{ kube_ovn_version }}" kube_router_image_repo: "{{ docker_image_repo }}/cloudnativelabs/kube-router" kube_router_image_tag: "{{ kube_router_version }}" multus_image_repo: "{{ docker_image_repo }}/nfvpe/multus" @@ -836,38 +833,11 @@ downloads: groups: - k8s-cluster - kube_ovn_db: + kube_ovn: enabled: "{{ kube_network_plugin == 'kube-ovn' }}" container: true - repo: "{{ kube_ovn_db_image_repo }}" - tag: "{{ kube_ovn_db_image_tag }}" - sha256: "{{ kube_ovn_digest_checksum|default(None) }}" - groups: - - k8s-cluster - - kube_ovn_node: - enabled: "{{ kube_network_plugin == 'kube-ovn' }}" - container: true - repo: "{{ kube_ovn_node_image_repo }}" - tag: "{{ kube_ovn_node_image_tag }}" - sha256: "{{ kube_ovn_digest_checksum|default(None) }}" - groups: - - k8s-cluster - - kube_ovn_controller: - enabled: "{{ kube_network_plugin == 'kube-ovn' }}" - container: true - repo: "{{ kube_ovn_controller_image_repo }}" - tag: "{{ kube_ovn_controller_image_tag }}" - sha256: "{{ kube_ovn_digest_checksum|default(None) }}" - groups: - - k8s-cluster - - kube_ovn_cni: - enabled: "{{ kube_network_plugin == 'kube-ovn' }}" - container: true - repo: "{{ kube_ovn_cni_image_repo }}" - tag: "{{ kube_ovn_cni_image_tag }}" + repo: "{{ kube_ovn_container_image_repo }}" + tag: "{{ kube_ovn_container_image_tag }}" sha256: "{{ kube_ovn_digest_checksum|default(None) }}" groups: - k8s-cluster diff --git a/roles/network_plugin/kube-ovn/defaults/main.yml b/roles/network_plugin/kube-ovn/defaults/main.yml index fb0cfc63d..9cc4c5c84 100644 --- a/roles/network_plugin/kube-ovn/defaults/main.yml +++ b/roles/network_plugin/kube-ovn/defaults/main.yml @@ -7,5 +7,10 @@ kube_ovn_node_cpu_request: 100m kube_ovn_node_memory_request: 300Mi kube_ovn_node_cpu_limit: 200m kube_ovn_node_memory_limit: 500Mi +kube_ovn_pinger_cpu_request: 100m +kube_ovn_pinger_memory_request: 300Mi +kube_ovn_pinger_cpu_limit: 200m +kube_ovn_pinger_memory_limit: 400Mi -traffic_mirror: true \ No newline at end of file +traffic_mirror: true +encap_checksum: true \ No newline at end of file diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 index 2f3c99f9b..2a16dcaf4 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 @@ -40,7 +40,12 @@ spec: kind: Subnet shortNames: - subnet + subresources: + status: {} additionalPrinterColumns: + - name: Provider + type: string + JSONPath: .spec.provider - name: Protocol type: string JSONPath: .spec.protocol @@ -53,11 +58,23 @@ spec: - name: NAT type: boolean JSONPath: .spec.natOutgoing + - name: Default + type: boolean + JSONPath: .spec.default + - name: GatewayType + type: string + JSONPath: .spec.gatewayType + - name: Used + type: integer + JSONPath: .status.usingIPs + - name: Available + type: integer + JSONPath: .status.availableIPs validation: openAPIV3Schema: properties: spec: - required: ["cidrBlock","gateway"] + required: ["cidrBlock"] properties: cidrBlock: type: "string" diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 index ea7fcf93b..c67eaf500 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 @@ -34,11 +34,12 @@ spec: matchLabels: app: kube-ovn-controller topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true containers: - name: kube-ovn-controller - image: {{ kube_ovn_controller_image_repo }}:{{ kube_ovn_controller_image_tag }} + image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: - /kube-ovn/start-controller.sh @@ -60,25 +61,19 @@ spec: readinessProbe: exec: command: - - nc - - -z - - -w3 - - 127.0.0.1 - - "10660" + - sh + - /kube-ovn/kube-ovn-controller-healthcheck.sh periodSeconds: 3 livenessProbe: exec: command: - - nc - - -z - - -w3 - - 127.0.0.1 - - "10660" + - sh + - /kube-ovn/kube-ovn-controller-healthcheck.sh initialDelaySeconds: 30 periodSeconds: 7 failureThreshold: 5 nodeSelector: - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" --- kind: DaemonSet @@ -94,7 +89,7 @@ spec: matchLabels: app: kube-ovn-cni updateStrategy: - type: RollingUpdate + type: OnDelete template: metadata: labels: @@ -105,14 +100,18 @@ spec: tolerations: - operator: Exists effect: NoSchedule + priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true hostPID: true initContainers: - name: install-cni - image: {{ kube_ovn_cni_image_repo }}:{{ kube_ovn_cni_image_tag }} + image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: ["/kube-ovn/install-cni.sh"] + securityContext: + runAsUser: 0 + privileged: true volumeMounts: - mountPath: /etc/cni/net.d name: cni-conf @@ -120,16 +119,18 @@ spec: name: cni-bin containers: - name: cni-server - image: {{ kube_ovn_cni_image_repo }}:{{ kube_ovn_cni_image_tag }} + image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} command: - sh - /kube-ovn/start-cniserver.sh args: - --enable-mirror={{ traffic_mirror }} + - --encap-checksum={{ encap_checksum }} + - --service-cluster-ip-range={{ kube_service_addresses }} securityContext: - runAsUser: 0 - privileged: true + capabilities: + add: ["NET_ADMIN", "SYS_ADMIN", "SYS_PTRACE"] env: - name: POD_IP valueFrom: @@ -142,6 +143,11 @@ spec: volumeMounts: - mountPath: /run/openvswitch name: host-run-ovs + - mountPath: /run/ovn + name: host-run-ovn + - mountPath: /var/run/netns + name: host-ns + mountPropagation: HostToContainer readinessProbe: exec: command: @@ -163,14 +169,165 @@ spec: periodSeconds: 7 failureThreshold: 5 nodeSelector: - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" volumes: - name: host-run-ovs hostPath: path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn - name: cni-conf hostPath: path: /etc/cni/net.d - name: cni-bin hostPath: - path: /opt/cni/bin \ No newline at end of file + path: /opt/cni/bin + - name: host-ns + hostPath: + path: /var/run/netns + +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: kube-ovn-pinger + namespace: kube-ovn + annotations: + kubernetes.io/description: | + This daemon set launches the openvswitch daemon. +spec: + selector: + matchLabels: + app: kube-ovn-pinger + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: kube-ovn-pinger + component: network + type: infra + spec: + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: ovn + hostPID: true + containers: + - name: pinger + image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} + command: ["/kube-ovn/kube-ovn-pinger", "--external-address=114.114.114.114"] + imagePullPolicy: {{ k8s_image_pull_policy }} + securityContext: + runAsUser: 0 + privileged: false + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /lib/modules + name: host-modules + readOnly: true + - mountPath: /run/openvswitch + name: host-run-ovs + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + resources: + requests: + cpu: {{ kube_ovn_pinger_cpu_request }} + memory: {{ kube_ovn_pinger_memory_request }} + limits: + cpu: {{ kube_ovn_pinger_cpu_limit }} + memory: {{ kube_ovn_pinger_memory_limit }} + nodeSelector: + kubernetes.io/os: "linux" + volumes: + - name: host-modules + hostPath: + path: /lib/modules + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-sys + hostPath: + path: /sys + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-pinger + namespace: kube-ovn + labels: + app: kube-ovn-pinger +spec: + selector: + app: kube-ovn-pinger + ports: + - port: 8080 + name: metrics +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-controller + namespace: kube-ovn + labels: + app: kube-ovn-controller +spec: + selector: + app: kube-ovn-controller + ports: + - port: 10660 + name: metrics +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-cni + namespace: kube-ovn + labels: + app: kube-ovn-cni +spec: + selector: + app: kube-ovn-cni + ports: + - port: 10665 + name: metrics \ No newline at end of file diff --git a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 index f86967b35..e4ee7e12c 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 @@ -29,6 +29,7 @@ rules: - "kubeovn.io" resources: - subnets + - subnets/status - ips verbs: - "*" @@ -55,6 +56,7 @@ rules: - services - endpoints - statefulsets + - daemonsets verbs: - get - list @@ -97,6 +99,7 @@ spec: type: ClusterIP selector: app: ovn-central + ovn-nb-leader: "true" sessionAffinity: None --- @@ -114,6 +117,7 @@ spec: type: ClusterIP selector: app: ovn-central + ovn-sb-leader: "true" sessionAffinity: None --- @@ -152,17 +156,30 @@ spec: matchLabels: app: ovn-central topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true containers: - name: ovn-central - image: {{ kube_ovn_db_image_repo }}:{{ kube_ovn_db_image_tag }} + image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} + command: ["/kube-ovn/start-db.sh"] + securityContext: + capabilities: + add: ["SYS_NICE"] env: - name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace resources: requests: cpu: {{ kube_ovn_db_cpu_request }} @@ -171,47 +188,55 @@ spec: cpu: {{ kube_ovn_db_cpu_limit }} memory: {{ kube_ovn_db_memory_limit }} volumeMounts: - - mountPath: /run/openvswitch - name: host-run-ovs - mountPath: /var/run/openvswitch name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn - mountPath: /sys name: host-sys readOnly: true - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /var/log/openvswitch - name: host-log + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn readinessProbe: exec: command: - sh - - /root/ovn-is-leader.sh + - /kube-ovn/ovn-is-leader.sh periodSeconds: 3 livenessProbe: exec: command: - sh - - /root/ovn-healthcheck.sh + - /kube-ovn/ovn-healthcheck.sh initialDelaySeconds: 30 periodSeconds: 7 failureThreshold: 5 nodeSelector: - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" kube-ovn/role: "master" volumes: - name: host-run-ovs hostPath: path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn - name: host-sys hostPath: path: /sys - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch - - name: host-log + - name: host-log-ovs hostPath: path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn --- kind: DaemonSet @@ -227,7 +252,7 @@ spec: matchLabels: app: ovs updateStrategy: - type: RollingUpdate + type: OnDelete template: metadata: labels: @@ -238,13 +263,15 @@ spec: tolerations: - operator: Exists effect: NoSchedule + priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true hostPID: true containers: - name: openvswitch - image: {{ kube_ovn_node_image_repo }}:{{ kube_ovn_node_image_tag }} + image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} + command: ["/kube-ovn/start-ovs.sh"] securityContext: runAsUser: 0 privileged: true @@ -257,28 +284,30 @@ spec: - mountPath: /lib/modules name: host-modules readOnly: true - - mountPath: /run/openvswitch - name: host-run-ovs - mountPath: /var/run/openvswitch name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn - mountPath: /sys name: host-sys readOnly: true - mountPath: /etc/openvswitch name: host-config-openvswitch - mountPath: /var/log/openvswitch - name: host-log + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn readinessProbe: exec: command: - sh - - /root/ovs-healthcheck.sh + - /kube-ovn/ovs-healthcheck.sh periodSeconds: 5 livenessProbe: exec: command: - sh - - /root/ovs-healthcheck.sh + - /kube-ovn/ovs-healthcheck.sh initialDelaySeconds: 10 periodSeconds: 5 failureThreshold: 5 @@ -290,7 +319,7 @@ spec: cpu: {{ kube_ovn_node_cpu_limit }} memory: {{ kube_ovn_node_memory_limit }} nodeSelector: - beta.kubernetes.io/os: "linux" + kubernetes.io/os: "linux" volumes: - name: host-modules hostPath: @@ -298,12 +327,18 @@ spec: - name: host-run-ovs hostPath: path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn - name: host-sys hostPath: path: /sys - name: host-config-openvswitch hostPath: path: /etc/origin/openvswitch - - name: host-log + - name: host-log-ovs hostPath: - path: /var/log/openvswitch \ No newline at end of file + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn \ No newline at end of file diff --git a/tests/files/packet_centos7-kube-ovn.yml b/tests/files/packet_centos8-kube-ovn.yml similarity index 55% rename from tests/files/packet_centos7-kube-ovn.yml rename to tests/files/packet_centos8-kube-ovn.yml index e954a887b..6fab50204 100644 --- a/tests/files/packet_centos7-kube-ovn.yml +++ b/tests/files/packet_centos8-kube-ovn.yml @@ -1,12 +1,9 @@ --- # Instance settings -cloud_image: centos-7 +cloud_image: centos-8 mode: default # Kubespray settings kube_network_plugin: kube-ovn deploy_netchecker: true -dns_min_replicas: 1 - -# Temp set k8s ver to 1.16.8 -kube_version: v1.16.7 \ No newline at end of file +dns_min_replicas: 1 \ No newline at end of file