From f1d0d1a9fe76aeaa42e221e60f73da6408418e3f Mon Sep 17 00:00:00 2001 From: Samuel Liu Date: Fri, 9 Dec 2022 07:58:11 +0800 Subject: [PATCH] [kube-ovn]: update version v1.10.7 (#9527) * [kube-ovn]: update version * update readme --- README.md | 2 +- roles/download/defaults/main.yml | 2 +- .../network_plugin/kube-ovn/defaults/main.yml | 10 + .../templates/cni-kube-ovn-crd.yml.j2 | 825 +++++++++++++----- .../kube-ovn/templates/cni-kube-ovn.yml.j2 | 75 +- .../kube-ovn/templates/cni-ovn.yml.j2 | 32 +- 6 files changed, 701 insertions(+), 245 deletions(-) diff --git a/README.md b/README.md index a803767c7..f088466b7 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ Note: Upstart/SysV init based OS types are not supported. - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) - [cilium](https://github.com/cilium/cilium) v1.12.1 - [flannel](https://github.com/flannel-io/flannel) v0.19.2 - - [kube-ovn](https://github.com/alauda/kube-ovn) v1.9.7 + - [kube-ovn](https://github.com/alauda/kube-ovn) v1.10.7 - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.5.1 - [multus](https://github.com/intel/multus-cni) v3.8 - [weave](https://github.com/weaveworks/weave) v2.8.1 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 5d0320cae..3e548438c 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -120,7 +120,7 @@ cilium_version: "v1.12.1" cilium_cli_version: "v0.12.5" cilium_enable_hubble: false -kube_ovn_version: "v1.9.7" +kube_ovn_version: "v1.10.7" kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}" kube_router_version: "v1.5.1" multus_version: "v3.8-{{ image_arch }}" diff --git a/roles/network_plugin/kube-ovn/defaults/main.yml b/roles/network_plugin/kube-ovn/defaults/main.yml index 78862b71d..430f69829 100644 --- a/roles/network_plugin/kube-ovn/defaults/main.yml +++ b/roles/network_plugin/kube-ovn/defaults/main.yml @@ -86,3 +86,13 @@ kube_ovn_enable_ssl: false ## dpdk kube_ovn_dpdk_enabled: false +kube_ovn_dpdk_tunnel_iface: br-phy + +## eip snat +kube_ovn_eip_snat_enabled: true + +## keep vm ip +kube_ovn_keep_vm_ip: true + +## cni config priority, default: 01 +kube_ovn_cni_config_priority: 01 diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 index 5aa61f4a5..5878d2cc9 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 @@ -1,5 +1,508 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + name: vpc-nat-gateways.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vpc-nat-gateways + singular: vpc-nat-gateway + shortNames: + - vpc-nat-gw + kind: VpcNatGateway + listKind: VpcNatGatewayList + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vpc + name: Vpc + type: string + - jsonPath: .spec.subnet + name: Subnet + type: string + - jsonPath: .spec.lanIp + name: LanIP + type: string + name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + lanIp: + type: string + subnet: + type: string + vpc: + type: string + selector: + type: array + items: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: iptables-eips.kubeovn.io +spec: + group: kubeovn.io + names: + plural: iptables-eips + singular: iptables-eip + shortNames: + - eip + kind: IptablesEIP + listKind: IptablesEIPList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .status.ip + name: IP + type: string + - jsonPath: .spec.macAddress + name: Mac + type: string + - jsonPath: .status.nat + name: Nat + type: string + - jsonPath: .spec.natGwDp + name: NatGwDp + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + ip: + type: string + nat: + type: string + redo: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + v4ip: + type: string + v6ip: + type: string + macAddress: + type: string + natGwDp: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: iptables-fip-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: iptables-fip-rules + singular: iptables-fip-rule + shortNames: + - fip + kind: IptablesFIPRule + listKind: IptablesFIPRuleList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .spec.eip + name: Eip + type: string + - jsonPath: .status.v4ip + name: V4ip + type: string + - jsonPath: .spec.internalIp + name: InternalIp + type: string + - jsonPath: .status.v6ip + name: V6ip + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + - jsonPath: .status.natGwDp + name: NatGwDp + type: string + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4ip: + type: string + v6ip: + type: string + natGwDp: + type: string + redo: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + eip: + type: string + internalIp: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: iptables-dnat-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: iptables-dnat-rules + singular: iptables-dnat-rule + shortNames: + - dnat + kind: IptablesDnatRule + listKind: IptablesDnatRuleList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .spec.eip + name: Eip + type: string + - jsonPath: .spec.protocol + name: Protocol + type: string + - jsonPath: .status.v4ip + name: V4ip + type: string + - jsonPath: .status.v6ip + name: V6ip + type: string + - jsonPath: .spec.internalIp + name: InternalIp + type: string + - jsonPath: .spec.externalPort + name: ExternalPort + type: string + - jsonPath: .spec.internalPort + name: InternalPort + type: string + - jsonPath: .status.natGwDp + name: NatGwDp + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4ip: + type: string + v6ip: + type: string + natGwDp: + type: string + redo: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + eip: + type: string + externalPort: + type: string + protocol: + type: string + internalIp: + type: string + internalPort: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: iptables-snat-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: iptables-snat-rules + singular: iptables-snat-rule + shortNames: + - snat + kind: IptablesSnatRule + listKind: IptablesSnatRuleList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - jsonPath: .spec.eip + name: EIP + type: string + - jsonPath: .status.v4ip + name: V4ip + type: string + - jsonPath: .status.v6ip + name: V6ip + type: string + - jsonPath: .spec.internalCIDR + name: InternalCIDR + type: string + - jsonPath: .status.natGwDp + name: NatGwDp + type: string + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4ip: + type: string + v6ip: + type: string + natGwDp: + type: string + redo: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + eip: + type: string + internalCIDR: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vpcs.kubeovn.io +spec: + group: kubeovn.io + versions: + - additionalPrinterColumns: + - jsonPath: .status.standby + name: Standby + type: boolean + - jsonPath: .status.subnets + name: Subnets + type: string + - jsonPath: .spec.namespaces + name: Namespaces + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + namespaces: + items: + type: string + type: array + staticRoutes: + items: + properties: + policy: + type: string + cidr: + type: string + nextHopIP: + type: string + type: object + type: array + policyRoutes: + items: + properties: + priority: + type: integer + action: + type: string + match: + type: string + nextHopIP: + type: string + type: object + type: array + vpcPeerings: + items: + properties: + remoteVpc: + type: string + localConnectIP: + type: string + type: object + type: array + type: object + status: + properties: + conditions: + items: + properties: + lastTransitionTime: + type: string + lastUpdateTime: + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + type: object + type: array + default: + type: boolean + defaultLogicalSwitch: + type: string + router: + type: string + standby: + type: boolean + subnets: + items: + type: string + type: array + vpcPeerings: + items: + type: string + type: array + tcpLoadBalancer: + type: string + tcpSessionLoadBalancer: + type: string + udpLoadBalancer: + type: string + udpSessionLoadBalancer: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} + names: + kind: Vpc + listKind: VpcList + plural: vpcs + shortNames: + - vpc + singular: vpc + scope: Cluster +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: name: ips.kubeovn.io spec: @@ -73,6 +576,111 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + name: vips.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vips + singular: vip + shortNames: + - vip + kind: Vip + listKind: VipList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: V4IP + type: string + jsonPath: .spec.v4ip + - name: PV4IP + type: string + jsonPath: .spec.parentV4ip + - name: Mac + type: string + jsonPath: .spec.macAddress + - name: PMac + type: string + jsonPath: .spec.ParentMac + - name: V6IP + type: string + jsonPath: .spec.v6ip + - name: PV6IP + type: string + jsonPath: .spec.parentV6ip + - name: Subnet + type: string + jsonPath: .spec.subnet + - jsonPath: .status.ready + name: Ready + type: boolean + schema: + openAPIV3Schema: + type: object + properties: + status: + type: object + properties: + ready: + type: boolean + v4ip: + type: string + v6ip: + type: string + mac: + type: string + pv4ip: + type: string + pv6ip: + type: string + pmac: + type: string + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string + spec: + type: object + properties: + namespace: + type: string + subnet: + type: string + attachSubnets: + type: array + items: + type: string + v4ip: + type: string + macAddress: + type: string + v6ip: + type: string + parentV4ip: + type: string + parentMac: + type: string + parentV6ip: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: name: subnets.kubeovn.io spec: @@ -228,6 +836,8 @@ spec: type: boolean disableInterConnection: type: boolean + htbqos: + type: string enableDHCP: type: boolean dhcpV4Options: @@ -238,8 +848,6 @@ spec: type: boolean ipv6RAConfigs: type: string - htbqos: - type: string acls: type: array items: @@ -373,6 +981,8 @@ spec: type: array items: type: string + exchangeLinkName: + type: boolean excludeNodes: type: array items: @@ -427,217 +1037,6 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition -metadata: - name: vpcs.kubeovn.io -spec: - group: kubeovn.io - versions: - - additionalPrinterColumns: - - jsonPath: .status.standby - name: Standby - type: boolean - - jsonPath: .status.subnets - name: Subnets - type: string - - jsonPath: .spec.namespaces - name: Namespaces - type: string - name: v1 - schema: - openAPIV3Schema: - properties: - spec: - properties: - namespaces: - items: - type: string - type: array - staticRoutes: - items: - properties: - policy: - type: string - cidr: - type: string - nextHopIP: - type: string - type: object - type: array - policyRoutes: - items: - properties: - priority: - type: integer - action: - type: string - match: - type: string - nextHopIP: - type: string - type: object - type: array - vpcPeerings: - items: - properties: - remoteVpc: - type: string - localConnectIP: - type: string - type: object - type: array - type: object - status: - properties: - conditions: - items: - properties: - lastTransitionTime: - type: string - lastUpdateTime: - type: string - message: - type: string - reason: - type: string - status: - type: string - type: - type: string - type: object - type: array - default: - type: boolean - defaultLogicalSwitch: - type: string - router: - type: string - standby: - type: boolean - subnets: - items: - type: string - type: array - vpcPeerings: - items: - type: string - type: array - tcpLoadBalancer: - type: string - tcpSessionLoadBalancer: - type: string - udpLoadBalancer: - type: string - udpSessionLoadBalancer: - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} - names: - kind: Vpc - listKind: VpcList - plural: vpcs - shortNames: - - vpc - singular: vpc - scope: Cluster ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: vpc-nat-gateways.kubeovn.io -spec: - group: kubeovn.io - names: - plural: vpc-nat-gateways - singular: vpc-nat-gateway - shortNames: - - vpc-nat-gw - kind: VpcNatGateway - listKind: VpcNatGatewayList - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .spec.vpc - name: Vpc - type: string - - jsonPath: .spec.subnet - name: Subnet - type: string - - jsonPath: .spec.lanIp - name: LanIP - type: string - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - spec: - type: object - properties: - dnatRules: - type: array - items: - type: object - properties: - eip: - type: string - externalPort: - type: string - internalIp: - type: string - internalPort: - type: string - protocol: - type: string - eips: - type: array - items: - type: object - properties: - eipCIDR: - type: string - gateway: - type: string - floatingIpRules: - type: array - items: - type: object - properties: - eip: - type: string - internalIp: - type: string - lanIp: - type: string - snatRules: - type: array - items: - type: object - properties: - eip: - type: string - internalCIDR: - type: string - subnet: - type: string - vpc: - type: string - selector: - type: array - items: - type: string - subresources: - status: {} - conversion: - strategy: None ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition metadata: name: security-groups.kubeovn.io spec: diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 index 36e322623..c6eacc128 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 @@ -56,10 +56,13 @@ spec: - --pod-nic-type={{ kube_ovn_pod_nic_type }} - --enable-lb={{ kube_ovn_enable_lb|string }} - --enable-np={{ kube_ovn_enable_np|string }} + - --enable-eip-snat={{ kube_ovn_eip_snat_enabled }} - --enable-external-vpc={{ kube_ovn_enable_external_vpc|string }} - --logtostderr=false - --alsologtostderr=true - --log_file=/var/log/kube-ovn/kube-ovn-controller.log + - --log_file_max_size=0 + - --keep-vm-ip={{ kube_ovn_keep_vm_ip }} env: - name: ENABLE_SSL value: "{{ kube_ovn_enable_ssl | lower }}" @@ -166,14 +169,17 @@ spec: - --encap-checksum={{ kube_ovn_encap_checksum | lower }} - --service-cluster-ip-range={{ kube_service_addresses }}{% if enable_dual_stack_networks %},{{ kube_service_addresses_ipv6 }}{% endif %}{{''}} - --iface={{ kube_ovn_iface|default('') }} + - --dpdk-tunnel-iface={{ kube_ovn_dpdk_tunnel_iface }} - --network-type={{ kube_ovn_network_type }} - --default-interface-name={{ kube_ovn_default_interface_name|default('') }} {% if kube_ovn_mtu is defined %} - --mtu={{ kube_ovn_mtu }} {% endif %} + - --cni-conf-name={{ kube_ovn_cni_config_priority }}-kube-ovn.conflist - --logtostderr=false - --alsologtostderr=true - --log_file=/var/log/kube-ovn/kube-ovn-cni.log + - --log_file_max_size=0 securityContext: runAsUser: 0 privileged: true @@ -188,13 +194,23 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: MODULES + value: kube_ovn_fastpath.ko + - name: RPMS + value: openvswitch-kmod volumeMounts: + - name: host-modules + mountPath: /lib/modules + readOnly: true + - name: shared-dir + mountPath: /var/lib/kubelet/pods - mountPath: /etc/openvswitch name: systemid - mountPath: /etc/cni/net.d name: cni-conf - mountPath: /run/openvswitch name: host-run-ovs + mountPropagation: Bidirectional - mountPath: /run/ovn name: host-run-ovn - mountPath: /var/run/netns @@ -202,30 +218,30 @@ spec: mountPropagation: HostToContainer - mountPath: /var/log/kube-ovn name: kube-ovn-log + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn - mountPath: /etc/localtime name: localtime - readinessProbe: - exec: - command: - - nc - - -z - - -w3 - - 127.0.0.1 - - "10665" - periodSeconds: 3 - timeoutSeconds: 5 + - mountPath: /tmp + name: tmp livenessProbe: - exec: - command: - - nc - - -z - - -w3 - - 127.0.0.1 - - "10665" + failureThreshold: 3 initialDelaySeconds: 30 periodSeconds: 7 - failureThreshold: 5 - timeoutSeconds: 5 + successThreshold: 1 + tcpSocket: + port: 10665 + timeoutSeconds: 3 + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 7 + successThreshold: 1 + tcpSocket: + port: 10665 + timeoutSeconds: 3 resources: requests: cpu: {{ kube_ovn_cni_server_cpu_request }} @@ -236,6 +252,12 @@ spec: nodeSelector: kubernetes.io/os: "linux" volumes: + - name: host-modules + hostPath: + path: /lib/modules + - name: shared-dir + hostPath: + path: /var/lib/kubelet/pods - name: systemid hostPath: path: /etc/origin/openvswitch @@ -254,13 +276,21 @@ spec: - name: host-ns hostPath: path: /var/run/netns + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch - name: kube-ovn-log hostPath: path: /var/log/kube-ovn + - name: host-log-ovn + hostPath: + path: /var/log/ovn - name: localtime hostPath: path: /etc/localtime - + - name: tmp + hostPath: + path: /tmp --- kind: DaemonSet apiVersion: apps/v1 @@ -297,6 +327,7 @@ spec: - --logtostderr=false - --alsologtostderr=true - --log_file=/var/log/kube-ovn/kube-ovn-pinger.log + - --log_file_max_size=0 securityContext: runAsUser: 0 privileged: false @@ -466,14 +497,14 @@ spec: exec: command: - cat - - /var/run/ovn/ovnnb_db.pid + - /var/run/ovn/ovn-controller.pid periodSeconds: 10 timeoutSeconds: 45 livenessProbe: exec: command: - cat - - /var/run/ovn/ovnnb_db.pid + - /var/run/ovn/ovn-controller.pid initialDelaySeconds: 30 periodSeconds: 10 failureThreshold: 5 diff --git a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 index ec9afc9a2..2d8a5c309 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 @@ -1,10 +1,3 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: ovn-config - namespace: kube-system -data: - defaultNetworkType: '{{ kube_ovn_network_type }}' --- apiVersion: v1 kind: ServiceAccount @@ -33,6 +26,8 @@ rules: - subnets - subnets/status - ips + - vips + - vips/status - vlans - vlans/status - provider-networks @@ -40,6 +35,14 @@ rules: - security-groups - security-groups/status - htbqoses + - iptables-eips + - iptables-fip-rules + - iptables-dnat-rules + - iptables-snat-rules + - iptables-eips/status + - iptables-fip-rules/status + - iptables-dnat-rules/status + - iptables-snat-rules/status verbs: - "*" - apiGroups: @@ -96,6 +99,12 @@ rules: - create - patch - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - "*" - apiGroups: - "k8s.cni.cncf.io" resources: @@ -279,7 +288,7 @@ spec: exec: command: - bash - - /kube-ovn/ovn-is-leader.sh + - /kube-ovn/ovn-healthcheck.sh periodSeconds: 15 timeoutSeconds: 45 livenessProbe: @@ -378,6 +387,9 @@ spec: fieldRef: fieldPath: spec.nodeName volumeMounts: + - mountPath: /var/run/netns + name: host-ns + mountPropagation: HostToContainer - mountPath: /lib/modules name: host-modules readOnly: true @@ -451,6 +463,7 @@ spec: {% endif %} nodeSelector: kubernetes.io/os: "linux" + ovn.kubernetes.io/ovs_dp_type: "kernel" volumes: - name: host-modules hostPath: @@ -464,6 +477,9 @@ spec: - name: host-sys hostPath: path: /sys + - name: host-ns + hostPath: + path: /var/run/netns - name: cni-conf hostPath: path: /etc/cni/net.d