diff --git a/README.md b/README.md index c93d49748..2516db2ee 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ Note: Upstart/SysV init based OS types are not supported. - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) - [cilium](https://github.com/cilium/cilium) v1.8.9 - [flanneld](https://github.com/coreos/flannel) v0.13.0 - - [kube-ovn](https://github.com/alauda/kube-ovn) v1.6.2 + - [kube-ovn](https://github.com/alauda/kube-ovn) v1.7.0 - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.2.2 - [multus](https://github.com/intel/multus-cni) v3.7.0 - [ovn4nfv](https://github.com/opnfv/ovn4nfv-k8s-plugin) v1.1.0 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 5a7490f47..820139110 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -79,7 +79,7 @@ cni_version: "v0.9.1" weave_version: 2.8.1 pod_infra_version: "3.3" cilium_version: "v1.8.9" -kube_ovn_version: "v1.6.2" +kube_ovn_version: "v1.7.0" kube_router_version: "v1.2.2" multus_version: "v3.7" ovn4nfv_ovn_image_version: "v1.0.0" diff --git a/roles/network_plugin/kube-ovn/defaults/main.yml b/roles/network_plugin/kube-ovn/defaults/main.yml index 5bbb84b77..d0eb52794 100644 --- a/roles/network_plugin/kube-ovn/defaults/main.yml +++ b/roles/network_plugin/kube-ovn/defaults/main.yml @@ -15,8 +15,10 @@ kube_ovn_pinger_cpu_request: 100m kube_ovn_pinger_memory_request: 200Mi kube_ovn_pinger_cpu_limit: 200m kube_ovn_pinger_memory_limit: 400Mi -kube_ovn_monitor_cpu_request: 500m -kube_ovn_monitor_memory_request: 300Mi +kube_ovn_monitor_memory_request: 200Mi +kube_ovn_monitor_cpu_request: 200m +kube_ovn_monitor_memory_limit: 200Mi +kube_ovn_monitor_cpu_limit: 200m traffic_mirror: true encap_checksum: false diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 index 396ebc6ea..70427af62 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 @@ -9,9 +9,12 @@ spec: served: true storage: true additionalPrinterColumns: - - name: IP + - name: V4IP type: string - jsonPath: .spec.ipAddress + jsonPath: .spec.v4IpAddress + - name: V6IP + type: string + jsonPath: .spec.v6IpAddress - name: Mac type: string jsonPath: .spec.macAddress @@ -42,6 +45,10 @@ spec: type: string ipAddress: type: string + v4IpAddress: + type: string + v6IpAddress: + type: string attachIps: type: array items: @@ -93,18 +100,33 @@ spec: - name: NAT type: boolean jsonPath: .spec.natOutgoing + - name: ExternalEgressGateway + type: string + jsonPath: .spec.externalEgressGateway + - name: PolicyRoutingPriority + type: integer + jsonPath: .spec.policyRoutingPriority + - name: PolicyRoutingTableID + type: integer + jsonPath: .spec.policyRoutingTableID - name: Default type: boolean jsonPath: .spec.default - name: GatewayType type: string jsonPath: .spec.gatewayType - - name: Used + - name: V4Used type: number - jsonPath: .status.usingIPs - - name: Available + jsonPath: .status.v4usingIPs + - name: V4Available type: number - jsonPath: .status.availableIPs + jsonPath: .status.v4availableIPs + - name: V6Used + type: number + jsonPath: .status.v6usingIPs + - name: V6Available + type: number + jsonPath: .status.v6availableIPs schema: openAPIV3Schema: type: object @@ -112,9 +134,13 @@ spec: status: type: object properties: - availableIPs: + v4availableIPs: type: number - usingIPs: + v4usingIPs: + type: number + v6availableIPs: + type: number + v6usingIPs: type: number activateGateway: type: string @@ -168,6 +194,22 @@ spec: type: string natOutgoing: type: boolean + externalEgressGateway: + type: string + policyRoutingPriority: + type: integer + minimum: 1 + maximum: 32765 + policyRoutingTableID: + type: integer + minimum: 1 + maximum: 2147483647 + not: + enum: + - 252 # compat + - 253 # default + - 254 # main + - 255 # local private: type: boolean vlan: @@ -316,3 +358,81 @@ spec: - vpc singular: vpc scope: Cluster +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vpc-nat-gateways.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vpc-nat-gateways + singular: vpc-nat-gateway + shortNames: + - vpc-nat-gw + kind: VpcNatGateway + listKind: VpcNatGatewayList + scope: Cluster + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + dnatRules: + type: array + items: + type: object + properties: + eip: + type: string + externalPort: + type: string + internalIp: + type: string + internalPort: + type: string + protocol: + type: string + eips: + type: array + items: + type: object + properties: + eipCIDR: + type: string + gateway: + type: string + floatingIpRules: + type: array + items: + type: object + properties: + eip: + type: string + internalIp: + type: string + lanIp: + type: string + snatRules: + type: array + items: + type: object + properties: + eip: + type: string + internalCIDR: + type: string + subnet: + type: string + vpc: + type: string + subresources: + status: {} + conversion: + strategy: None diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 index c0f9ee104..ec05e76c4 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 @@ -44,6 +44,7 @@ spec: - /kube-ovn/start-controller.sh args: - --default-cidr={{ kube_pods_subnet }} + - --pod-nic-type=veth-pair env: - name: ENABLE_SSL value: "{{ enable_ssl | lower }}" @@ -60,6 +61,8 @@ spec: fieldRef: fieldPath: spec.nodeName volumeMounts: + - mountPath: /etc/localtime + name: localtime - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: @@ -88,6 +91,9 @@ spec: nodeSelector: kubernetes.io/os: "linux" volumes: + - name: localtime + hostPath: + path: /etc/localtime - name: kube-ovn-tls secret: optional: true @@ -166,6 +172,8 @@ spec: - mountPath: /var/run/netns name: host-ns mountPropagation: HostToContainer + - mountPath: /etc/localtime + name: localtime readinessProbe: exec: command: @@ -207,6 +215,9 @@ spec: - name: host-ns hostPath: path: /var/run/netns + - name: localtime + hostPath: + path: /etc/localtime --- kind: DaemonSet @@ -280,6 +291,8 @@ spec: name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn + - mountPath: /etc/localtime + name: localtime - mountPath: /var/run/tls name: kube-ovn-tls resources: @@ -313,11 +326,157 @@ spec: - name: host-log-ovn hostPath: path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime - name: kube-ovn-tls secret: optional: true secretName: kube-ovn-tls --- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: kube-ovn-monitor + namespace: kube-system + annotations: + kubernetes.io/description: | + Metrics for OVN components: northd, nb and sb. +spec: + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: kube-ovn-monitor + template: + metadata: + labels: + app: kube-ovn-monitor + component: network + type: infra + spec: + tolerations: + - operator: Exists + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: kube-ovn-monitor + topologyKey: kubernetes.io/hostname + priorityClassName: system-cluster-critical + serviceAccountName: ovn + containers: + - name: kube-ovn-monitor + image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + command: ["/kube-ovn/start-ovn-monitor.sh"] + securityContext: + runAsUser: 0 + privileged: false + env: + - name: ENABLE_SSL + value: "{{ enable_ssl | lower }}" + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: {{ kube_ovn_monitor_cpu_request }} + memory: {{ kube_ovn_monitor_memory_request }} + limits: + cpu: {{ kube_ovn_monitor_cpu_limit }} + memory: {{ kube_ovn_monitor_memory_limit }} + volumeMounts: + - mountPath: /var/run/openvswitch + name: host-run-ovs + - mountPath: /var/run/ovn + name: host-run-ovn + - mountPath: /sys + name: host-sys + readOnly: true + - mountPath: /etc/openvswitch + name: host-config-openvswitch + - mountPath: /etc/ovn + name: host-config-ovn + - mountPath: /var/log/openvswitch + name: host-log-ovs + - mountPath: /var/log/ovn + name: host-log-ovn + - mountPath: /etc/localtime + name: localtime + - mountPath: /var/run/tls + name: kube-ovn-tls + readinessProbe: + exec: + command: + - cat + - /var/run/ovn/ovnnb_db.pid + periodSeconds: 3 + timeoutSeconds: 45 + livenessProbe: + exec: + command: + - cat + - /var/run/ovn/ovn-nbctl.pid + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 5 + timeoutSeconds: 45 + nodeSelector: + kubernetes.io/os: "linux" + kube-ovn/role: "master" + volumes: + - name: host-run-ovs + hostPath: + path: /run/openvswitch + - name: host-run-ovn + hostPath: + path: /run/ovn + - name: host-sys + hostPath: + path: /sys + - name: host-config-openvswitch + hostPath: + path: /etc/origin/openvswitch + - name: host-config-ovn + hostPath: + path: /etc/origin/ovn + - name: host-log-ovs + hostPath: + path: /var/log/openvswitch + - name: host-log-ovn + hostPath: + path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime + - name: kube-ovn-tls + secret: + optional: true + secretName: kube-ovn-tls +--- +kind: Service +apiVersion: v1 +metadata: + name: kube-ovn-monitor + namespace: kube-system + labels: + app: kube-ovn-monitor +spec: + ports: + - name: metrics + port: 10661 + type: ClusterIP + selector: + app: kube-ovn-monitor + sessionAffinity: None +--- kind: Service apiVersion: v1 metadata: diff --git a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 index 0d5b4a3ee..6bf400390 100644 --- a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 +++ b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 @@ -27,19 +27,18 @@ spec: rule: 'RunAsAny' --- + apiVersion: v1 kind: ConfigMap metadata: name: ovn-config namespace: kube-system - --- apiVersion: v1 kind: ServiceAccount metadata: name: ovn namespace: kube-system - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -63,6 +62,7 @@ rules: - subnets/status - vpcs - vpcs/status + - vpc-nat-gateways - ips - vlans - networks @@ -72,6 +72,7 @@ rules: - "" resources: - pods + - pods/exec - namespaces - nodes - configmaps @@ -82,6 +83,16 @@ rules: - watch - patch - update + - apiGroups: + - "k8s.cni.cncf.io" + resources: + - network-attachment-definitions + verbs: + - create + - delete + - get + - list + - update - apiGroups: - "" - networking.k8s.io @@ -95,6 +106,10 @@ rules: - daemonsets - deployments verbs: + - create + - delete + - update + - patch - get - list - watch @@ -106,7 +121,6 @@ rules: - create - patch - update - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -172,22 +186,6 @@ spec: ovn-northd-leader: "true" sessionAffinity: None --- -kind: Service -apiVersion: v1 -metadata: - name: kube-ovn-monitor - namespace: kube-system - labels: - app: kube-ovn-monitor -spec: - ports: - - name: metrics - port: 10661 - type: ClusterIP - selector: - app: ovn-central - sessionAffinity: None ---- kind: Deployment apiVersion: apps/v1 metadata: @@ -214,7 +212,7 @@ spec: type: infra spec: tolerations: - - operator: Exists + - operator: Exists affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -225,7 +223,6 @@ spec: priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true - shareProcessNamespace: true containers: - name: ovn-central image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} @@ -272,6 +269,8 @@ spec: name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn + - mountPath: /etc/localtime + name: localtime - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: @@ -284,69 +283,12 @@ spec: livenessProbe: exec: command: - - bash - - /kube-ovn/ovn-healthcheck.sh + - bash + - /kube-ovn/ovn-healthcheck.sh initialDelaySeconds: 30 periodSeconds: 7 failureThreshold: 5 timeoutSeconds: 45 - - name: ovn-monitor - image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }} - imagePullPolicy: {{ k8s_image_pull_policy }} - command: ["/kube-ovn/start-ovn-monitor.sh"] - env: - - name: ENABLE_SSL - value: "{{ enable_ssl | lower }}" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - resources: - requests: - cpu: {{ kube_ovn_monitor_cpu_request }} - memory: {{ kube_ovn_monitor_memory_request }} - volumeMounts: - - mountPath: /var/run/openvswitch - name: host-run-ovs - - mountPath: /var/run/ovn - name: host-run-ovn - - mountPath: /sys - name: host-sys - readOnly: true - - mountPath: /etc/openvswitch - name: host-config-openvswitch - - mountPath: /etc/ovn - name: host-config-ovn - - mountPath: /var/log/openvswitch - name: host-log-ovs - - mountPath: /var/log/ovn - name: host-log-ovn - - mountPath: /var/run/tls - name: kube-ovn-tls - readinessProbe: - exec: - command: - - cat - - /var/run/ovn/ovnnb_db.pid - periodSeconds: 3 - timeoutSeconds: 45 - livenessProbe: - exec: - command: - - cat - - /var/run/ovn/ovn-nbctl.pid - initialDelaySeconds: 30 - periodSeconds: 10 - failureThreshold: 5 - timeoutSeconds: 45 nodeSelector: kubernetes.io/os: "linux" kube-ovn/role: "master" @@ -372,6 +314,9 @@ spec: - name: host-log-ovn hostPath: path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime - name: kube-ovn-tls secret: optional: true @@ -399,7 +344,7 @@ spec: type: infra spec: tolerations: - - operator: Exists + - operator: Exists priorityClassName: system-cluster-critical serviceAccountName: ovn hostNetwork: true @@ -444,20 +389,22 @@ spec: name: host-log-ovs - mountPath: /var/log/ovn name: host-log-ovn + - mountPath: /etc/localtime + name: localtime - mountPath: /var/run/tls name: kube-ovn-tls readinessProbe: exec: command: - - bash - - /kube-ovn/ovs-healthcheck.sh + - bash + - /kube-ovn/ovs-healthcheck.sh periodSeconds: 5 timeoutSeconds: 45 livenessProbe: exec: command: - - bash - - /kube-ovn/ovs-healthcheck.sh + - bash + - /kube-ovn/ovs-healthcheck.sh initialDelaySeconds: 10 periodSeconds: 5 failureThreshold: 5 @@ -496,6 +443,9 @@ spec: - name: host-log-ovn hostPath: path: /var/log/ovn + - name: localtime + hostPath: + path: /etc/localtime - name: kube-ovn-tls secret: optional: true