Upgrade kube-ovn to v1.1.0 and move test from centos7 to centos8 (#5852)

2020-04-15 12:10:03 +02:00 · 2020-04-15 12:10:03 +02:00 · c929b5e82e
commit c929b5e82e
parent 58f48500b1
8 changed files with 271 additions and 84 deletions
--- a/.gitlab-ci/packet.yml
+++ b/.gitlab-ci/packet.yml
@ -108,7 +108,7 @@ packet_centos7-calico-ha-once-localhost:
  services:
    - docker:18.09.9-dind

-packet_centos7-kube-ovn:
+packet_centos8-kube-ovn:
  stage: deploy-part2
  extends: .packet
  when: on_success
--- a/docs/kube-ovn.md
+++ b/docs/kube-ovn.md
@ -4,6 +4,12 @@ Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It off

 For more information please check [Kube-OVN documentation](https://github.com/alauda/kube-ovn)

+**Warning:** Kernel version (`cat /proc/version`) needs to be different than `3.10.0-862` or kube-ovn won't start and will print this message:
+
+```bash
+kernel version 3.10.0-862 has a nat related bug that will affect ovs function, please update to a version greater than 3.10.0-898
+```
+
 ## How to use it

 Enable kube-ovn in `group_vars/k8s-cluster/k8s-cluster.yml`
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@ -63,6 +63,9 @@ docker_image_repo: "docker.io"
 # quay image repo define
 quay_image_repo: "quay.io"

+# alauda.cn image repo (for kube-ovn...)
+alauda_image_repo: "index.alauda.cn"
+
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
 calico_version: "v3.13.2"
@ -81,7 +84,7 @@ weave_version: 2.6.2
 pod_infra_version: 3.1
 contiv_version: 1.2.1
 cilium_version: "v1.7.2"
-kube_ovn_version: "v0.6.0"
+kube_ovn_version: "v1.1.0"
 kube_router_version: "v0.4.0"
 multus_version: "v3.4.1"

@ -472,14 +475,8 @@ cilium_init_image_repo: "{{ docker_image_repo }}/cilium/cilium-init"
 cilium_init_image_tag: "2019-04-05"
 cilium_operator_image_repo: "{{ docker_image_repo }}/cilium/operator"
 cilium_operator_image_tag: "{{ cilium_version }}"
-kube_ovn_db_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-db"
-kube_ovn_node_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-node"
-kube_ovn_cni_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-cni"
-kube_ovn_controller_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-controller"
-kube_ovn_db_image_tag: "{{ kube_ovn_version }}"
-kube_ovn_node_image_tag: "{{ kube_ovn_version }}"
-kube_ovn_controller_image_tag: "{{ kube_ovn_version }}"
-kube_ovn_cni_image_tag: "{{ kube_ovn_version }}"
+kube_ovn_container_image_repo: "{{ alauda_image_repo }}/alaudak8s/kube-ovn"
+kube_ovn_container_image_tag: "{{ kube_ovn_version }}"
 kube_router_image_repo: "{{ docker_image_repo }}/cloudnativelabs/kube-router"
 kube_router_image_tag: "{{ kube_router_version }}"
 multus_image_repo: "{{ docker_image_repo }}/nfvpe/multus"
@ -836,38 +833,11 @@ downloads:
    groups:
      - k8s-cluster

-  kube_ovn_db:
+  kube_ovn:
    enabled: "{{ kube_network_plugin == 'kube-ovn' }}"
    container: true
-    repo: "{{ kube_ovn_db_image_repo }}"
-    tag: "{{ kube_ovn_db_image_tag }}"
-    sha256: "{{ kube_ovn_digest_checksum|default(None) }}"
-    groups:
-      - k8s-cluster
-
-  kube_ovn_node:
-    enabled: "{{ kube_network_plugin == 'kube-ovn' }}"
-    container: true
-    repo: "{{ kube_ovn_node_image_repo }}"
-    tag: "{{ kube_ovn_node_image_tag }}"
-    sha256: "{{ kube_ovn_digest_checksum|default(None) }}"
-    groups:
-      - k8s-cluster
-
-  kube_ovn_controller:
-    enabled: "{{ kube_network_plugin == 'kube-ovn' }}"
-    container: true
-    repo: "{{ kube_ovn_controller_image_repo }}"
-    tag: "{{ kube_ovn_controller_image_tag }}"
-    sha256: "{{ kube_ovn_digest_checksum|default(None) }}"
-    groups:
-      - k8s-cluster
-
-  kube_ovn_cni:
-    enabled: "{{ kube_network_plugin == 'kube-ovn' }}"
-    container: true
-    repo: "{{ kube_ovn_cni_image_repo }}"
-    tag: "{{ kube_ovn_cni_image_tag }}"
+    repo: "{{ kube_ovn_container_image_repo }}"
+    tag: "{{ kube_ovn_container_image_tag }}"
    sha256: "{{ kube_ovn_digest_checksum|default(None) }}"
    groups:
      - k8s-cluster
--- a/roles/network_plugin/kube-ovn/defaults/main.yml
+++ b/roles/network_plugin/kube-ovn/defaults/main.yml
@ -7,5 +7,10 @@ kube_ovn_node_cpu_request: 100m
 kube_ovn_node_memory_request: 300Mi
 kube_ovn_node_cpu_limit: 200m
 kube_ovn_node_memory_limit: 500Mi
+kube_ovn_pinger_cpu_request: 100m
+kube_ovn_pinger_memory_request: 300Mi
+kube_ovn_pinger_cpu_limit: 200m
+kube_ovn_pinger_memory_limit: 400Mi

-traffic_mirror: true
+traffic_mirror: true
+encap_checksum: true
--- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2
+++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2
@ -40,7 +40,12 @@ spec:
    kind: Subnet
    shortNames:
      - subnet
+  subresources:
+    status: {}
  additionalPrinterColumns:
+    - name: Provider
+      type: string
+      JSONPath: .spec.provider
    - name: Protocol
      type: string
      JSONPath: .spec.protocol
@ -53,11 +58,23 @@ spec:
    - name: NAT
      type: boolean
      JSONPath: .spec.natOutgoing
+    - name: Default
+      type: boolean
+      JSONPath: .spec.default
+    - name: GatewayType
+      type: string
+      JSONPath: .spec.gatewayType
+    - name: Used
+      type: integer
+      JSONPath: .status.usingIPs
+    - name: Available
+      type: integer
+      JSONPath: .status.availableIPs
  validation:
    openAPIV3Schema:
      properties:
        spec:
-          required: ["cidrBlock","gateway"]
+          required: ["cidrBlock"]
          properties:
            cidrBlock:
              type: "string"
--- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2
+++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2
@ -34,11 +34,12 @@ spec:
                matchLabels:
                  app: kube-ovn-controller
              topologyKey: kubernetes.io/hostname
+      priorityClassName: system-cluster-critical
      serviceAccountName: ovn
      hostNetwork: true
      containers:
        - name: kube-ovn-controller
-          image: {{ kube_ovn_controller_image_repo }}:{{ kube_ovn_controller_image_tag }}
+          image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
          imagePullPolicy: {{ k8s_image_pull_policy }}
          command:
          - /kube-ovn/start-controller.sh
@ -60,25 +61,19 @@ spec:
          readinessProbe:
            exec:
              command:
-                - nc
-                - -z
-                - -w3
-                - 127.0.0.1
-                - "10660"
+                - sh
+                - /kube-ovn/kube-ovn-controller-healthcheck.sh
            periodSeconds: 3
          livenessProbe:
            exec:
              command:
-                - nc
-                - -z
-                - -w3
-                - 127.0.0.1
-                - "10660"
+                - sh
+                - /kube-ovn/kube-ovn-controller-healthcheck.sh
            initialDelaySeconds: 30
            periodSeconds: 7
            failureThreshold: 5
      nodeSelector:
-        beta.kubernetes.io/os: "linux"
+        kubernetes.io/os: "linux"

 ---
 kind: DaemonSet
@ -94,7 +89,7 @@ spec:
    matchLabels:
      app: kube-ovn-cni
  updateStrategy:
-    type: RollingUpdate
+    type: OnDelete
  template:
    metadata:
      labels:
@ -105,14 +100,18 @@ spec:
      tolerations:
      - operator: Exists
        effect: NoSchedule
+      priorityClassName: system-cluster-critical
      serviceAccountName: ovn
      hostNetwork: true
      hostPID: true
      initContainers:
      - name: install-cni
-        image: {{ kube_ovn_cni_image_repo }}:{{ kube_ovn_cni_image_tag }}
+        image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
        imagePullPolicy: {{ k8s_image_pull_policy }}
        command: ["/kube-ovn/install-cni.sh"]
+        securityContext:
+          runAsUser: 0
+          privileged: true
        volumeMounts:
          - mountPath: /etc/cni/net.d
            name: cni-conf
@ -120,16 +119,18 @@ spec:
            name: cni-bin
      containers:
      - name: cni-server
-        image: {{ kube_ovn_cni_image_repo }}:{{ kube_ovn_cni_image_tag }}
+        image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
        imagePullPolicy: {{ k8s_image_pull_policy }}
        command:
          - sh
          - /kube-ovn/start-cniserver.sh
        args:
          - --enable-mirror={{ traffic_mirror }}
+          - --encap-checksum={{ encap_checksum }}
+          - --service-cluster-ip-range={{ kube_service_addresses }}
        securityContext:
-          runAsUser: 0
-          privileged: true
+          capabilities:
+            add: ["NET_ADMIN", "SYS_ADMIN", "SYS_PTRACE"]
        env:
          - name: POD_IP
            valueFrom:
@ -142,6 +143,11 @@ spec:
        volumeMounts:
          - mountPath: /run/openvswitch
            name: host-run-ovs
+          - mountPath: /run/ovn
+            name: host-run-ovn
+          - mountPath: /var/run/netns
+            name: host-ns
+            mountPropagation: HostToContainer
        readinessProbe:
          exec:
            command:
@ -163,14 +169,165 @@ spec:
          periodSeconds: 7
          failureThreshold: 5
      nodeSelector:
-        beta.kubernetes.io/os: "linux"
+        kubernetes.io/os: "linux"
      volumes:
        - name: host-run-ovs
          hostPath:
            path: /run/openvswitch
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
        - name: cni-conf
          hostPath:
            path: /etc/cni/net.d
        - name: cni-bin
          hostPath:
-            path: /opt/cni/bin
+            path: /opt/cni/bin
+        - name: host-ns
+          hostPath:
+            path: /var/run/netns
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: kube-ovn-pinger
+  namespace: kube-ovn
+  annotations:
+    kubernetes.io/description: |
+      This daemon set launches the openvswitch daemon.
+spec:
+  selector:
+    matchLabels:
+      app: kube-ovn-pinger
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: kube-ovn-pinger
+        component: network
+        type: infra
+    spec:
+      tolerations:
+        - operator: Exists
+          effect: NoSchedule
+      serviceAccountName: ovn
+      hostPID: true
+      containers:
+        - name: pinger
+          image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
+          command: ["/kube-ovn/kube-ovn-pinger", "--external-address=114.114.114.114"]
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          securityContext:
+            runAsUser: 0
+            privileged: false
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: HOST_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /lib/modules
+              name: host-modules
+              readOnly: true
+            - mountPath: /run/openvswitch
+              name: host-run-ovs
+            - mountPath: /var/run/openvswitch
+              name: host-run-ovs
+            - mountPath: /var/run/ovn
+              name: host-run-ovn
+            - mountPath: /sys
+              name: host-sys
+              readOnly: true
+            - mountPath: /etc/openvswitch
+              name: host-config-openvswitch
+            - mountPath: /var/log/openvswitch
+              name: host-log-ovs
+            - mountPath: /var/log/ovn
+              name: host-log-ovn
+          resources:
+            requests:
+              cpu: {{ kube_ovn_pinger_cpu_request }}
+              memory: {{ kube_ovn_pinger_memory_request }}
+            limits:
+              cpu: {{ kube_ovn_pinger_cpu_limit }}
+              memory: {{ kube_ovn_pinger_memory_limit }}
+      nodeSelector:
+        kubernetes.io/os: "linux"
+      volumes:
+        - name: host-modules
+          hostPath:
+            path: /lib/modules
+        - name: host-run-ovs
+          hostPath:
+            path: /run/openvswitch
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: host-config-openvswitch
+          hostPath:
+            path: /etc/origin/openvswitch
+        - name: host-log-ovs
+          hostPath:
+            path: /var/log/openvswitch
+        - name: host-log-ovn
+          hostPath:
+            path: /var/log/ovn
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-ovn-pinger
+  namespace: kube-ovn
+  labels:
+    app: kube-ovn-pinger
+spec:
+  selector:
+    app: kube-ovn-pinger
+  ports:
+    - port: 8080
+      name: metrics
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-ovn-controller
+  namespace: kube-ovn
+  labels:
+    app: kube-ovn-controller
+spec:
+  selector:
+    app: kube-ovn-controller
+  ports:
+    - port: 10660
+      name: metrics
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: kube-ovn-cni
+  namespace: kube-ovn
+  labels:
+    app: kube-ovn-cni
+spec:
+  selector:
+    app: kube-ovn-cni
+  ports:
+    - port: 10665
+      name: metrics
--- a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2
+++ b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2
@ -29,6 +29,7 @@ rules:
      - "kubeovn.io"
    resources:
      - subnets
+      - subnets/status
      - ips
    verbs:
      - "*"
@ -55,6 +56,7 @@ rules:
      - services
      - endpoints
      - statefulsets
+      - daemonsets
    verbs:
      - get
      - list
@ -97,6 +99,7 @@ spec:
  type: ClusterIP
  selector:
    app: ovn-central
+    ovn-nb-leader: "true"
  sessionAffinity: None

 ---
@ -114,6 +117,7 @@ spec:
  type: ClusterIP
  selector:
    app: ovn-central
+    ovn-sb-leader: "true"
  sessionAffinity: None

 ---
@ -152,17 +156,30 @@ spec:
                matchLabels:
                  app: ovn-central
              topologyKey: kubernetes.io/hostname
+      priorityClassName: system-cluster-critical
      serviceAccountName: ovn
      hostNetwork: true
      containers:
        - name: ovn-central
-          image: {{ kube_ovn_db_image_repo }}:{{ kube_ovn_db_image_tag }}
+          image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
          imagePullPolicy: {{ k8s_image_pull_policy }}
+          command: ["/kube-ovn/start-db.sh"]
+          securityContext:
+            capabilities:
+              add: ["SYS_NICE"]
          env:
            - name: POD_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.podIP
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
          resources:
            requests:
              cpu: {{ kube_ovn_db_cpu_request }}
@ -171,47 +188,55 @@ spec:
              cpu: {{ kube_ovn_db_cpu_limit }}
              memory: {{ kube_ovn_db_memory_limit }}
          volumeMounts:
-            - mountPath: /run/openvswitch
-              name: host-run-ovs
            - mountPath: /var/run/openvswitch
              name: host-run-ovs
+            - mountPath: /var/run/ovn
+              name: host-run-ovn
            - mountPath: /sys
              name: host-sys
              readOnly: true
            - mountPath: /etc/openvswitch
              name: host-config-openvswitch
            - mountPath: /var/log/openvswitch
-              name: host-log
+              name: host-log-ovs
+            - mountPath: /var/log/ovn
+              name: host-log-ovn
          readinessProbe:
            exec:
              command:
                - sh
-                - /root/ovn-is-leader.sh
+                - /kube-ovn/ovn-is-leader.sh
            periodSeconds: 3
          livenessProbe:
            exec:
              command:
              - sh
-              - /root/ovn-healthcheck.sh
+              - /kube-ovn/ovn-healthcheck.sh
            initialDelaySeconds: 30
            periodSeconds: 7
            failureThreshold: 5
      nodeSelector:
-        beta.kubernetes.io/os: "linux"
+        kubernetes.io/os: "linux"
        kube-ovn/role: "master"
      volumes:
        - name: host-run-ovs
          hostPath:
            path: /run/openvswitch
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
        - name: host-sys
          hostPath:
            path: /sys
        - name: host-config-openvswitch
          hostPath:
            path: /etc/origin/openvswitch
-        - name: host-log
+        - name: host-log-ovs
          hostPath:
            path: /var/log/openvswitch
+        - name: host-log-ovn
+          hostPath:
+            path: /var/log/ovn

 ---
 kind: DaemonSet
@ -227,7 +252,7 @@ spec:
    matchLabels:
      app: ovs
  updateStrategy:
-    type: RollingUpdate
+    type: OnDelete
  template:
    metadata:
      labels:
@ -238,13 +263,15 @@ spec:
      tolerations:
      - operator: Exists
        effect: NoSchedule
+      priorityClassName: system-cluster-critical
      serviceAccountName: ovn
      hostNetwork: true
      hostPID: true
      containers:
        - name: openvswitch
-          image: {{ kube_ovn_node_image_repo }}:{{ kube_ovn_node_image_tag }}
+          image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
          imagePullPolicy: {{ k8s_image_pull_policy }}
+          command: ["/kube-ovn/start-ovs.sh"]
          securityContext:
            runAsUser: 0
            privileged: true
@ -257,28 +284,30 @@ spec:
            - mountPath: /lib/modules
              name: host-modules
              readOnly: true
-            - mountPath: /run/openvswitch
-              name: host-run-ovs
            - mountPath: /var/run/openvswitch
              name: host-run-ovs
+            - mountPath: /var/run/ovn
+              name: host-run-ovn
            - mountPath: /sys
              name: host-sys
              readOnly: true
            - mountPath: /etc/openvswitch
              name: host-config-openvswitch
            - mountPath: /var/log/openvswitch
-              name: host-log
+              name: host-log-ovs
+            - mountPath: /var/log/ovn
+              name: host-log-ovn
          readinessProbe:
            exec:
              command:
              - sh
-              - /root/ovs-healthcheck.sh
+              - /kube-ovn/ovs-healthcheck.sh
            periodSeconds: 5
          livenessProbe:
            exec:
              command:
              - sh
-              - /root/ovs-healthcheck.sh
+              - /kube-ovn/ovs-healthcheck.sh
            initialDelaySeconds: 10
            periodSeconds: 5
            failureThreshold: 5
@ -290,7 +319,7 @@ spec:
              cpu: {{ kube_ovn_node_cpu_limit }}
              memory: {{ kube_ovn_node_memory_limit }}
      nodeSelector:
-        beta.kubernetes.io/os: "linux"
+        kubernetes.io/os: "linux"
      volumes:
        - name: host-modules
          hostPath:
@ -298,12 +327,18 @@ spec:
        - name: host-run-ovs
          hostPath:
            path: /run/openvswitch
+        - name: host-run-ovn
+          hostPath:
+            path: /run/ovn
        - name: host-sys
          hostPath:
            path: /sys
        - name: host-config-openvswitch
          hostPath:
            path: /etc/origin/openvswitch
-        - name: host-log
+        - name: host-log-ovs
          hostPath:
-            path: /var/log/openvswitch
+            path: /var/log/openvswitch
+        - name: host-log-ovn
+          hostPath:
+            path: /var/log/ovn
--- a/tests/files/packet_centos8-kube-ovn.yml
+++ b/tests/files/packet_centos8-kube-ovn.yml
@ -1,12 +1,9 @@
 ---
 # Instance settings
-cloud_image: centos-7
+cloud_image: centos-8
 mode: default

 # Kubespray settings
 kube_network_plugin: kube-ovn
 deploy_netchecker: true
-dns_min_replicas: 1
-
-# Temp set k8s ver to 1.16.8
-kube_version: v1.16.7
+dns_min_replicas: 1