Add calico 3.7.3 support (#4953)

* Add calico 3.7.3 support * add calico_datastore variable to policy controller role * add missing clusterrole rules for calico policy controller * disable calico kube controller when kdd mode is used for versions < 3.6
2019-07-09 21:42:28 +02:00 · 2019-07-09 21:42:28 +02:00 · 20c7e31ea3
commit 20c7e31ea3
parent 65065e7fdf
10 changed files with 235 additions and 19 deletions
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@ -58,12 +58,12 @@ kube_image_repo: "gcr.io/google-containers"

 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
-calico_version: "v3.4.0"
-calico_ctl_version: "v3.4.4"
-calico_cni_version: "v3.4.0"
-calico_policy_version: "v3.4.0"
+calico_version: "v3.7.3"
+calico_ctl_version: "v3.7.3"
+calico_cni_version: "v3.7.3"
+calico_policy_version: "v3.7.3"
 calico_rr_version: "v0.6.1"
-calico_typha_version: "v3.4.4"
+calico_typha_version: "v3.7.3"

 flannel_version: "v0.11.0"
 flannel_cni_version: "v0.3.0"
@ -156,14 +156,17 @@ calicoctl_binary_checksums:
    v3.6.1: 0
    v3.5.4: 0
    v3.4.4: 0
+    v3.7.3: 0
  amd64:
    v3.6.1: 3b01336de37550e020343d62a38c96c4605d33a3ed7ddba2fe38bc172a5b42b5
    v3.5.4: 197194b838cc2a9a7455c2ebd5505a5e24f8f3d994eb75c17f5dd568944100b8
    v3.4.4: 93bd084e053cf1bf3b7fef369677bd6767c30fe7135e2c7e044e31693422ef61
+    v3.7.3: 932f68e893e80e95e10f064f1e7745e438d456f41a6ff12d11bb16ca0cab735c
  arm64:
    v3.6.1: 60fbaeb257061647bdf12b5ede7a0d4298a5ee216f6472e5a92bb14ef5c2a5d3
    v3.5.4: a4481178665658658a73e4ceca9a1dff5cccded4179615c91d1c3e49fd96f237
    v3.4.4: ff35d9e8b5c00e9fe47d05e8f5123ec98fd641370f8cd93f4fbb3d913da77ab6
+    v3.7.3: 7cfaab25c287f7ef93b2682d060b55bf39f76b668540de50376b5ed174209832

 etcd_binary_checksum: "{{ etcd_binary_checksums[image_arch] }}"
 cni_binary_checksum: "{{ cni_binary_checksums[image_arch] }}"
--- a/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml
+++ b/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml
@ -8,3 +8,6 @@ calico_policy_controller_memory_requests: 64M
 # SSL
 calico_cert_dir: "/etc/calico/certs"
 canal_cert_dir: "/etc/canal/certs"
+
+# Datastore type
+calico_datastore: "etcd"
--- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
+++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
@ -46,7 +46,20 @@ spec:
            requests:
              cpu: {{ calico_policy_controller_cpu_requests }}
              memory: {{ calico_policy_controller_memory_requests }}
+{% if calico_version is version('v3.3.0', '>=') %}
+          readinessProbe:
+            exec:
+              command:
+              - /usr/bin/check-status
+              - -r
+{% endif %}
          env:
+{% if calico_datastore == "kdd" and calico_version is version('v3.6.0', '>=') %}
+            - name: ENABLED_CONTROLLERS
+              value: node
+            - name: DATASTORE_TYPE
+              value: kubernetes
+{% else %}
            - name: ETCD_ENDPOINTS
              value: "{{ etcd_access_addresses }}"
            - name: ETCD_CA_CERT_FILE
@ -55,13 +68,6 @@ spec:
              value: "{{ calico_cert_dir }}/cert.crt"
            - name: ETCD_KEY_FILE
              value: "{{ calico_cert_dir }}/key.pem"
-{% if calico_version is version('v3.3.0', '>=') %}
-          readinessProbe:
-            exec:
-              command:
-              - /usr/bin/check-status
-              - -r
-{% endif %}
          volumeMounts:
          - mountPath: {{ calico_cert_dir }}
            name: etcd-certs
@ -70,3 +76,4 @@ spec:
      - hostPath:
          path: {{ calico_cert_dir }}
        name: etcd-certs
+{% endif %}
--- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2
+++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2
@ -5,6 +5,7 @@ metadata:
  name: calico-kube-controllers
  namespace: kube-system
 rules:
+{% if calico_datastore == "etcd"  %}
  - apiGroups:
    - ""
    - extensions
@ -24,3 +25,44 @@ rules:
    verbs:
      - watch
      - list
+{% elif calico_datastore == "kdd" %}
+  # Nodes are watched to monitor for deletions.
+  - apiGroups: [""]
+    resources:
+      - nodes
+    verbs:
+      - watch
+      - list
+      - get
+  # Pods are queried to check for existence.
+  - apiGroups: [""]
+    resources:
+      - pods
+    verbs:
+      - get
+  # IPAM resources are manipulated when nodes are deleted.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - ippools
+    verbs:
+      - list
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - blockaffinities
+      - ipamblocks
+      - ipamhandles
+    verbs:
+      - get
+      - list
+      - create
+      - update
+      - delete
+  # Needs access to update clusterinformations.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - clusterinformations
+    verbs:
+      - get
+      - create
+      - update
+{% endif %}
--- a/roles/kubernetes-apps/policy_controller/meta/main.yml
+++ b/roles/kubernetes-apps/policy_controller/meta/main.yml
@ -4,11 +4,13 @@ dependencies:
    when:
      - kube_network_plugin == 'calico'
      - enable_network_policy
+      - calico_datastore != "kdd" or calico_policy_version is version('v3.6.0', '>=')
    tags:
      - policy-controller

  - role: policy_controller/calico
    when:
      - kube_network_plugin == 'canal'
+      - calico_datastore != "kdd" or calico_policy_version is version('v3.6.0', '>=')
    tags:
      - policy-controller
--- a/roles/network_plugin/calico/templates/calico-cr.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-cr.yml.j2
@ -71,9 +71,15 @@ rules:
      - globalbgpconfigs
      - bgpconfigurations
      - ippools
+{% if calico_version is version('v3.6.0', '>=') %}
+      - ipamblocks
+{% endif %}
      - globalnetworkpolicies
      - globalnetworksets
      - networkpolicies
+{% if calico_version is version('v3.7.0', '>=') %}
+      - networksets
+{% endif %}
      - clusterinformations
      - hostendpoints
    verbs:
@ -106,4 +112,36 @@ rules:
    verbs:
      - create
      - update
+{% if calico_version is version('v3.6.0', '>=') %}
+  # These permissions are required for Calico CNI to perform IPAM allocations.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - blockaffinities
+      - ipamblocks
+      - ipamhandles
+    verbs:
+      - get
+      - list
+      - create
+      - update
+      - delete
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - ipamconfigs
+    verbs:
+      - get
+  # Block affinities must also be watchable by confd for route aggregation.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - blockaffinities
+    verbs:
+      - watch
+  # The Calico IPAM migration needs to get daemonsets. These permissions can be
+  # removed if not upgrading from an installation using host-local IPAM.
+  - apiGroups: ["apps"]
+    resources:
+      - daemonsets
+    verbs:
+      - get
+{% endif %}
 {% endif %}
--- a/roles/network_plugin/calico/templates/calico-node.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-node.yml.j2
@ -44,6 +44,29 @@ spec:
      terminationGracePeriodSeconds: 0
 {% if calico_version is version('v3.4.0', '>=') %}
      initContainers:
+{% if calico_datastore == "kdd" and calico_version is version('v3.6.0', '>=') %}
+        # This container performs upgrade from host-local IPAM to calico-ipam.
+        # It can be deleted if this is a fresh installation, or if you have already
+        # upgraded to use calico-ipam.
+        - name: upgrade-ipam
+          image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
+          env:
+            - name: KUBERNETES_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CALICO_NETWORKING_BACKEND
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: calico_backend
+          volumeMounts:
+            - mountPath: /var/lib/cni/networks
+              name: host-local-net-dir
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+{% endif %}
        # This container installs the Calico CNI binaries
        # and CNI network config file on each node.
        - name: install-cni
@ -304,6 +327,14 @@ spec:
          hostPath:
            path: /run/xtables.lock
            type: FileOrCreate
+{% if calico_datastore == "kdd" and calico_version is version('v3.6.0', '>=') %}
+        # Mount in the directory for host-local IPAM allocations. This is
+        # used when upgrading from host-local to calico-ipam, and can be removed
+        # if not using the upgrade-ipam init container.
+        - name: host-local-net-dir
+          hostPath:
+            path: /var/lib/cni/networks
+{% endif %}
  updateStrategy:
    rollingUpdate:
      maxUnavailable: {{ serial | default('20%') }}
--- a/roles/network_plugin/calico/templates/calico-typha.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-typha.yml.j2
@ -90,6 +90,12 @@ spec:
          #  value: "true"
          #- name: TYPHA_PROMETHEUSMETRICSPORT
          #  value: "9093"
+
+          # Needed for version >=3.7 when the 'host-local' ipam is used
+          # Should never happen given templates/cni-calico.conflist.j2
+          # Configure route aggregation based on pod CIDR.
+          # - name: USE_POD_CIDR
+          #   value: "true"
        livenessProbe:
          exec:
            command:
--- a/roles/network_plugin/calico/templates/cni-calico.conflist.j2
+++ b/roles/network_plugin/calico/templates/cni-calico.conflist.j2
@ -3,10 +3,15 @@
  "cniVersion":"0.3.1",
  "plugins":[
    {
+{% if calico_datastore == "kdd" %}
+      "datastore_type": "kubernetes",
+      "nodename": "__KUBERNETES_NODE_NAME__",
+{% else %}
 {% if cloud_provider is defined %}
      "nodename": "{{ calico_kubelet_name.stdout }}",
    {% else %}
      "nodename": "{{ calico_baremetal_nodename }}",
+    {% endif %}
 {% endif %}
      "type": "calico",
      "log_level": "info",
@ -15,18 +20,18 @@
      "etcd_cert_file": "{{ calico_cert_dir }}/cert.crt",
      "etcd_key_file": "{{ calico_cert_dir }}/key.pem",
      "etcd_ca_cert_file": "{{ calico_cert_dir }}/ca_cert.crt",
+{% endif %}
+{% if calico_datastore == "kdd" and calico_version is version('v3.6.0', '<') %}
+      "ipam": {
+        "type": "host-local",
+        "subnet": "usePodCidr"
+      },
+{% else %}
      "ipam": {
        "type": "calico-ipam",
        "assign_ipv4": "true",
        "ipv4_pools": ["{{ calico_pool_cidr | default(kube_pods_subnet) }}"]
      },
-{% elif calico_datastore == "kdd" %}
-      "datastore_type": "kubernetes",
-      "nodename": "__KUBERNETES_NODE_NAME__",
-      "ipam": {
-        "type": "host-local",
-        "subnet": "usePodCidr"
-      },
 {% endif %}
 {% if (calico_feature_control is defined) and (calico_feature_control|length > 0) %}
      "feature_control": {
--- a/roles/network_plugin/calico/templates/kdd-crds.yml.j2
+++ b/roles/network_plugin/calico/templates/kdd-crds.yml.j2
@ -14,6 +14,68 @@ spec:
    singular: felixconfiguration
 ---

+{% if calico_version is version('v3.6.0', '>=') %}
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: ipamblocks.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: IPAMBlock
+    plural: ipamblocks
+    singular: ipamblock
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: blockaffinities.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: BlockAffinity
+    plural: blockaffinities
+    singular: blockaffinity
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: ipamhandles.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: IPAMHandle
+    plural: ipamhandles
+    singular: ipamhandle
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: ipamconfigs.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: IPAMConfig
+    plural: ipamconfigs
+    singular: ipamconfig
+
+---
+
+{% endif %}
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
@ -131,3 +193,20 @@ spec:
    kind: NetworkPolicy
    plural: networkpolicies
    singular: networkpolicy
+
+{% if calico_version is version('v3.7.0', '>=') %}
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: networksets.crd.projectcalico.org
+spec:
+  scope: Namespaced
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: NetworkSet
+    plural: networksets
+    singular: networkset
+{% endif %}