Add calico 3.7.3 support (#4953)

* Add calico 3.7.3 support

* add calico_datastore variable to policy controller role

* add missing clusterrole rules for calico policy controller

* disable calico kube controller when kdd mode is used for versions < 3.6

roles/download/defaults/main.yml

@@ -58,12 +58,12 @@ kube_image_repo: "gcr.io/google-containers"
 
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
-calico_version: "v3.4.0"
+calico_version: "v3.7.3"
-calico_ctl_version: "v3.4.4"
+calico_ctl_version: "v3.7.3"
-calico_cni_version: "v3.4.0"
+calico_cni_version: "v3.7.3"
-calico_policy_version: "v3.4.0"
+calico_policy_version: "v3.7.3"
 calico_rr_version: "v0.6.1"
-calico_typha_version: "v3.4.4"
+calico_typha_version: "v3.7.3"
 
 flannel_version: "v0.11.0"
 flannel_cni_version: "v0.3.0"
@@ -156,14 +156,17 @@ calicoctl_binary_checksums:
     v3.6.1: 0
     v3.5.4: 0
     v3.4.4: 0
+    v3.7.3: 0
   amd64:
     v3.6.1: 3b01336de37550e020343d62a38c96c4605d33a3ed7ddba2fe38bc172a5b42b5
     v3.5.4: 197194b838cc2a9a7455c2ebd5505a5e24f8f3d994eb75c17f5dd568944100b8
     v3.4.4: 93bd084e053cf1bf3b7fef369677bd6767c30fe7135e2c7e044e31693422ef61
+    v3.7.3: 932f68e893e80e95e10f064f1e7745e438d456f41a6ff12d11bb16ca0cab735c
   arm64:
     v3.6.1: 60fbaeb257061647bdf12b5ede7a0d4298a5ee216f6472e5a92bb14ef5c2a5d3
     v3.5.4: a4481178665658658a73e4ceca9a1dff5cccded4179615c91d1c3e49fd96f237
     v3.4.4: ff35d9e8b5c00e9fe47d05e8f5123ec98fd641370f8cd93f4fbb3d913da77ab6
+    v3.7.3: 7cfaab25c287f7ef93b2682d060b55bf39f76b668540de50376b5ed174209832
 
 etcd_binary_checksum: "{{ etcd_binary_checksums[image_arch] }}"
 cni_binary_checksum: "{{ cni_binary_checksums[image_arch] }}"

roles/kubernetes-apps/policy_controller/calico/defaults/main.yml

@@ -8,3 +8,6 @@ calico_policy_controller_memory_requests: 64M
 # SSL
 calico_cert_dir: "/etc/calico/certs"
 canal_cert_dir: "/etc/canal/certs"
+
+# Datastore type
+calico_datastore: "etcd"

roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2

@@ -46,7 +46,20 @@ spec:
             requests:
               cpu: {{ calico_policy_controller_cpu_requests }}
               memory: {{ calico_policy_controller_memory_requests }}
+{% if calico_version is version('v3.3.0', '>=') %}
+          readinessProbe:
+            exec:
+              command:
+              - /usr/bin/check-status
+              - -r
+{% endif %}
           env:
+{% if calico_datastore == "kdd" and calico_version is version('v3.6.0', '>=') %}
+            - name: ENABLED_CONTROLLERS
+              value: node
+            - name: DATASTORE_TYPE
+              value: kubernetes
+{% else %}
             - name: ETCD_ENDPOINTS
               value: "{{ etcd_access_addresses }}"
             - name: ETCD_CA_CERT_FILE
@@ -55,13 +68,6 @@ spec:
               value: "{{ calico_cert_dir }}/cert.crt"
             - name: ETCD_KEY_FILE
               value: "{{ calico_cert_dir }}/key.pem"
-{% if calico_version is version('v3.3.0', '>=') %}
-          readinessProbe:
-            exec:
-              command:
-              - /usr/bin/check-status
-              - -r
-{% endif %}
           volumeMounts:
           - mountPath: {{ calico_cert_dir }}
             name: etcd-certs
@@ -70,3 +76,4 @@ spec:
       - hostPath:
           path: {{ calico_cert_dir }}
         name: etcd-certs
+{% endif %}

roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2

@@ -5,6 +5,7 @@ metadata:
   name: calico-kube-controllers
   namespace: kube-system
 rules:
+{% if calico_datastore == "etcd"  %}
   - apiGroups:
     - ""
     - extensions
@@ -24,3 +25,44 @@ rules:
     verbs:
       - watch
       - list
+{% elif calico_datastore == "kdd" %}
+  # Nodes are watched to monitor for deletions.
+  - apiGroups: [""]
+    resources:
+      - nodes
+    verbs:
+      - watch
+      - list
+      - get
+  # Pods are queried to check for existence.
+  - apiGroups: [""]
+    resources:
+      - pods
+    verbs:
+      - get
+  # IPAM resources are manipulated when nodes are deleted.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - ippools
+    verbs:
+      - list
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - blockaffinities
+      - ipamblocks
+      - ipamhandles
+    verbs:
+      - get
+      - list
+      - create
+      - update
+      - delete
+  # Needs access to update clusterinformations.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - clusterinformations
+    verbs:
+      - get
+      - create
+      - update
+{% endif %}

roles/kubernetes-apps/policy_controller/meta/main.yml

@@ -4,11 +4,13 @@ dependencies:
     when:
       - kube_network_plugin == 'calico'
       - enable_network_policy
+      - calico_datastore != "kdd" or calico_policy_version is version('v3.6.0', '>=')
     tags:
       - policy-controller
 
   - role: policy_controller/calico
     when:
       - kube_network_plugin == 'canal'
+      - calico_datastore != "kdd" or calico_policy_version is version('v3.6.0', '>=')
     tags:
       - policy-controller

roles/network_plugin/calico/templates/calico-cr.yml.j2

@@ -71,9 +71,15 @@ rules:
       - globalbgpconfigs
       - bgpconfigurations
       - ippools
+{% if calico_version is version('v3.6.0', '>=') %}
+      - ipamblocks
+{% endif %}
       - globalnetworkpolicies
       - globalnetworksets
       - networkpolicies
+{% if calico_version is version('v3.7.0', '>=') %}
+      - networksets
+{% endif %}
       - clusterinformations
       - hostendpoints
     verbs:
@@ -106,4 +112,36 @@ rules:
     verbs:
       - create
       - update
+{% if calico_version is version('v3.6.0', '>=') %}
+  # These permissions are required for Calico CNI to perform IPAM allocations.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - blockaffinities
+      - ipamblocks
+      - ipamhandles
+    verbs:
+      - get
+      - list
+      - create
+      - update
+      - delete
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - ipamconfigs
+    verbs:
+      - get
+  # Block affinities must also be watchable by confd for route aggregation.
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - blockaffinities
+    verbs:
+      - watch
+  # The Calico IPAM migration needs to get daemonsets. These permissions can be
+  # removed if not upgrading from an installation using host-local IPAM.
+  - apiGroups: ["apps"]
+    resources:
+      - daemonsets
+    verbs:
+      - get
+{% endif %}
 {% endif %}

roles/network_plugin/calico/templates/calico-node.yml.j2

@@ -44,6 +44,29 @@ spec:
       terminationGracePeriodSeconds: 0
 {% if calico_version is version('v3.4.0', '>=') %}
       initContainers:
+{% if calico_datastore == "kdd" and calico_version is version('v3.6.0', '>=') %}
+        # This container performs upgrade from host-local IPAM to calico-ipam.
+        # It can be deleted if this is a fresh installation, or if you have already
+        # upgraded to use calico-ipam.
+        - name: upgrade-ipam
+          image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+          command: ["/opt/cni/bin/calico-ipam", "-upgrade"]
+          env:
+            - name: KUBERNETES_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CALICO_NETWORKING_BACKEND
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: calico_backend
+          volumeMounts:
+            - mountPath: /var/lib/cni/networks
+              name: host-local-net-dir
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+{% endif %}
         # This container installs the Calico CNI binaries
         # and CNI network config file on each node.
         - name: install-cni
@@ -304,6 +327,14 @@ spec:
           hostPath:
             path: /run/xtables.lock
             type: FileOrCreate
+{% if calico_datastore == "kdd" and calico_version is version('v3.6.0', '>=') %}
+        # Mount in the directory for host-local IPAM allocations. This is
+        # used when upgrading from host-local to calico-ipam, and can be removed
+        # if not using the upgrade-ipam init container.
+        - name: host-local-net-dir
+          hostPath:
+            path: /var/lib/cni/networks
+{% endif %}
   updateStrategy:
     rollingUpdate:
       maxUnavailable: {{ serial | default('20%') }}

roles/network_plugin/calico/templates/calico-typha.yml.j2

@@ -90,6 +90,12 @@ spec:
           #  value: "true"
           #- name: TYPHA_PROMETHEUSMETRICSPORT
           #  value: "9093"
+
+          # Needed for version >=3.7 when the 'host-local' ipam is used
+          # Should never happen given templates/cni-calico.conflist.j2
+          # Configure route aggregation based on pod CIDR.
+          # - name: USE_POD_CIDR
+          #   value: "true"
         livenessProbe:
           exec:
             command:

roles/network_plugin/calico/templates/cni-calico.conflist.j2

@@ -3,11 +3,16 @@
   "cniVersion":"0.3.1",
   "plugins":[
     {
+{% if calico_datastore == "kdd" %}
+      "datastore_type": "kubernetes",
+      "nodename": "__KUBERNETES_NODE_NAME__",
+{% else %}
 {% if cloud_provider is defined %}
       "nodename": "{{ calico_kubelet_name.stdout }}",
     {% else %}
       "nodename": "{{ calico_baremetal_nodename }}",
     {% endif %}
+{% endif %}
       "type": "calico",
       "log_level": "info",
 {% if calico_datastore == "etcd" %}
@@ -15,18 +20,18 @@
       "etcd_cert_file": "{{ calico_cert_dir }}/cert.crt",
       "etcd_key_file": "{{ calico_cert_dir }}/key.pem",
       "etcd_ca_cert_file": "{{ calico_cert_dir }}/ca_cert.crt",
+{% endif %}
+{% if calico_datastore == "kdd" and calico_version is version('v3.6.0', '<') %}
+      "ipam": {
+        "type": "host-local",
+        "subnet": "usePodCidr"
+      },
+{% else %}
       "ipam": {
         "type": "calico-ipam",
         "assign_ipv4": "true",
         "ipv4_pools": ["{{ calico_pool_cidr | default(kube_pods_subnet) }}"]
       },
-{% elif calico_datastore == "kdd" %}
-      "datastore_type": "kubernetes",
-      "nodename": "__KUBERNETES_NODE_NAME__",
-      "ipam": {
-        "type": "host-local",
-        "subnet": "usePodCidr"
-      },
 {% endif %}
 {% if (calico_feature_control is defined) and (calico_feature_control|length > 0) %}
       "feature_control": {

roles/network_plugin/calico/templates/kdd-crds.yml.j2

@@ -14,6 +14,68 @@ spec:
     singular: felixconfiguration
 ---
 
+{% if calico_version is version('v3.6.0', '>=') %}
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: ipamblocks.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: IPAMBlock
+    plural: ipamblocks
+    singular: ipamblock
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: blockaffinities.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: BlockAffinity
+    plural: blockaffinities
+    singular: blockaffinity
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: ipamhandles.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: IPAMHandle
+    plural: ipamhandles
+    singular: ipamhandle
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: ipamconfigs.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: IPAMConfig
+    plural: ipamconfigs
+    singular: ipamconfig
+
+---
+
+{% endif %}
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
@@ -131,3 +193,20 @@ spec:
     kind: NetworkPolicy
     plural: networkpolicies
     singular: networkpolicy
+
+{% if calico_version is version('v3.7.0', '>=') %}
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: networksets.crd.projectcalico.org
+spec:
+  scope: Namespaced
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: NetworkSet
+    plural: networksets
+    singular: networkset
+{% endif %}