From 20c7e31ea3d81d559f4d8f5c73e17ff5835ee0e5 Mon Sep 17 00:00:00 2001 From: jlacoline Date: Tue, 9 Jul 2019 21:42:28 +0200 Subject: [PATCH] Add calico 3.7.3 support (#4953) * Add calico 3.7.3 support * add calico_datastore variable to policy controller role * add missing clusterrole rules for calico policy controller * disable calico kube controller when kdd mode is used for versions < 3.6 --- roles/download/defaults/main.yml | 13 +-- .../calico/defaults/main.yml | 3 + .../templates/calico-kube-controllers.yml.j2 | 21 +++-- .../calico/templates/calico-kube-cr.yml.j2 | 42 ++++++++++ .../policy_controller/meta/main.yml | 2 + .../calico/templates/calico-cr.yml.j2 | 38 +++++++++ .../calico/templates/calico-node.yml.j2 | 31 ++++++++ .../calico/templates/calico-typha.yml.j2 | 6 ++ .../calico/templates/cni-calico.conflist.j2 | 19 +++-- .../calico/templates/kdd-crds.yml.j2 | 79 +++++++++++++++++++ 10 files changed, 235 insertions(+), 19 deletions(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index c04019ebb..12f10f141 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -58,12 +58,12 @@ kube_image_repo: "gcr.io/google-containers" # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download -calico_version: "v3.4.0" -calico_ctl_version: "v3.4.4" -calico_cni_version: "v3.4.0" -calico_policy_version: "v3.4.0" +calico_version: "v3.7.3" +calico_ctl_version: "v3.7.3" +calico_cni_version: "v3.7.3" +calico_policy_version: "v3.7.3" calico_rr_version: "v0.6.1" -calico_typha_version: "v3.4.4" +calico_typha_version: "v3.7.3" flannel_version: "v0.11.0" flannel_cni_version: "v0.3.0" @@ -156,14 +156,17 @@ calicoctl_binary_checksums: v3.6.1: 0 v3.5.4: 0 v3.4.4: 0 + v3.7.3: 0 amd64: v3.6.1: 3b01336de37550e020343d62a38c96c4605d33a3ed7ddba2fe38bc172a5b42b5 v3.5.4: 197194b838cc2a9a7455c2ebd5505a5e24f8f3d994eb75c17f5dd568944100b8 v3.4.4: 93bd084e053cf1bf3b7fef369677bd6767c30fe7135e2c7e044e31693422ef61 + v3.7.3: 932f68e893e80e95e10f064f1e7745e438d456f41a6ff12d11bb16ca0cab735c arm64: v3.6.1: 60fbaeb257061647bdf12b5ede7a0d4298a5ee216f6472e5a92bb14ef5c2a5d3 v3.5.4: a4481178665658658a73e4ceca9a1dff5cccded4179615c91d1c3e49fd96f237 v3.4.4: ff35d9e8b5c00e9fe47d05e8f5123ec98fd641370f8cd93f4fbb3d913da77ab6 + v3.7.3: 7cfaab25c287f7ef93b2682d060b55bf39f76b668540de50376b5ed174209832 etcd_binary_checksum: "{{ etcd_binary_checksums[image_arch] }}" cni_binary_checksum: "{{ cni_binary_checksums[image_arch] }}" diff --git a/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml b/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml index 93d12c901..a872b5be3 100644 --- a/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml +++ b/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml @@ -8,3 +8,6 @@ calico_policy_controller_memory_requests: 64M # SSL calico_cert_dir: "/etc/calico/certs" canal_cert_dir: "/etc/canal/certs" + +# Datastore type +calico_datastore: "etcd" diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 index 30e8b56ff..f015046e5 100644 --- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 +++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 @@ -46,7 +46,20 @@ spec: requests: cpu: {{ calico_policy_controller_cpu_requests }} memory: {{ calico_policy_controller_memory_requests }} +{% if calico_version is version('v3.3.0', '>=') %} + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r +{% endif %} env: +{% if calico_datastore == "kdd" and calico_version is version('v3.6.0', '>=') %} + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes +{% else %} - name: ETCD_ENDPOINTS value: "{{ etcd_access_addresses }}" - name: ETCD_CA_CERT_FILE @@ -55,13 +68,6 @@ spec: value: "{{ calico_cert_dir }}/cert.crt" - name: ETCD_KEY_FILE value: "{{ calico_cert_dir }}/key.pem" -{% if calico_version is version('v3.3.0', '>=') %} - readinessProbe: - exec: - command: - - /usr/bin/check-status - - -r -{% endif %} volumeMounts: - mountPath: {{ calico_cert_dir }} name: etcd-certs @@ -70,3 +76,4 @@ spec: - hostPath: path: {{ calico_cert_dir }} name: etcd-certs +{% endif %} diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 index 36e4dcc40..22395f678 100644 --- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 +++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 @@ -5,6 +5,7 @@ metadata: name: calico-kube-controllers namespace: kube-system rules: +{% if calico_datastore == "etcd" %} - apiGroups: - "" - extensions @@ -24,3 +25,44 @@ rules: verbs: - watch - list +{% elif calico_datastore == "kdd" %} + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are queried to check for existence. + - apiGroups: [""] + resources: + - pods + verbs: + - get + # IPAM resources are manipulated when nodes are deleted. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - create + - update +{% endif %} diff --git a/roles/kubernetes-apps/policy_controller/meta/main.yml b/roles/kubernetes-apps/policy_controller/meta/main.yml index 91cb0276c..5a3a489f7 100644 --- a/roles/kubernetes-apps/policy_controller/meta/main.yml +++ b/roles/kubernetes-apps/policy_controller/meta/main.yml @@ -4,11 +4,13 @@ dependencies: when: - kube_network_plugin == 'calico' - enable_network_policy + - calico_datastore != "kdd" or calico_policy_version is version('v3.6.0', '>=') tags: - policy-controller - role: policy_controller/calico when: - kube_network_plugin == 'canal' + - calico_datastore != "kdd" or calico_policy_version is version('v3.6.0', '>=') tags: - policy-controller diff --git a/roles/network_plugin/calico/templates/calico-cr.yml.j2 b/roles/network_plugin/calico/templates/calico-cr.yml.j2 index 64d953585..9e10d8523 100644 --- a/roles/network_plugin/calico/templates/calico-cr.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-cr.yml.j2 @@ -71,9 +71,15 @@ rules: - globalbgpconfigs - bgpconfigurations - ippools +{% if calico_version is version('v3.6.0', '>=') %} + - ipamblocks +{% endif %} - globalnetworkpolicies - globalnetworksets - networkpolicies +{% if calico_version is version('v3.7.0', '>=') %} + - networksets +{% endif %} - clusterinformations - hostendpoints verbs: @@ -106,4 +112,36 @@ rules: verbs: - create - update +{% if calico_version is version('v3.6.0', '>=') %} + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get +{% endif %} {% endif %} diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2 index 6d0b6a63f..f201bfec0 100644 --- a/roles/network_plugin/calico/templates/calico-node.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-node.yml.j2 @@ -44,6 +44,29 @@ spec: terminationGracePeriodSeconds: 0 {% if calico_version is version('v3.4.0', '>=') %} initContainers: +{% if calico_datastore == "kdd" and calico_version is version('v3.6.0', '>=') %} + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir +{% endif %} # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni @@ -304,6 +327,14 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate +{% if calico_datastore == "kdd" and calico_version is version('v3.6.0', '>=') %} + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks +{% endif %} updateStrategy: rollingUpdate: maxUnavailable: {{ serial | default('20%') }} diff --git a/roles/network_plugin/calico/templates/calico-typha.yml.j2 b/roles/network_plugin/calico/templates/calico-typha.yml.j2 index 36181281b..d493ff657 100644 --- a/roles/network_plugin/calico/templates/calico-typha.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-typha.yml.j2 @@ -90,6 +90,12 @@ spec: # value: "true" #- name: TYPHA_PROMETHEUSMETRICSPORT # value: "9093" + + # Needed for version >=3.7 when the 'host-local' ipam is used + # Should never happen given templates/cni-calico.conflist.j2 + # Configure route aggregation based on pod CIDR. + # - name: USE_POD_CIDR + # value: "true" livenessProbe: exec: command: diff --git a/roles/network_plugin/calico/templates/cni-calico.conflist.j2 b/roles/network_plugin/calico/templates/cni-calico.conflist.j2 index cfee547d7..5a8414a9a 100644 --- a/roles/network_plugin/calico/templates/cni-calico.conflist.j2 +++ b/roles/network_plugin/calico/templates/cni-calico.conflist.j2 @@ -3,11 +3,16 @@ "cniVersion":"0.3.1", "plugins":[ { +{% if calico_datastore == "kdd" %} + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", +{% else %} {% if cloud_provider is defined %} "nodename": "{{ calico_kubelet_name.stdout }}", {% else %} "nodename": "{{ calico_baremetal_nodename }}", {% endif %} +{% endif %} "type": "calico", "log_level": "info", {% if calico_datastore == "etcd" %} @@ -15,18 +20,18 @@ "etcd_cert_file": "{{ calico_cert_dir }}/cert.crt", "etcd_key_file": "{{ calico_cert_dir }}/key.pem", "etcd_ca_cert_file": "{{ calico_cert_dir }}/ca_cert.crt", +{% endif %} +{% if calico_datastore == "kdd" and calico_version is version('v3.6.0', '<') %} + "ipam": { + "type": "host-local", + "subnet": "usePodCidr" + }, +{% else %} "ipam": { "type": "calico-ipam", "assign_ipv4": "true", "ipv4_pools": ["{{ calico_pool_cidr | default(kube_pods_subnet) }}"] }, -{% elif calico_datastore == "kdd" %} - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "ipam": { - "type": "host-local", - "subnet": "usePodCidr" - }, {% endif %} {% if (calico_feature_control is defined) and (calico_feature_control|length > 0) %} "feature_control": { diff --git a/roles/network_plugin/calico/templates/kdd-crds.yml.j2 b/roles/network_plugin/calico/templates/kdd-crds.yml.j2 index 3f14729f4..7decacc49 100644 --- a/roles/network_plugin/calico/templates/kdd-crds.yml.j2 +++ b/roles/network_plugin/calico/templates/kdd-crds.yml.j2 @@ -14,6 +14,68 @@ spec: singular: felixconfiguration --- +{% if calico_version is version('v3.6.0', '>=') %} +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ipamblocks.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPAMBlock + plural: ipamblocks + singular: ipamblock + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: blockaffinities.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BlockAffinity + plural: blockaffinities + singular: blockaffinity + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ipamhandles.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPAMHandle + plural: ipamhandles + singular: ipamhandle + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ipamconfigs.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPAMConfig + plural: ipamconfigs + singular: ipamconfig + +--- + +{% endif %} apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -131,3 +193,20 @@ spec: kind: NetworkPolicy plural: networkpolicies singular: networkpolicy + +{% if calico_version is version('v3.7.0', '>=') %} +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: networksets.crd.projectcalico.org +spec: + scope: Namespaced + group: crd.projectcalico.org + version: v1 + names: + kind: NetworkSet + plural: networksets + singular: networkset +{% endif %}