Switch calicoctl from a container to a binary (#4524)

roles/download/defaults/main.yml

@@ -45,7 +45,7 @@ kube_image_repo: "gcr.io/google-containers"
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
 calico_version: "v3.4.0"
-calico_ctl_version: "v3.4.0"
+calico_ctl_version: "v3.4.4"
 calico_cni_version: "v3.4.0"
 calico_policy_version: "v3.4.0"
 calico_rr_version: "v0.6.1"
@@ -67,6 +67,7 @@ kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release
 hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/{{ image_arch }}/hyperkube"
 etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz"
 cni_download_url: "https://github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-{{ image_arch }}-{{ cni_version }}.tgz"
+calicoctl_download_url: "https://github.com/projectcalico/calicoctl/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
 
 # Checksums
 hyperkube_checksums:
@@ -138,11 +139,19 @@ etcd_binary_checksums:
 cni_binary_checksums:
   arm64: 016bbc989877e35e3cd49fafe11415fb2717e52c74fde6b1650411154cb91b81
   amd64: f04339a21b8edf76d415e7f17b620e63b8f37a76b2f706671587ab6464411f2d
+calicoctl_binary_checksums:
+  amd64:
+    v3.5.4: 197194b838cc2a9a7455c2ebd5505a5e24f8f3d994eb75c17f5dd568944100b8
+    v3.4.4: 93bd084e053cf1bf3b7fef369677bd6767c30fe7135e2c7e044e31693422ef61
+  arm64:
+    v3.5.4: a4481178665658658a73e4ceca9a1dff5cccded4179615c91d1c3e49fd96f237
+    v3.4.4: ff35d9e8b5c00e9fe47d05e8f5123ec98fd641370f8cd93f4fbb3d913da77ab6
 
 etcd_binary_checksum: "{{ etcd_binary_checksums[image_arch] }}"
 cni_binary_checksum: "{{ cni_binary_checksums[image_arch] }}"
 hyperkube_binary_checksum: "{{ hyperkube_checksums[image_arch][kube_version] }}"
 kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}"
+calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}"
 
 # Containers
 # In some cases, we need a way to set --registry-mirror or --insecure-registry for docker,
@@ -158,8 +167,6 @@ flannel_image_repo: "quay.io/coreos/flannel"
 flannel_image_tag: "{{ flannel_version }}"
 flannel_cni_image_repo: "quay.io/coreos/flannel-cni"
 flannel_cni_image_tag: "{{ flannel_cni_version }}"
-calicoctl_image_repo: "docker.io/calico/ctl"
-calicoctl_image_tag: "{{ calico_ctl_version }}"
 calico_node_image_repo: "docker.io/calico/node"
 calico_node_image_tag: "{{ calico_version }}"
 calico_cni_image_repo: "docker.io/calico/cni"
@@ -195,7 +202,7 @@ cilium_image_repo: "docker.io/cilium/cilium"
 cilium_image_tag: "{{ cilium_version }}"
 cilium_init_image_repo: "docker.io/library/busybox"
 cilium_init_image_tag: "1.28.4"
-kube_router_image_repo: "docker.io/cloudnativelabs/kube-router"
+kube_router_image_repo: "cloudnativelabs/kube-router"
 kube_router_image_tag: "{{ kube_router_version }}"
 multus_image_repo: "docker.io/nfvpe/multus"
 multus_image_tag: "{{ multus_version }}"
@@ -372,10 +379,14 @@ downloads:
 
   calicoctl:
     enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}"
-    container: true
+    file: true
-    repo: "{{ calicoctl_image_repo }}"
+    version: "{{ calico_ctl_version }}"
-    tag: "{{ calicoctl_image_tag }}"
+    dest: "{{local_release_dir}}/calicoctl"
-    sha256: "{{ calicoctl_digest_checksum|default(None) }}"
+    sha256: "{{ calicoctl_binary_checksum }}"
+    url: "{{ calicoctl_download_url }}"
+    unarchive: false
+    owner: "root"
+    mode: "0755"
     groups:
       - k8s-cluster
 

roles/kubernetes/preinstall/tasks/0020-verify-settings.yml

@@ -130,13 +130,11 @@
     - facts
 
 - name: "Get current version of calico cluster version"
-  shell: "{{ bin_dir }}/calicoctl version  | grep 'Cluster Version:' | awk '{ print $3}'"
+  shell: "{{ bin_dir }}/calicoctl.sh version  | grep 'Cluster Version:' | awk '{ print $3}'"
   register: calico_version_on_server
-  async: 10
-  poll: 3
   run_once: yes
-  delegate_to: "{{ groups['kube-master'][0] }}"
   changed_when: false
+  failed_when: false
   when:
     - kube_network_plugin == 'calico'
 

roles/network_plugin/calico/tasks/check.yml

@@ -13,10 +13,13 @@
   shell: "{{ bin_dir }}/calicoctl version  | grep 'Cluster Version:' | awk '{ print $3}'"
   register: calico_version_on_server
   run_once: yes
-  delegate_to: "{{ groups['kube-master'][0] }}"
-  async: 10
-  poll: 3
   changed_when: false
+  environment:
+    ETCD_ENDPOINTS: "{{ etcd_access_addresses }}"
+    ETCD_CA_CERT_FILE: "{{ calico_cert_dir }}/ca_cert.crt"
+    ETCD_CERT_FILE: "{{ calico_cert_dir }}/cert.crt"
+    ETCD_KEY_FILE: "{{ calico_cert_dir }}/key.pem"
+
 
 - name: "Determine if calico upgrade is needed"
   block:

roles/network_plugin/calico/tasks/install.yml

@@ -1,4 +1,11 @@
 ---
+- name: Calico | Copy calicoctl binary from download dir
+  copy:
+    src: "{{ local_release_dir }}/calicoctl"
+    dest: "{{ bin_dir }}/calicoctl"
+    mode: 0755
+    remote_src: yes
+
 - name: Calico | Write Calico cni config
   template:
     src: "cni-calico.conflist.j2"
@@ -24,14 +31,13 @@
     - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
     - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
 
-- name: Calico | Install calicoctl container script
+- name: Calico | Install calicoctl wrapper script
   template:
-    src: calicoctl-container.j2
+    src: calicoctl.sh.j2
-    dest: "{{ bin_dir }}/calicoctl"
+    dest: "{{ bin_dir }}/calicoctl.sh"
     mode: 0755
     owner: root
     group: root
-  changed_when: false
 
 - name: Calico | wait for etcd
   uri:
@@ -47,7 +53,7 @@
 
 - name: Calico | Check if calico network pool has already been configured
   shell: >
-    {{ bin_dir }}/calicoctl get ippool | grep -w "{{ calico_pool_cidr | default(kube_pods_subnet) }}" | wc -l
+    {{ bin_dir }}/calicoctl.sh get ippool | grep -w "{{ calico_pool_cidr | default(kube_pods_subnet) }}" | wc -l
   register: calico_conf
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
@@ -76,7 +82,7 @@
         "spec": {
           "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}",
           "ipipMode": "{{ ipip_mode }}",
-          "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl create -f -
+          "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl.sh create -f -
   run_once: true
   delegate_to: "{{ groups['kube-master'][0] }}"
   when:
@@ -96,7 +102,7 @@
           "blockSize": "{{ kube_network_node_prefix }}",
           "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}",
           "ipipMode": "{{ ipip_mode }}",
-          "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl create -f -
+          "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl.sh create -f -
   run_once: true
   delegate_to: "{{ groups['kube-master'][0] }}"
   when:
@@ -111,7 +117,7 @@
                  "nat-outgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }}},
         "apiVersion": "v1",
         "metadata": {"cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}"}
-      }' | {{ bin_dir }}/calicoctl apply -f -
+      }' | {{ bin_dir }}/calicoctl.sh apply -f -
   environment:
     NO_DEFAULT_POOLS: true
   run_once: true
@@ -139,20 +145,20 @@
     "spec": {
         "logSeverityScreen": "Info",
         "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled|default('true') }} ,
-        "asNumber": {{ global_as_num }} }} ' | {{ bin_dir }}/calicoctl create --skip-exists -f -
+        "asNumber": {{ global_as_num }} }} ' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
   run_once: true
   delegate_to: "{{ groups['kube-master'][0] }}"
   when:
     - calico_version is version('v3.0.0', '>=')
 
 - name: Calico | Set global as_num (legacy)
-  command: "{{ bin_dir}}/calicoctl config set asNumber {{ global_as_num }}"
+  command: "{{ bin_dir}}/calicoctl.sh config set asNumber {{ global_as_num }}"
   run_once: true
   when:
     - calico_version is version('v3.0.0', '<')
 
 - name: Calico | Disable node mesh (legacy)
-  command: "{{ bin_dir }}/calicoctl config set nodeToNodeMesh off"
+  command: "{{ bin_dir }}/calicoctl.sh config set nodeToNodeMesh off"
   run_once: yes
   when:
     - calico_version is version('v3.0.0', '<')
@@ -171,7 +177,7 @@
         "asNumber": "{{ local_as }}"
       },
       "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}]
-   }}' | {{ bin_dir }}/calicoctl create --skip-exists -f -
+   }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   when:
@@ -194,7 +200,7 @@
         "asNumber": "{{ local_as }}"
       },
       "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}]
-   }}' | {{ bin_dir }}/calicoctl create --skip-exists -f -
+   }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   when:
@@ -216,7 +222,7 @@
       "asNumber": "{{ item.as }}",
       "node": "{{ inventory_hostname }}",
       "peerIP": "{{ item.router_id }}"
-   }}' | {{ bin_dir }}/calicoctl create --skip-exists -f -
+   }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   with_items:
@@ -234,7 +240,7 @@
    "apiVersion": "v1",
    "metadata": {"node": "{{ inventory_hostname }}", "scope": "node", "peerIP": "{{ item.router_id }}"}
    }'
-   | {{ bin_dir }}/calicoctl create --skip-exists -f -
+   | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{ peers|selectattr('scope','undefined')|list|default([]) | union(peers|selectattr('scope','defined')|selectattr('scope','equalto', 'node')|list|default([])) }}"
@@ -254,7 +260,7 @@
    "spec": {
       "asNumber": "{{ item.as }}",
       "peerIP": "{{ item.router_id }}"
-   }}' | {{ bin_dir }}/calicoctl create --skip-exists -f -
+   }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   with_items:
@@ -273,7 +279,7 @@
    "apiVersion": "v1",
    "metadata": {"scope": "global", "peerIP": "{{ item.router_id }}"}
    }'
-   | {{ bin_dir }}/calicoctl create --skip-exists -f -
+   | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{ peers|selectattr('scope','defined')|selectattr('scope','equalto', 'global')|default([]) }}"
@@ -295,7 +301,7 @@
       "asNumber": "{{ local_as | default(global_as_num)}}",
       "node": "{{ inventory_hostname }}",
       "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}"
-   }}' | {{ bin_dir }}/calicoctl create --skip-exists -f -
+   }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   with_items:
@@ -316,7 +322,7 @@
      "scope": "node",
      "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}"}
    }'
-   | {{ bin_dir }}/calicoctl create --skip-exists -f -
+   | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{ groups['calico-rr'] | default([]) }}"

roles/network_plugin/calico/templates/calicoctl-container.j2

@@ -1,14 +0,0 @@
-#!/bin/bash
-{{ docker_bin_dir }}/docker run -i --privileged --rm \
---net=host --pid=host \
--e ETCD_ENDPOINTS={{ etcd_access_addresses }} \
--e ETCD_CA_CERT_FILE={{ calico_cert_dir }}/ca_cert.crt \
--e ETCD_CERT_FILE={{ calico_cert_dir }}/cert.crt \
--e ETCD_KEY_FILE={{ calico_cert_dir }}/key.pem \
--v {{ docker_bin_dir }}/docker:{{ docker_bin_dir }}/docker \
--v /var/run/docker.sock:/var/run/docker.sock \
--v /var/run/calico:/var/run/calico \
--v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \
---memory={{ calicoctl_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calicoctl_cpu_limit|regex_replace('m', '') }} \
-{{ calicoctl_image_repo }}:{{ calicoctl_image_tag}} \
-"$@"

roles/network_plugin/calico/templates/calicoctl.sh.j2

@@ -0,0 +1,6 @@
+#!/bin/bash
+ETCD_ENDPOINTS={{ etcd_access_addresses }} \
+ETCD_CA_CERT_FILE={{ calico_cert_dir }}/ca_cert.crt \
+ETCD_CERT_FILE={{ calico_cert_dir }}/cert.crt \
+ETCD_KEY_FILE={{ calico_cert_dir }}/key.pem \
+{{ bin_dir }}/calicoctl "$@"

roles/network_plugin/canal/tasks/main.yml

@@ -54,10 +54,10 @@
   when:
     - inventory_hostname in groups['kube-master']
 
-- name: Canal | Install calicoctl container script
+- name: Canal | Install calicoctl wrapper script
   template:
-    src: calicoctl-container.j2
+    src: calicoctl.sh.j2
-    dest: "{{ bin_dir }}/calicoctl"
+    dest: "{{ bin_dir }}/calicoctl.sh"
     mode: 0755
     owner: root
     group: root

roles/network_plugin/canal/templates/calicoctl-container.j2

@@ -1,15 +0,0 @@
-#!/bin/bash
-{{ docker_bin_dir }}/docker run -i --privileged --rm \
---net=host --pid=host \
--e ETCD_ENDPOINTS={{ etcd_access_addresses }} \
--e ETCD_CA_CERT_FILE={{ canal_cert_dir }}/ca_cert.crt \
--e ETCD_CERT_FILE={{ canal_cert_dir }}/cert.crt \
--e ETCD_KEY_FILE={{ canal_cert_dir }}/key.pem \
--v {{ docker_bin_dir }}/docker:{{ docker_bin_dir }}/docker \
--v /var/run/docker.sock:/var/run/docker.sock \
--v /var/run/calico:/var/run/calico \
--v {{ canal_cert_dir }}:{{ canal_cert_dir }}:ro \
--v {{ canal_policy_dir }}:{{ canal_policy_dir }}:ro \
---memory={{ calicoctl_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calicoctl_cpu_limit|regex_replace('m', '') }} \
-{{ calicoctl_image_repo }}:{{ calicoctl_image_tag}} \
-"$@"

roles/network_plugin/canal/templates/calicoctl.sh.j2

@@ -0,0 +1,6 @@
+#!/bin/bash
+ETCD_ENDPOINTS={{ etcd_access_addresses }} \
+ETCD_CA_CERT_FILE={{ calico_cert_dir }}/ca_cert.crt \
+ETCD_CERT_FILE={{ calico_cert_dir }}/cert.crt \
+ETCD_KEY_FILE={{ calico_cert_dir }}/key.pem \
+{{ bin_dir }}/calicoctl "$@"