From c5fb73409890983b3c0b0cd896f958b998a95512 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Mon, 15 Apr 2019 14:24:04 +0300 Subject: [PATCH] Switch calicoctl from a container to a binary (#4524) --- roles/download/defaults/main.yml | 27 ++++++++---- .../preinstall/tasks/0020-verify-settings.yml | 6 +-- roles/network_plugin/calico/tasks/check.yml | 9 ++-- roles/network_plugin/calico/tasks/install.yml | 44 +++++++++++-------- .../calico/templates/calicoctl-container.j2 | 14 ------ .../calico/templates/calicoctl.sh.j2 | 6 +++ roles/network_plugin/canal/tasks/main.yml | 6 +-- .../canal/templates/calicoctl-container.j2 | 15 ------- .../canal/templates/calicoctl.sh.j2 | 6 +++ 9 files changed, 67 insertions(+), 66 deletions(-) delete mode 100644 roles/network_plugin/calico/templates/calicoctl-container.j2 create mode 100644 roles/network_plugin/calico/templates/calicoctl.sh.j2 delete mode 100644 roles/network_plugin/canal/templates/calicoctl-container.j2 create mode 100644 roles/network_plugin/canal/templates/calicoctl.sh.j2 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 352309311..db38cc7c6 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -45,7 +45,7 @@ kube_image_repo: "gcr.io/google-containers" # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download calico_version: "v3.4.0" -calico_ctl_version: "v3.4.0" +calico_ctl_version: "v3.4.4" calico_cni_version: "v3.4.0" calico_policy_version: "v3.4.0" calico_rr_version: "v0.6.1" @@ -67,6 +67,7 @@ kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/{{ image_arch }}/hyperkube" etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-{{ image_arch }}.tar.gz" cni_download_url: "https://github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-{{ image_arch }}-{{ cni_version }}.tgz" +calicoctl_download_url: "https://github.com/projectcalico/calicoctl/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}" # Checksums hyperkube_checksums: @@ -138,11 +139,19 @@ etcd_binary_checksums: cni_binary_checksums: arm64: 016bbc989877e35e3cd49fafe11415fb2717e52c74fde6b1650411154cb91b81 amd64: f04339a21b8edf76d415e7f17b620e63b8f37a76b2f706671587ab6464411f2d +calicoctl_binary_checksums: + amd64: + v3.5.4: 197194b838cc2a9a7455c2ebd5505a5e24f8f3d994eb75c17f5dd568944100b8 + v3.4.4: 93bd084e053cf1bf3b7fef369677bd6767c30fe7135e2c7e044e31693422ef61 + arm64: + v3.5.4: a4481178665658658a73e4ceca9a1dff5cccded4179615c91d1c3e49fd96f237 + v3.4.4: ff35d9e8b5c00e9fe47d05e8f5123ec98fd641370f8cd93f4fbb3d913da77ab6 etcd_binary_checksum: "{{ etcd_binary_checksums[image_arch] }}" cni_binary_checksum: "{{ cni_binary_checksums[image_arch] }}" hyperkube_binary_checksum: "{{ hyperkube_checksums[image_arch][kube_version] }}" kubeadm_binary_checksum: "{{ kubeadm_checksums[image_arch][kubeadm_version] }}" +calicoctl_binary_checksum: "{{ calicoctl_binary_checksums[image_arch][calico_ctl_version] }}" # Containers # In some cases, we need a way to set --registry-mirror or --insecure-registry for docker, @@ -158,8 +167,6 @@ flannel_image_repo: "quay.io/coreos/flannel" flannel_image_tag: "{{ flannel_version }}" flannel_cni_image_repo: "quay.io/coreos/flannel-cni" flannel_cni_image_tag: "{{ flannel_cni_version }}" -calicoctl_image_repo: "docker.io/calico/ctl" -calicoctl_image_tag: "{{ calico_ctl_version }}" calico_node_image_repo: "docker.io/calico/node" calico_node_image_tag: "{{ calico_version }}" calico_cni_image_repo: "docker.io/calico/cni" @@ -195,7 +202,7 @@ cilium_image_repo: "docker.io/cilium/cilium" cilium_image_tag: "{{ cilium_version }}" cilium_init_image_repo: "docker.io/library/busybox" cilium_init_image_tag: "1.28.4" -kube_router_image_repo: "docker.io/cloudnativelabs/kube-router" +kube_router_image_repo: "cloudnativelabs/kube-router" kube_router_image_tag: "{{ kube_router_version }}" multus_image_repo: "docker.io/nfvpe/multus" multus_image_tag: "{{ multus_version }}" @@ -372,10 +379,14 @@ downloads: calicoctl: enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}" - container: true - repo: "{{ calicoctl_image_repo }}" - tag: "{{ calicoctl_image_tag }}" - sha256: "{{ calicoctl_digest_checksum|default(None) }}" + file: true + version: "{{ calico_ctl_version }}" + dest: "{{local_release_dir}}/calicoctl" + sha256: "{{ calicoctl_binary_checksum }}" + url: "{{ calicoctl_download_url }}" + unarchive: false + owner: "root" + mode: "0755" groups: - k8s-cluster diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index 599484878..6ac994c58 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -130,13 +130,11 @@ - facts - name: "Get current version of calico cluster version" - shell: "{{ bin_dir }}/calicoctl version | grep 'Cluster Version:' | awk '{ print $3}'" + shell: "{{ bin_dir }}/calicoctl.sh version | grep 'Cluster Version:' | awk '{ print $3}'" register: calico_version_on_server - async: 10 - poll: 3 run_once: yes - delegate_to: "{{ groups['kube-master'][0] }}" changed_when: false + failed_when: false when: - kube_network_plugin == 'calico' diff --git a/roles/network_plugin/calico/tasks/check.yml b/roles/network_plugin/calico/tasks/check.yml index 88fbe63a2..719bbe78a 100644 --- a/roles/network_plugin/calico/tasks/check.yml +++ b/roles/network_plugin/calico/tasks/check.yml @@ -13,10 +13,13 @@ shell: "{{ bin_dir }}/calicoctl version | grep 'Cluster Version:' | awk '{ print $3}'" register: calico_version_on_server run_once: yes - delegate_to: "{{ groups['kube-master'][0] }}" - async: 10 - poll: 3 changed_when: false + environment: + ETCD_ENDPOINTS: "{{ etcd_access_addresses }}" + ETCD_CA_CERT_FILE: "{{ calico_cert_dir }}/ca_cert.crt" + ETCD_CERT_FILE: "{{ calico_cert_dir }}/cert.crt" + ETCD_KEY_FILE: "{{ calico_cert_dir }}/key.pem" + - name: "Determine if calico upgrade is needed" block: diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index acf3fd761..992710944 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -1,4 +1,11 @@ --- +- name: Calico | Copy calicoctl binary from download dir + copy: + src: "{{ local_release_dir }}/calicoctl" + dest: "{{ bin_dir }}/calicoctl" + mode: 0755 + remote_src: yes + - name: Calico | Write Calico cni config template: src: "cni-calico.conflist.j2" @@ -24,14 +31,13 @@ - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"} - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"} -- name: Calico | Install calicoctl container script +- name: Calico | Install calicoctl wrapper script template: - src: calicoctl-container.j2 - dest: "{{ bin_dir }}/calicoctl" + src: calicoctl.sh.j2 + dest: "{{ bin_dir }}/calicoctl.sh" mode: 0755 owner: root group: root - changed_when: false - name: Calico | wait for etcd uri: @@ -47,7 +53,7 @@ - name: Calico | Check if calico network pool has already been configured shell: > - {{ bin_dir }}/calicoctl get ippool | grep -w "{{ calico_pool_cidr | default(kube_pods_subnet) }}" | wc -l + {{ bin_dir }}/calicoctl.sh get ippool | grep -w "{{ calico_pool_cidr | default(kube_pods_subnet) }}" | wc -l register: calico_conf retries: 4 delay: "{{ retry_stagger | random + 3 }}" @@ -76,7 +82,7 @@ "spec": { "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}", "ipipMode": "{{ ipip_mode }}", - "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl create -f - + "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl.sh create -f - run_once: true delegate_to: "{{ groups['kube-master'][0] }}" when: @@ -96,7 +102,7 @@ "blockSize": "{{ kube_network_node_prefix }}", "cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}", "ipipMode": "{{ ipip_mode }}", - "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl create -f - + "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl.sh create -f - run_once: true delegate_to: "{{ groups['kube-master'][0] }}" when: @@ -111,7 +117,7 @@ "nat-outgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }}}, "apiVersion": "v1", "metadata": {"cidr": "{{ calico_pool_cidr | default(kube_pods_subnet) }}"} - }' | {{ bin_dir }}/calicoctl apply -f - + }' | {{ bin_dir }}/calicoctl.sh apply -f - environment: NO_DEFAULT_POOLS: true run_once: true @@ -139,20 +145,20 @@ "spec": { "logSeverityScreen": "Info", "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled|default('true') }} , - "asNumber": {{ global_as_num }} }} ' | {{ bin_dir }}/calicoctl create --skip-exists -f - + "asNumber": {{ global_as_num }} }} ' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - run_once: true delegate_to: "{{ groups['kube-master'][0] }}" when: - calico_version is version('v3.0.0', '>=') - name: Calico | Set global as_num (legacy) - command: "{{ bin_dir}}/calicoctl config set asNumber {{ global_as_num }}" + command: "{{ bin_dir}}/calicoctl.sh config set asNumber {{ global_as_num }}" run_once: true when: - calico_version is version('v3.0.0', '<') - name: Calico | Disable node mesh (legacy) - command: "{{ bin_dir }}/calicoctl config set nodeToNodeMesh off" + command: "{{ bin_dir }}/calicoctl.sh config set nodeToNodeMesh off" run_once: yes when: - calico_version is version('v3.0.0', '<') @@ -171,7 +177,7 @@ "asNumber": "{{ local_as }}" }, "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}] - }}' | {{ bin_dir }}/calicoctl create --skip-exists -f - + }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" when: @@ -194,7 +200,7 @@ "asNumber": "{{ local_as }}" }, "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}] - }}' | {{ bin_dir }}/calicoctl create --skip-exists -f - + }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" when: @@ -216,7 +222,7 @@ "asNumber": "{{ item.as }}", "node": "{{ inventory_hostname }}", "peerIP": "{{ item.router_id }}" - }}' | {{ bin_dir }}/calicoctl create --skip-exists -f - + }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: @@ -234,7 +240,7 @@ "apiVersion": "v1", "metadata": {"node": "{{ inventory_hostname }}", "scope": "node", "peerIP": "{{ item.router_id }}"} }' - | {{ bin_dir }}/calicoctl create --skip-exists -f - + | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: "{{ peers|selectattr('scope','undefined')|list|default([]) | union(peers|selectattr('scope','defined')|selectattr('scope','equalto', 'node')|list|default([])) }}" @@ -254,7 +260,7 @@ "spec": { "asNumber": "{{ item.as }}", "peerIP": "{{ item.router_id }}" - }}' | {{ bin_dir }}/calicoctl create --skip-exists -f - + }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: @@ -273,7 +279,7 @@ "apiVersion": "v1", "metadata": {"scope": "global", "peerIP": "{{ item.router_id }}"} }' - | {{ bin_dir }}/calicoctl create --skip-exists -f - + | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: "{{ peers|selectattr('scope','defined')|selectattr('scope','equalto', 'global')|default([]) }}" @@ -295,7 +301,7 @@ "asNumber": "{{ local_as | default(global_as_num)}}", "node": "{{ inventory_hostname }}", "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}" - }}' | {{ bin_dir }}/calicoctl create --skip-exists -f - + }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: @@ -316,7 +322,7 @@ "scope": "node", "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}"} }' - | {{ bin_dir }}/calicoctl create --skip-exists -f - + | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: "{{ groups['calico-rr'] | default([]) }}" diff --git a/roles/network_plugin/calico/templates/calicoctl-container.j2 b/roles/network_plugin/calico/templates/calicoctl-container.j2 deleted file mode 100644 index c9a1b5d40..000000000 --- a/roles/network_plugin/calico/templates/calicoctl-container.j2 +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -{{ docker_bin_dir }}/docker run -i --privileged --rm \ ---net=host --pid=host \ --e ETCD_ENDPOINTS={{ etcd_access_addresses }} \ --e ETCD_CA_CERT_FILE={{ calico_cert_dir }}/ca_cert.crt \ --e ETCD_CERT_FILE={{ calico_cert_dir }}/cert.crt \ --e ETCD_KEY_FILE={{ calico_cert_dir }}/key.pem \ --v {{ docker_bin_dir }}/docker:{{ docker_bin_dir }}/docker \ --v /var/run/docker.sock:/var/run/docker.sock \ --v /var/run/calico:/var/run/calico \ --v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \ ---memory={{ calicoctl_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calicoctl_cpu_limit|regex_replace('m', '') }} \ -{{ calicoctl_image_repo }}:{{ calicoctl_image_tag}} \ -"$@" diff --git a/roles/network_plugin/calico/templates/calicoctl.sh.j2 b/roles/network_plugin/calico/templates/calicoctl.sh.j2 new file mode 100644 index 000000000..8343ef8f8 --- /dev/null +++ b/roles/network_plugin/calico/templates/calicoctl.sh.j2 @@ -0,0 +1,6 @@ +#!/bin/bash +ETCD_ENDPOINTS={{ etcd_access_addresses }} \ +ETCD_CA_CERT_FILE={{ calico_cert_dir }}/ca_cert.crt \ +ETCD_CERT_FILE={{ calico_cert_dir }}/cert.crt \ +ETCD_KEY_FILE={{ calico_cert_dir }}/key.pem \ +{{ bin_dir }}/calicoctl "$@" diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml index 945274937..f51da3bb5 100644 --- a/roles/network_plugin/canal/tasks/main.yml +++ b/roles/network_plugin/canal/tasks/main.yml @@ -54,10 +54,10 @@ when: - inventory_hostname in groups['kube-master'] -- name: Canal | Install calicoctl container script +- name: Canal | Install calicoctl wrapper script template: - src: calicoctl-container.j2 - dest: "{{ bin_dir }}/calicoctl" + src: calicoctl.sh.j2 + dest: "{{ bin_dir }}/calicoctl.sh" mode: 0755 owner: root group: root diff --git a/roles/network_plugin/canal/templates/calicoctl-container.j2 b/roles/network_plugin/canal/templates/calicoctl-container.j2 deleted file mode 100644 index cc0a38bfc..000000000 --- a/roles/network_plugin/canal/templates/calicoctl-container.j2 +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -{{ docker_bin_dir }}/docker run -i --privileged --rm \ ---net=host --pid=host \ --e ETCD_ENDPOINTS={{ etcd_access_addresses }} \ --e ETCD_CA_CERT_FILE={{ canal_cert_dir }}/ca_cert.crt \ --e ETCD_CERT_FILE={{ canal_cert_dir }}/cert.crt \ --e ETCD_KEY_FILE={{ canal_cert_dir }}/key.pem \ --v {{ docker_bin_dir }}/docker:{{ docker_bin_dir }}/docker \ --v /var/run/docker.sock:/var/run/docker.sock \ --v /var/run/calico:/var/run/calico \ --v {{ canal_cert_dir }}:{{ canal_cert_dir }}:ro \ --v {{ canal_policy_dir }}:{{ canal_policy_dir }}:ro \ ---memory={{ calicoctl_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calicoctl_cpu_limit|regex_replace('m', '') }} \ -{{ calicoctl_image_repo }}:{{ calicoctl_image_tag}} \ -"$@" diff --git a/roles/network_plugin/canal/templates/calicoctl.sh.j2 b/roles/network_plugin/canal/templates/calicoctl.sh.j2 new file mode 100644 index 000000000..8343ef8f8 --- /dev/null +++ b/roles/network_plugin/canal/templates/calicoctl.sh.j2 @@ -0,0 +1,6 @@ +#!/bin/bash +ETCD_ENDPOINTS={{ etcd_access_addresses }} \ +ETCD_CA_CERT_FILE={{ calico_cert_dir }}/ca_cert.crt \ +ETCD_CERT_FILE={{ calico_cert_dir }}/cert.crt \ +ETCD_KEY_FILE={{ calico_cert_dir }}/key.pem \ +{{ bin_dir }}/calicoctl "$@"