C12s/c12s-kubespray
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

calico upgrade to v3 (#3086)

Browse source 
* calico upgrade to v3

* update calico_rr version

* add missing file

* change contents of main.yml as it was left old version

* enable network policy by default

* remove unneeded task

* Fix kubelet calico settings

* fix when statement

* switch back to node-kubeconfig.yaml
This commit is contained in:
Aivars Sterns 2018-08-23 17:17:18 +03:00 committed by Matthew Mosesohn
parent f453567cce
commit 23fd3461bc
27 changed files with 399 additions and 252 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -95,7 +95,7 @@ Supported Components
- [docker](https://www.docker.com/) v17.03 (see note)
- [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2)
- Network Plugin
- [calico](https://github.com/projectcalico/calico) v2.6.8
- [calico](https://github.com/projectcalico/calico) v3.1.3
- [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
- [cilium](https://github.com/cilium/cilium) v1.2.0
- [contiv](https://github.com/contiv/install) v1.1.7

9
docs/calico.md
View file

@ -1,6 +1,13 @@
Calico
===========
---
**N.B. Version 2.6.5 upgrade to 3.1.1 is upgrading etcd store to etcdv3**
If you create automated backups of etcdv2 please switch for creating etcdv3 backups, as kubernetes and calico now uses etcdv3
After migration you can check `/tmp/calico_upgrade/` directory for converted items to etcdv3.
**PLEASE TEST upgrade before upgrading production cluster.**
---
Check if the calico-node container is running
```
@ -86,7 +93,7 @@ To do so you can deploy BGP route reflectors and peer `calico-node` with them as
recommended here:
* https://hub.docker.com/r/calico/routereflector/
* http://docs.projectcalico.org/v2.0/reference/private-cloud/l3-interconnect-fabric
* https://docs.projectcalico.org/v3.1/reference/private-cloud/l3-interconnect-fabric
You need to edit your inventory and add:

3
inventory/sample/group_vars/k8s-cluster.yml
View file

@ -83,9 +83,6 @@ kube_network_plugin: calico
# weave_no_masq_local: true
# weave_extra_args: ~
# Enable kubernetes network policies
enable_network_policy: false
# Kubernetes internal network for services, unused block of space.
kube_service_addresses: 10.233.0.0/18

10
roles/download/defaults/main.yml
View file

@ -32,11 +32,11 @@ kubeadm_version: "{{ kube_version }}"
etcd_version: v3.2.18
# TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
# after migration to container download
calico_version: "v2.6.8"
calico_ctl_version: "v1.6.3"
calico_cni_version: "v1.11.4"
calico_policy_version: "v1.0.3"
calico_rr_version: "v0.4.2"
calico_version: "v3.1.3"
calico_ctl_version: "v3.1.3"
calico_cni_version: "v3.1.3"
calico_policy_version: "v3.1.3"
calico_rr_version: "v0.6.1"
flannel_version: "v0.10.0"
flannel_cni_version: "v0.3.0"
vault_version: 0.10.1

12
roles/kubernetes-apps/network_plugin/calico/tasks/main.yml
View file

@ -7,5 +7,13 @@
resource: "{{item.item.type}}"
filename: "{{kube_config_dir}}/{{item.item.file}}"
state: "latest"
with_items: "{{ calico_node_manifests.results }}"
when: inventory_hostname == groups['kube-master'][0] and not item|skipped
with_items:
- "{{ calico_node_manifests.results }}"
when:
- inventory_hostname == groups['kube-master'][0] and not item|skipped
- name: "calico upgrade complete"
shell: "{{ bin_dir }}/calico-upgrade complete --no-prompts --apiconfigv1 /etc/calico/etcdv2.yml --apiconfigv3 /etc/calico/etcdv3.yml"
when:
- inventory_hostname == groups['kube-master'][0]
- calico_upgrade_needed|default(False)

23
roles/kubernetes-apps/policy_controller/calico/tasks/main.yml
View file

@ -2,20 +2,12 @@
- name: Set cert dir
set_fact:
calico_cert_dir: "{{ canal_cert_dir }}"
when: kube_network_plugin == 'canal'
when:
- kube_network_plugin == 'canal'
tags:
- facts
- canal
- name: Delete the old calico-policy-controller if it exist
kube:
name: calico-policy-controller
kubectl: "{{bin_dir}}/kubectl"
resource: rs
namespace: "kube-system"
state: absent
run_once: true
- name: Create calico-kube-controllers manifests
template:
src: "{{item.file}}.j2"
@ -26,7 +18,9 @@
- {name: calico-kube-controllers, file: calico-kube-cr.yml, type: clusterrole}
- {name: calico-kube-controllers, file: calico-kube-crb.yml, type: clusterrolebinding}
register: calico_kube_manifests
when: inventory_hostname == groups['kube-master'][0] and not item|skipped
when:
- inventory_hostname == groups['kube-master'][0]
- rbac_enabled or item.type not in rbac_resources
- name: Start of Calico kube controllers
kube:
@ -36,5 +30,8 @@
resource: "{{item.item.type}}"
filename: "{{kube_config_dir}}/{{item.item.file}}"
state: "latest"
with_items: "{{ calico_kube_manifests.results }}"
when: inventory_hostname == groups['kube-master'][0] and not item|skipped
with_items:
- "{{ calico_kube_manifests.results }}"
when:
- inventory_hostname == groups['kube-master'][0]
- not item|skipped

15
roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
View file

@ -6,8 +6,12 @@ metadata:
labels:
k8s-app: calico-kube-controllers
kubernetes.io/cluster-service: "true"
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
kubernetes.io/cluster-service: "true"
@ -45,17 +49,6 @@ spec:
value: "{{ calico_cert_dir }}/cert.crt"
- name: ETCD_KEY_FILE
value: "{{ calico_cert_dir }}/key.pem"
# Location of the Kubernetes API - this shouldn't need to be
# changed so long as it is used in conjunction with
# CONFIGURE_ETC_HOSTS="true".
- name: K8S_API
value: "https://kubernetes.default"
# Configure /etc/hosts within the container to resolve
# the kubernetes.default Service to the correct clusterIP
# using the environment provided by the kubelet.
# This removes the need for KubeDNS to resolve the Service.
- name: CONFIGURE_ETC_HOSTS
value: "true"
volumeMounts:
- mountPath: {{ calico_cert_dir }}
name: etcd-certs

8
roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2
View file

@ -12,6 +12,14 @@ rules:
- pods
- namespaces
- networkpolicies
- nodes
verbs:
- watch
- list
- apiGroups:
- networking.k8s.io
resources:
- networkpolicies
verbs:
- watch
- list

9
roles/kubernetes/kubeadm/tasks/main.yml
View file

@ -88,6 +88,15 @@
- is_kube_master
- kubeadm_discovery_address != kube_apiserver_endpoint
# FIXME(mattymo): Reconcile kubelet kubeconfig filename for both deploy modes
- name: Symlink kubelet kubeconfig for calico/canal
file:
src: "{{ kube_config_dir }}/kubelet.conf"
dest: "{{ kube_config_dir }}/node-kubeconfig.yaml"
state: link
force: yes
when: kube_network_plugin in ['calico','canal']
- name: Restart all kube-proxy pods to ensure that they load the new configmap
shell: "{{ bin_dir }}/kubectl delete pod -n kube-system -l k8s-app=kube-proxy"
delegate_to: "{{groups['kube-master']|first}}"

1
roles/kubernetes/node/templates/kubelet-container.j2
View file

@ -22,6 +22,7 @@
-v {{ docker_daemon_graph }}:{{ docker_daemon_graph }}:rw \
-v /var/log:/var/log:rw \
-v /var/lib/kubelet:/var/lib/kubelet:shared \
-v /var/lib/calico:/var/lib/calico:shared \
-v /var/lib/cni:/var/lib/cni:shared \
-v /var/run:/var/run:rw \
{# we can run into issues with double mounting /var/lib/kubelet #}

7
roles/kubernetes/node/templates/kubelet.rkt.service.j2
View file

@ -37,6 +37,10 @@ ExecStart=/usr/bin/rkt run \
--volume etc-cni,kind=host,source=/etc/cni,readOnly=true \
--volume opt-cni,kind=host,source=/opt/cni,readOnly=true \
--volume var-lib-cni,kind=host,source=/var/lib/cni,readOnly=false \
{% endif %}
{% if kube_network_plugin in ["calico", "canal"] %}
--volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=false \
{% endif %}
{# we can run into issues with double mounting /var/lib/kubelet #}
{# surely there's a better way to do this #}
{% if '/var/lib/kubelet' not in kubelet_flexvolumes_plugins_dir %}
@ -55,6 +59,9 @@ ExecStart=/usr/bin/rkt run \
--mount volume=etc-cni,target=/etc/cni \
--mount volume=opt-cni,target=/opt/cni \
--mount volume=var-lib-cni,target=/var/lib/cni \
{% if kube_network_plugin in ["calico", "canal"] %}
--mount volume=var-lib-calico,target=/var/lib/calico \
{% endif %}
{% endif %}
--mount volume=os-release,target=/etc/os-release \
--mount volume=dns,target=/etc/resolv.conf \

1
roles/kubernetes/preinstall/tasks/main.yml
View file

@ -89,6 +89,7 @@
with_items:
- "/etc/cni/net.d"
- "/opt/cni/bin"
- "/var/lib/calico"
when:
- kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium"]
- inventory_hostname in groups['k8s-cluster']

2
roles/kubernetes/preinstall/tasks/set_resolv_facts.yml
View file

@ -98,7 +98,7 @@
{%- elif dns_mode == 'coredns_dual' and not dns_early|bool -%}
{{ [ skydns_server ] + [ skydns_server_secondary ] + upstream_dns_servers|default([]) }}
{%- elif dns_mode == 'manual' and not dns_early|bool -%}
{{ [ manual_dns_server ] + upstream_dns_servers|default([]) }}
{{ ( manual_dns_server.split(',') | list) + upstream_dns_servers|default([]) }}
{%- elif dns_early|bool -%}
{{ upstream_dns_servers|default([]) }}
{%- else -%}

2
roles/kubespray-defaults/defaults/main.yaml
View file

@ -187,7 +187,7 @@ dashboard_enabled: true
efk_enabled: false
helm_enabled: false
registry_enabled: false
enable_network_policy: false
enable_network_policy: true
local_volume_provisioner_enabled: "{{ local_volumes_enabled | default('false') }}"
persistent_volumes_enabled: false
cephfs_provisioner_enabled: false

11
roles/network_plugin/calico/defaults/main.yml
View file

@ -2,9 +2,12 @@
# Enables Internet connectivity from containers
nat_outgoing: true
#add default ippool name
calico_pool_name: "default-pool"
# Use IP-over-IP encapsulation across hosts
ipip: true
ipip_mode: always # change to "cross-subnet" if you only want ipip encapsulation on traffic going across subnets
ipip_mode: Always # change to "CrossSubnet" if you only want ipip encapsulation on traffic going across subnets
# Set to true if you want your calico cni binaries to overwrite the
# ones from hyperkube while leaving other cni plugins intact.
@ -37,6 +40,9 @@ calico_felix_prometheusmetricsport: 9091
calico_felix_prometheusgometricsenabled: "true"
calico_felix_prometheusprocessmetricsenabled: "true"
### check latest version https://github.com/projectcalico/calico-upgrade/releases
calico_upgrade_version: v1.0.5
# Should calico ignore kernel's RPF check setting,
# see https://github.com/projectcalico/felix/blob/ab8799eaea66627e5db7717e62fca61fd9c08646/python/calico/felix/config.py#L198
calico_node_ignorelooserpf: false
@ -48,3 +54,6 @@ calico_node_ignorelooserpf: false
# calico_ip_auto_method: "interface=eth.*"
calico_baremetal_nodename: "{{ inventory_hostname }}"
### do not enable this, this is detected in scope of tasks, this is just a default value
calico_upgrade_needed: false

37
roles/network_plugin/calico/tasks/check.yml Normal file
View file

@ -0,0 +1,37 @@
---
- name: "Check vars defined correctly"
assert:
that:
- "calico_pool_name is defined"
- "calico_pool_name | match('^[a-zA-Z0-9-_\\\\.]{2,63}$')"
- "ipip_mode is defined"
- "ipip_mode in ['Always', 'CrossSubnet', 'Never']"
msg: "Check variable definitions seems something is wrong"
run_once: yes
- name: "Get current version of calico cluster version"
shell: "{{ bin_dir }}/calicoctl version | grep 'Cluster Version' | awk '{ print $3}'"
register: calico_version_on_server
run_once: yes
delegate_to: "{{ groups['kube-master'][0] }}"
- name: "Determine if calico upgrade is needed"
block:
- name: "Check that calico version is enought for upgrade"
assert:
that:
- calico_version_on_server.stdout|version_compare('v2.6.5', '>=')
msg: "Your version of calico is not fresh enough for upgrade"
- name: "Set upgrade flag when version needs to be updated"
set_fact:
calico_upgrade_needed: True
when:
- calico_version_on_server.stdout|version_compare('v2.6.5', '>=')
- calico_version_on_server.stdout|version_compare('v3.0.0', '<')
when:
- 'calico_version_on_server.stdout is defined'
- 'calico_version_on_server.stdout != ""'
- inventory_hostname == groups['kube-master'][0]
run_once: yes

189
roles/network_plugin/calico/tasks/install.yml Normal file
View file

@ -0,0 +1,189 @@
---
- name: Calico | Write Calico cni config
template:
src: "cni-calico.conflist.j2"
dest: "/etc/cni/net.d/10-calico.conflist"
owner: kube
- name: Calico | Create calico certs directory
file:
dest: "{{ calico_cert_dir }}"
state: directory
mode: 0750
owner: root
group: root
- name: Calico | Link etcd certificates for calico-node
file:
src: "{{ etcd_cert_dir }}/{{ item.s }}"
dest: "{{ calico_cert_dir }}/{{ item.d }}"
state: hard
force: yes
with_items:
- {s: "ca.pem", d: "ca_cert.crt"}
- {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
- {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
- name: Calico | Install calicoctl container script
template:
src: calicoctl-container.j2
dest: "{{ bin_dir }}/calicoctl"
mode: 0755
owner: root
group: root
changed_when: false
- name: Calico | Copy cni plugins from hyperkube
command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/"
register: cni_task_result
until: cni_task_result.rc == 0
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
changed_when: false
tags:
- hyperkube
- upgrade
- name: Calico | Copy cni plugins from calico/cni container
command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp /opt/cni/bin/* /cnibindir/'"
register: cni_task_result
until: cni_task_result.rc == 0
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
changed_when: false
when:
- "overwrite_hyperkube_cni|bool"
tags:
- hyperkube
- upgrade
- name: Calico | Set cni directory permissions
file:
path: /opt/cni/bin
state: directory
owner: kube
recurse: true
mode: 0755
- name: Calico | wait for etcd
uri:
url: "{{ etcd_access_addresses.split(',') | first }}/health"
validate_certs: no
client_cert: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
client_key: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
register: result
until: result.status == 200 or result.status == 401
retries: 10
delay: 5
run_once: true
- name: Calico | Check if calico network pool has already been configured
shell: >
{{ bin_dir }}/calicoctl get ippool | grep -w "{{ kube_pods_subnet }}" | wc -l
register: calico_conf
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
delegate_to: "{{ groups['kube-master'][0] }}"
run_once: true
- name: Calico | Configure calico network pool
shell: >
echo "
{ "kind": "IPPool",
"apiVersion": "projectcalico.org/v3",
"metadata": {
"name": "{{ calico_pool_name }}",
},
"spec": {
"cidr": "{{ kube_pods_subnet }}",
"ipipMode": "{{ ipip_mode|capitalize }}",
"natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl create -f -
run_once: true
delegate_to: "{{ groups['kube-master'][0] }}"
when:
- 'calico_conf.stdout == "0"'
- name: "Determine nodeToNodeMesh needed state"
set_fact:
nodeToNodeMeshEnabled: "false"
when:
- peer_with_router|default(false) or peer_with_calico_rr|default(false)
- inventory_hostname in groups['k8s-cluster']
run_once: yes
- name: Calico | Set global as_num
shell: >
echo '
{ "kind": "BGPConfiguration",
"apiVersion": "projectcalico.org/v3",
"metadata": {
"name": "default",
},
"spec": {
"logSeverityScreen": "Info",
"nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled|default('true') }} ,
"asNumber": {{ global_as_num }} }} ' | {{ bin_dir }}/calicoctl --skip-exists create -f -
run_once: true
delegate_to: "{{ groups['kube-master'][0] }}"
- name: Calico | Configure peering with router(s)
shell: >
echo '{
"apiVersion": "projectcalico.org/v3",
"kind": "bgpPeer",
"metadata": {
"name": "{{ inventory_hostname }}-bgp"
},
"spec": {
"asNumber": "{{ item.as }}",
"node": "{{ inventory_hostname }}",
"scope": "node",
"peerIP": "{{ item.router_id }}"
}}' | {{ bin_dir }}/calicoctl create --skip-exists -f -
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
with_items:
- "{{ peers|default([]) }}"
when:
- peer_with_router|default(false)
- inventory_hostname in groups['k8s-cluster']
- name: Calico | Configure peering with route reflectors
shell: >
echo '{
"apiVersion": "projectcalico.org/v3",
"kind": "bgpPeer",
"metadata": {
"name": "{{ inventory_hostname }}"
},
"spec": {
"asNumber": "{{ local_as | default(global_as_num)}}",
"scope": "node",
"node": "{{ inventory_hostname }}",
"peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}"
}}' | {{ bin_dir }}/calicoctl create --skip-exists -f -
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
with_items:
- "{{ groups['calico-rr'] | default([]) }}"
when:
- (peer_with_calico_rr|default(false)
- inventory_hostname in groups['k8s-cluster']
- hostvars[item]['cluster_id'] == cluster_id)
- name: Calico | Create calico manifests
template:
src: "{{item.file}}.j2"
dest: "{{kube_config_dir}}/{{item.file}}"
with_items:
- {name: calico-config, file: calico-config.yml, type: cm}
- {name: calico-node, file: calico-node.yml, type: ds}
- {name: calico, file: calico-node-sa.yml, type: sa}
- {name: calico, file: calico-cr.yml, type: clusterrole}
- {name: calico, file: calico-crb.yml, type: clusterrolebinding}
register: calico_node_manifests
when:
- inventory_hostname in groups['kube-master']
- rbac_enabled or item.type not in rbac_resources

197
roles/network_plugin/calico/tasks/main.yml
View file

@ -1,193 +1,12 @@
---
- name: Calico | Disable calico-node service if it exists
service:
name: calico-node
state: stopped
enabled: yes
failed_when: false
- include_tasks: check.yml
- name: Calico | Get kubelet hostname
shell: >-
{{ bin_dir }}/kubectl get node -o custom-columns='NAME:.metadata.name,INTERNAL-IP:.status.addresses[?(@.type=="InternalIP")].address'
| egrep "{{ ansible_all_ipv4_addresses | join('$|') }}$" | cut -d" " -f1
register: calico_kubelet_name
delegate_to: "{{ groups['kube-master'][0] }}"
when: cloud_provider is defined
- include_tasks: pre.yml
- name: Calico | Write Calico cni config
template:
src: "cni-calico.conflist.j2"
dest: "/etc/cni/net.d/10-calico.conflist"
owner: kube
- name: Calico | Create calico certs directory
file:
dest: "{{ calico_cert_dir }}"
state: directory
mode: 0750
owner: root
group: root
- name: Calico | Link etcd certificates for calico-node
file:
src: "{{ etcd_cert_dir }}/{{ item.s }}"
dest: "{{ calico_cert_dir }}/{{ item.d }}"
state: hard
force: yes
with_items:
- {s: "ca.pem", d: "ca_cert.crt"}
- {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
- {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
- name: Calico | Install calicoctl container script
template:
src: calicoctl-container.j2
dest: "{{ bin_dir }}/calicoctl"
mode: 0755
owner: root
group: root
changed_when: false
- name: Calico | Copy cni plugins from hyperkube
command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -rf /opt/cni/bin/. /cnibindir/"
register: cni_task_result
until: cni_task_result.rc == 0
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
changed_when: false
tags:
- hyperkube
- upgrade
- name: Calico | Copy cni plugins from calico/cni container
command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp /opt/cni/bin/* /cnibindir/'"
register: cni_task_result
until: cni_task_result.rc == 0
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
changed_when: false
when: overwrite_hyperkube_cni|bool
tags:
- hyperkube
- upgrade
- name: Calico | Set cni directory permissions
file:
path: /opt/cni/bin
state: directory
owner: kube
recurse: true
mode: 0755
- name: Calico | wait for etcd
uri:
url: "{{ etcd_access_addresses.split(',') | first }}/health"
validate_certs: no
client_cert: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
client_key: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
register: result
until: result.status == 200 or result.status == 401
retries: 10
delay: 5
run_once: true
- name: Calico | Check if calico network pool has already been configured
command: |-
curl \
--cacert {{ etcd_cert_dir }}/ca.pem \
--cert {{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem \
--key {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem \
{{ etcd_access_addresses.split(',') | first }}/v2/keys/calico/v1/ipam/v4/pool
register: calico_conf
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
run_once: true
changed_when: false
- name: Calico | Configure calico network pool
shell: >
echo '{
"kind": "ipPool",
"spec": {"disabled": false, "ipip": {"enabled": {{ ipip }}, "mode": "{{ ipip_mode }}"},
"nat-outgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }}},
"apiVersion": "v1",
"metadata": {"cidr": "{{ kube_pods_subnet }}"}
}'
| {{ bin_dir }}/calicoctl apply -f -
environment:
NO_DEFAULT_POOLS: true
run_once: true
when: ("Key not found" in calico_conf.stdout or "nodes" not in calico_conf.stdout)
- name: Calico | Get calico configuration from etcd
command: |-
curl \
--cacert {{ etcd_cert_dir }}/ca.pem \
--cert {{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem \
--key {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem \
{{ etcd_access_addresses.split(',') | first }}/v2/keys/calico/v1/ipam/v4/pool
register: calico_pools_raw
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
run_once: true
- set_fact:
calico_pools: "{{ calico_pools_raw.stdout | from_json }}"
run_once: true
- name: Calico | Set global as_num
command: "{{ bin_dir}}/calicoctl config set asNumber {{ global_as_num }}"
run_once: true
- name: Calico | Disable node mesh
shell: "{{ bin_dir }}/calicoctl config set nodeToNodeMesh off"
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
when: ((peer_with_router|default(false) or peer_with_calico_rr|default(false))
and inventory_hostname in groups['k8s-cluster'])
run_once: true
- name: Calico | Configure peering with router(s)
shell: >
echo '{
"kind": "bgpPeer",
"spec": {"asNumber": "{{ item.as }}"},
"apiVersion": "v1",
"metadata": {"node": "{{ inventory_hostname }}", "scope": "node", "peerIP": "{{ item.router_id }}"}
}'
| {{ bin_dir }}/calicoctl create --skip-exists -f -
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
with_items: "{{ peers|default([]) }}"
when: peer_with_router|default(false) and inventory_hostname in groups['k8s-cluster']
- name: Calico | Configure peering with route reflectors
shell: >
echo '{
"kind": "bgpPeer",
"spec": {"asNumber": "{{ local_as | default(global_as_num)}}"},
"apiVersion": "v1",
"metadata": {"node": "{{ inventory_hostname }}",
"scope": "node",
"peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}"}
}'
| {{ bin_dir }}/calicoctl create --skip-exists -f -
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
with_items: "{{ groups['calico-rr'] | default([]) }}"
when: (peer_with_calico_rr|default(false) and inventory_hostname in groups['k8s-cluster']
and hostvars[item]['cluster_id'] == cluster_id)
- name: Calico | Create calico manifests
template:
src: "{{item.file}}.j2"
dest: "{{kube_config_dir}}/{{item.file}}"
with_items:
- {name: calico-config, file: calico-config.yml, type: cm}
- {name: calico-node, file: calico-node.yml, type: ds}
- {name: calico, file: calico-node-sa.yml, type: sa}
- {name: calico, file: calico-cr.yml, type: clusterrole}
- {name: calico, file: calico-crb.yml, type: clusterrolebinding}
register: calico_node_manifests
- include_tasks: upgrade.yml
when:
- inventory_hostname in groups['kube-master']
- calico_upgrade_needed
run_once: yes
delegate_to: "{{ groups['kube-master'][0] }}"
- include_tasks: install.yml

16
roles/network_plugin/calico/tasks/pre.yml Normal file
View file

@ -0,0 +1,16 @@
---
- name: Calico | Disable calico-node service if it exists
service:
name: calico-node
state: stopped
enabled: no
failed_when: false
- name: Calico | Get kubelet hostname
shell: >-
{{ bin_dir }}/kubectl get node -o custom-columns='NAME:.metadata.name,INTERNAL-IP:.status.addresses[?(@.type=="InternalIP")].address'
| egrep "{{ ansible_all_ipv4_addresses | join('$|') }}$" | cut -d" " -f1
register: calico_kubelet_name
delegate_to: "{{ groups['kube-master'][0] }}"
when:
- "cloud_provider is defined"

26
roles/network_plugin/calico/tasks/upgrade.yml Normal file
View file

@ -0,0 +1,26 @@
---
- name: "Download calico-upgrade tool (force version)"
get_url:
url: "https://github.com/projectcalico/calico-upgrade/releases/download/{{ calico_upgrade_version }}/calico-upgrade"
dest: "{{ bin_dir }}/calico-upgrade"
mode: 0755
owner: root
group: root
force: yes
- name: "Create etcdv2 and etcdv3 calicoApiConfig"
template:
src: "{{ item }}-store.yml.j2"
dest: "/etc/calico/{{ item }}.yml"
with_items:
- "etcdv2"
- "etcdv3"
- name: "Tests data migration (dry-run)"
shell: "{{ bin_dir }}/calico-upgrade dry-run --output-dir=/tmp --apiconfigv1 /etc/calico/etcdv2.yml --apiconfigv3 /etc/calico/etcdv3.yml"
register: calico_upgrade_test_data
failed_when: '"Successfully" not in calico_upgrade_test_data.stdout'
- name: "If test migration is success continue with calico data real migration"
shell: "{{ bin_dir }}/calico-upgrade start --no-prompts --apiconfigv1 /etc/calico/etcdv2.yml --apiconfigv3 /etc/calico/etcdv3.yml --output-dir=/tmp/calico_upgrade"
register: calico_upgrade_migration_data

12
roles/network_plugin/calico/templates/calico-node.yml.j2
View file

@ -77,12 +77,6 @@ spec:
# Set Felix logging to "info"
- name: FELIX_LOGSEVERITYSCREEN
value: "info"
# Disable autocreation of pools
- name: CALICO_NO_DEFAULT_POOLS
value: "true"
# Enable libnetwork
- name: CALICO_LIBNETWORK_ENABLED
value: "true"
# Set MTU for tunnel device used if ipip is enabled
{% if calico_mtu is defined %}
- name: FELIX_IPINIPMTU
@ -158,6 +152,8 @@ spec:
readOnly: true
- mountPath: /var/run/calico
name: var-run-calico
- mountPath: /var/lib/calico
name: var-lib-calico
readOnly: false
- mountPath: /calico-secrets
name: etcd-certs
@ -169,6 +165,9 @@ spec:
- name: var-run-calico
hostPath:
path: /var/run/calico
- name: var-lib-calico
hostPath:
path: /var/lib/calico
# Used to install CNI.
- name: cni-bin-dir
hostPath:
@ -184,4 +183,3 @@ spec:
rollingUpdate:
maxUnavailable: {{ serial | default('20%') }}
type: RollingUpdate

9
roles/network_plugin/calico/templates/etcdv2-store.yml.j2 Normal file
View file

@ -0,0 +1,9 @@
apiVersion: v1
kind: calicoApiConfig
metadata:
spec:
datastoreType: "etcdv2"
etcdEndpoints: "{{ etcd_access_addresses }}"
etcdKeyFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
etcdCertFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
etcdCACertFile: "{{ etcd_cert_dir }}/ca.pem"

9
roles/network_plugin/calico/templates/etcdv3-store.yml.j2 Normal file
View file

@ -0,0 +1,9 @@
apiVersion: projectcalico.org/v3
kind: CalicoAPIConfig
metadata:
spec:
datastoreType: "etcdv3"
etcdEndpoints: "{{ etcd_access_addresses }}"
etcdKeyFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
etcdCertFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
etcdCACertFile: "{{ etcd_cert_dir }}/ca.pem"

6
roles/network_plugin/canal/templates/canal-node.yaml.j2
View file

@ -28,6 +28,9 @@ spec:
- name: lib-modules
hostPath:
path: /lib/modules
- name: var-lib-calico
hostPath:
path: /var/lib/calico
- name: var-run-calico
hostPath:
path: /var/run/calico
@ -202,6 +205,9 @@ spec:
- mountPath: /var/run/calico
name: var-run-calico
readOnly: false
- mountPath: /var/lib/calico
name: var-lib-calico
readOnly: false
- name: "canal-certs"
mountPath: "{{ canal_cert_dir }}"
readOnly: true

1
roles/reset/tasks/main.yml
View file

@ -136,6 +136,7 @@
- "{{ bin_dir }}/kubeadm"
- "{{ bin_dir }}/helm"
- "{{ bin_dir }}/calicoctl"
- "{{ bin_dir }}/calico-upgrade"
- "{{ bin_dir }}/weave"
- /var/lib/rkt
- /etc/vault

20
upgrade-cluster.yml
View file

@ -78,10 +78,19 @@
- { role: kubernetes/master, tags: master }
- { role: kubernetes/client, tags: client }
- { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
- { role: network_plugin, tags: network }
- { role: upgrade/post-upgrade, tags: post-upgrade }
environment: "{{proxy_env}}"
#Upgrade calico on all masters and nodes
- hosts: kube-master:kube-node
any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
serial: "{{ serial | default('20%') }}"
roles:
- { role: kubespray-defaults}
- { role: network_plugin, tags: network }
- { role: kubernetes-apps/network_plugin, tags: network }
- { role: kubernetes-apps/policy_controller, tags: policy-controller }
#Finally handle worker upgrades, based on given batch size
- hosts: kube-node:!kube-master
any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
@ -90,7 +99,6 @@
- { role: kubespray-defaults}
- { role: upgrade/pre-upgrade, tags: pre-upgrade }
- { role: kubernetes/node, tags: node }
- { role: network_plugin, tags: network }
- { role: upgrade/post-upgrade, tags: post-upgrade }
- { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" }
- { role: kubespray-defaults}
@ -102,14 +110,6 @@
- { role: kubespray-defaults}
- { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" }
- hosts: kube-master
any_errors_fatal: true
roles:
- { role: kubespray-defaults}
- { role: kubernetes-apps/network_plugin, tags: network }
- { role: kubernetes-apps/policy_controller, tags: policy-controller }
- { role: kubernetes/client, tags: client }
- hosts: calico-rr
any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
roles: