C12s/c12s-kubespray
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Move calico to daemonset (#1605)

Browse source 
* Drop legacy calico logic

* add calico as a daemonset
This commit is contained in:
Matthew Mosesohn 2017-09-04 11:29:51 +03:00 committed by GitHub
parent a3e6896a43
commit 77602dbb93
17 changed files with 262 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitlab-ci.yml
View file

@ -259,6 +259,7 @@ before_script:
# Test matrix. Leave the comments for markup scripts.
.coreos_calico_sep_variables: &coreos_calico_sep_variables
# stage: deploy-gce-part1
AUTHORIZATION_MODES: "{ 'authorization_modes': [ 'RBAC' ] }"
KUBE_NETWORK_PLUGIN: calico
CLOUD_IMAGE: coreos-stable-1465-6-0-v20170817
CLOUD_REGION: us-west1-b

10
roles/kubernetes-apps/network_plugin/calico/tasks/main.yml Normal file
View file

@ -0,0 +1,10 @@
---
- name: Start Calico resources
kube:
name: "{{item.item.name}}"
namespace: "{{ system_namespace }}"
kubectl: "{{bin_dir}}/kubectl"
resource: "{{item.item.type}}"
filename: "{{kube_config_dir}}/{{item.item.file}}"
state: "{{item.changed | ternary('latest','present') }}"
with_items: "{{ calico_node_manifests.results }}"

3
roles/kubernetes-apps/network_plugin/meta/main.yml
View file

@ -1,5 +1,8 @@
---
dependencies:
- role: kubernetes-apps/network_plugin/calico
when: kube_network_plugin == 'calico'
tags: calico
- role: kubernetes-apps/network_plugin/canal
when: kube_network_plugin == 'canal'
tags: canal

5
roles/kubernetes/node/templates/kubelet.docker.service.j2
View file

@ -1,13 +1,8 @@
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
After=docker.service docker.socket calico-node.service
Wants=docker.socket calico-node.service
{% else %}
After=docker.service
Wants=docker.socket
{% endif %}
[Service]
EnvironmentFile={{kube_config_dir}}/kubelet.env

5
roles/kubernetes/node/templates/kubelet.host.service.j2
View file

@ -1,13 +1,8 @@
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
After=docker.service docker.socket calico-node.service
Wants=docker.socket calico-node.service
{% else %}
After=docker.service
Wants=docker.socket
{% endif %}
[Service]
EnvironmentFile={{kube_config_dir}}/kubelet.env

5
roles/kubernetes/node/templates/kubelet.rkt.service.j2
View file

@ -1,12 +1,7 @@
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
After=calico-node.service
Wants=network.target calico-node.service
{% else %}
Wants=network.target
{% endif %}
[Service]
Restart=on-failure

5
roles/network_plugin/calico/defaults/main.yml
View file

@ -34,3 +34,8 @@ calicoctl_cpu_requests: 50m
# Should calico ignore kernel's RPF check setting,
# see https://github.com/projectcalico/felix/blob/ab8799eaea66627e5db7717e62fca61fd9c08646/python/calico/felix/config.py#L198
calico_node_ignorelooserpf: false
rbac_resources:
- sa
- clusterrole
- clusterrolebinding

127
roles/network_plugin/calico/tasks/main.yml
View file

@ -1,9 +1,10 @@
---
- name: Calico | Check calicoctl version
run_once: true
set_fact:
legacy_calicoctl: "{{ calicoctl_image_tag | version_compare('v1.0.0', '<') }}"
tags: facts
- name: Calico | Disable calico-node service if it exists
service:
name: calico-node
state: stopped
enabled: yes
failed_when: false
- name: Calico | Write Calico cni config
template:
@ -38,7 +39,6 @@
owner: root
group: root
changed_when: false
notify: restart calico-node
- name: Calico | Copy cni plugins from hyperkube
command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -ac /opt/cni/bin/ /cnibindir/"
@ -103,38 +103,7 @@
environment:
NO_DEFAULT_POOLS: true
run_once: true
when: not legacy_calicoctl and
("Key not found" in calico_conf.stdout or "nodes" not in calico_conf.stdout)
- name: Calico (old) | Define ipip pool argument
run_once: true
set_fact:
ipip_arg: "--ipip"
when: (legacy_calicoctl and ipip )
tags: facts
- name: Calico (old) | Define nat-outgoing pool argument
run_once: true
set_fact:
nat_arg: "--nat-outgoing"
when: (legacy_calicoctl and
nat_outgoing|default(false) and not peer_with_router|default(false))
tags: facts
- name: Calico (old) | Define calico pool task name
run_once: true
set_fact:
pool_task_name: "with options {{ ipip_arg|default('') }} {{ nat_arg|default('') }}"
when: (legacy_calicoctl and ipip_arg|default(false) or nat_arg|default(false))
tags: facts
- name: Calico (old) | Configure calico network pool {{ pool_task_name|default('') }}
command: "{{ bin_dir}}/calicoctl pool add {{ kube_pods_subnet }} {{ ipip_arg|default('') }} {{ nat_arg|default('') }}"
environment:
NO_DEFAULT_POOLS: true
run_once: true
when: legacy_calicoctl and
("Key not found" in calico_conf.stdout or "nodes" not in calico_conf.stdout)
when: ("Key not found" in calico_conf.stdout or "nodes" not in calico_conf.stdout)
- name: Calico | Get calico configuration from etcd
command: |-
@ -162,52 +131,11 @@
- name: Calico | Set global as_num
command: "{{ bin_dir}}/calicoctl config set asNumber {{ global_as_num }}"
run_once: true
when: not legacy_calicoctl
- name: Calico (old) | Set global as_num
command: "{{ bin_dir}}/calicoctl bgp default-node-as {{ global_as_num }}"
run_once: true
when: legacy_calicoctl
- name: Calico (old) | Write calico-node systemd init file
template:
src: calico-node.service.legacy.j2
dest: /etc/systemd/system/calico-node.service
when: legacy_calicoctl
notify: restart calico-node
- name: Calico | Write calico.env for systemd init file
template:
src: calico.env.j2
dest: /etc/calico/calico.env
when: not legacy_calicoctl
notify: restart calico-node
- name: Calico | Write calico-node systemd init file
template:
src: calico-node.service.j2
dest: /etc/systemd/system/calico-node.service
when: not legacy_calicoctl
notify: restart calico-node
- name: Calico | Restart calico-node if secrets changed
command: /bin/true
when: secret_changed|default(false) or etcd_secret_changed|default(false)
notify: restart calico-node
- meta: flush_handlers
- name: Calico | Enable calico-node
service:
name: calico-node
state: started
enabled: yes
- name: Calico | Disable node mesh
shell: "{{ bin_dir }}/calicoctl config set nodeToNodeMesh off"
when: ((peer_with_router|default(false) or peer_with_calico_rr|default(false))
and inventory_hostname in groups['k8s-cluster']
and not legacy_calicoctl)
and inventory_hostname in groups['k8s-cluster'])
run_once: true
- name: Calico | Configure peering with router(s)
@ -220,8 +148,7 @@
}'
| {{ bin_dir }}/calicoctl create --skip-exists -f -
with_items: "{{ peers|default([]) }}"
when: (not legacy_calicoctl and
peer_with_router|default(false) and inventory_hostname in groups['k8s-cluster'])
when: peer_with_router|default(false) and inventory_hostname in groups['k8s-cluster']
- name: Calico | Configure peering with route reflectors
shell: >
@ -235,26 +162,20 @@
}'
| {{ bin_dir }}/calicoctl create --skip-exists -f -
with_items: "{{ groups['calico-rr'] | default([]) }}"
when: (not legacy_calicoctl and
peer_with_calico_rr|default(false) and inventory_hostname in groups['k8s-cluster']
when: (peer_with_calico_rr|default(false) and inventory_hostname in groups['k8s-cluster']
and hostvars[item]['cluster_id'] == cluster_id)
- name: Calico (old) | Disable node mesh
shell: "{{ bin_dir }}/calicoctl bgp node-mesh off"
when: ((peer_with_router|default(false) or peer_with_calico_rr|default(false))
and inventory_hostname in groups['k8s-cluster']
and legacy_calicoctl)
run_once: true
- name: Calico (old) | Configure peering with router(s)
shell: "{{ bin_dir }}/calicoctl node bgp peer add {{ item.router_id }} as {{ item.as }}"
with_items: "{{ peers|default([]) }}"
when: (legacy_calicoctl and
peer_with_router|default(false) and inventory_hostname in groups['k8s-cluster'])
- name: Calico (old) | Configure peering with route reflectors
shell: "{{ bin_dir }}/calicoctl node bgp peer add {{ hostvars[item]['calico_rr_ip']|default(hostvars[item]['ip'])|default(hostvars[item]['ansible_default_ipv4.address']) }} as {{ local_as | default(global_as_num) }}"
with_items: "{{ groups['calico-rr'] | default([]) }}"
when: (legacy_calicoctl and
peer_with_calico_rr|default(false) and inventory_hostname in groups['k8s-cluster']
and hostvars[item]['cluster_id'] == cluster_id)
- name: Calico | Create calico manifests
template:
src: "{{item.file}}.j2"
dest: "{{kube_config_dir}}/{{item.file}}"
with_items:
- {name: calico-config, file: calico-config.yml, type: cm}
- {name: calico-node, file: calico-node.yml, type: ds}
- {name: calico, file: calico-node-sa.yml, type: sa}
- {name: calico, file: calico-cr.yml, type: clusterrole}
- {name: calico, file: calico-crb.yml, type: clusterrolebinding}
register: calico_node_manifests
when:
- inventory_hostname in groups['kube-master']
- rbac_enabled or item.type not in rbac_resources

19
roles/network_plugin/calico/templates/calico-config.yml.j2 Normal file
View file

@ -0,0 +1,19 @@
kind: ConfigMap
apiVersion: v1
metadata:
name: calico-config
namespace: {{ system_namespace }}
data:
etcd_endpoints: "{{ etcd_access_endpoint }}"
etcd_ca: "/calico-secrets/ca_cert.crt"
etcd_cert: "/calico-secrets/cert.crt"
etcd_key: "/calico-secrets/key.pem"
{% if calico_network_backend is defined and calico_network_backend == 'none' %}
cluster_type: "kubespray"
{%- else %}
cluster_type: "kubespray,bgp"
{% endif %}
calico_backend: "bird"
{%- if inventory_hostname in groups['k8s-cluster'] and peer_with_router|default(false) %}
as: "{{ local_as }}"
{% endif -%}

13
roles/network_plugin/calico/templates/calico-cr.yml.j2 Normal file
View file

@ -0,0 +1,13 @@
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: calico-node
namespace: {{ system_namespace }}
rules:
- apiGroups: [""]
resources:
- pods
- nodes
verbs:
- get

13
roles/network_plugin/calico/templates/calico-crb.yml.j2 Normal file
View file

@ -0,0 +1,13 @@
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: calico-node
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: calico-node
subjects:
- kind: ServiceAccount
name: calico-node
namespace: {{ system_namespace }}

8
roles/network_plugin/calico/templates/calico-node-sa.yml.j2 Normal file
View file

@ -0,0 +1,8 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: calico-node
namespace: {{ system_namespace }}
labels:
kubernetes.io/cluster-service: "true"

41
roles/network_plugin/calico/templates/calico-node.service.j2
View file

@ -1,41 +0,0 @@
[Unit]
Description=calico-node
After=docker.service
Requires=docker.service
[Service]
EnvironmentFile=/etc/calico/calico.env
ExecStartPre=-{{ docker_bin_dir }}/docker rm -f calico-node
ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \
--name=calico-node \
-e HOSTNAME=${CALICO_HOSTNAME} \
-e IP=${CALICO_IP} \
-e IP6=${CALICO_IP6} \
-e CLUSTER_TYPE=${CLUSTER_TYPE} \
-e CALICO_NETWORKING_BACKEND=${CALICO_NETWORKING_BACKEND} \
-e FELIX_DEFAULTENDPOINTTOHOSTACTION={{ calico_endpoint_to_host_action|default('RETURN') }} \
-e AS=${CALICO_AS} \
-e NO_DEFAULT_POOLS=${CALICO_NO_DEFAULT_POOLS} \
-e CALICO_LIBNETWORK_ENABLED=${CALICO_LIBNETWORK_ENABLED} \
-e ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \
-e ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \
-e ETCD_CERT_FILE=${ETCD_CERT_FILE} \
-e ETCD_KEY_FILE=${ETCD_KEY_FILE} \
{% if calico_node_ignorelooserpf %}
-e FELIX_IGNORELOOSERPF=true \
{% endif %}
-v /var/log/calico:/var/log/calico \
-v /run/docker/plugins:/run/docker/plugins \
-v /lib/modules:/lib/modules \
-v /var/run/calico:/var/run/calico \
-v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \
--memory={{ calico_node_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_node_cpu_limit|regex_replace('m', '') }} \
{{ calico_node_image_repo }}:{{ calico_node_image_tag }}
Restart=always
RestartSec=10s
ExecStop=-{{ docker_bin_dir }}/docker stop calico-node
[Install]
WantedBy=multi-user.target

19
roles/network_plugin/calico/templates/calico-node.service.legacy.j2
View file

@ -1,19 +0,0 @@
[Unit]
Description=Calico per-node agent
Documentation=https://github.com/projectcalico/calico-docker
After=docker.service docker.socket
Wants=docker.socket
[Service]
User=root
PermissionsStartOnly=true
{% if inventory_hostname in groups['k8s-cluster'] and peer_with_router|default(false)%}
ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --as={{ local_as }} --detach=false --node-image={{ calico_node_image_repo }}:{{ calico_node_image_tag }}
{% else %}
ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --detach=false --node-image={{ calico_node_image_repo }}:{{ calico_node_image_tag }}
{% endif %}
Restart=always
RestartSec=10s
[Install]
WantedBy=multi-user.target

166
roles/network_plugin/calico/templates/calico-node.yml.j2 Normal file
View file

@ -0,0 +1,166 @@
---
# This manifest installs the calico/node container, as well
# as the Calico CNI plugins and network config on
# each master and worker node in a Kubernetes cluster.
kind: DaemonSet
apiVersion: extensions/v1beta1
metadata:
name: calico-node
namespace: {{ system_namespace }}
labels:
k8s-app: calico-node
spec:
selector:
matchLabels:
k8s-app: calico-node
template:
metadata:
labels:
k8s-app: calico-node
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
kubespray.etcd-cert/serial: "{{ etcd_node_cert_serial }}"
spec:
hostNetwork: true
{% if rbac_enabled %}
serviceAccountName: calico-node
{% endif %}
tolerations:
- effect: NoSchedule
operator: Exists
containers:
# Runs calico/node container on each Kubernetes node. This
# container programs network policy and routes on each
# host.
- name: calico-node
image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }}
env:
# The location of the Calico etcd cluster.
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_endpoints
# Choose the backend to use.
- name: CALICO_NETWORKING_BACKEND
valueFrom:
configMapKeyRef:
name: calico-config
key: calico_backend
# Cluster type to identify the deployment type
- name: CLUSTER_TYPE
valueFrom:
configMapKeyRef:
name: calico-config
key: cluster_type
# Disable file logging so `kubectl logs` works.
- name: CALICO_DISABLE_FILE_LOGGING
value: "true"
# Set Felix endpoint to host default action to ACCEPT.
- name: FELIX_DEFAULTENDPOINTTOHOSTACTION
value: "{{ calico_endpoint_to_host_action|default('RETURN') }}"
# should be set in etcd before deployment
# # Configure the IP Pool from which Pod IPs will be chosen.
# - name: CALICO_IPV4POOL_CIDR
# value: "192.168.0.0/16"
# - name: CALICO_IPV4POOL_IPIP
# value: "always"
# Disable IPv6 on Kubernetes.
- name: FELIX_IPV6SUPPORT
value: "false"
# Set Felix logging to "info"
- name: FELIX_LOGSEVERITYSCREEN
value: "info"
# Disable autocreation of pools
- name: CALICO_NO_DEFAULT_POOLS
value: "true"
# Enable libnetwork
- name: CALICO_LIBNETWORK_ENABLED
value: "true"
# Set MTU for tunnel device used if ipip is enabled
{% if calico_mtu is defined %}
- name: FELIX_IPINIPMTU
value: "{{ calico_mtu }}"
{% endif %}
# Location of the CA certificate for etcd.
- name: ETCD_CA_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_ca
# Location of the client key for etcd.
- name: ETCD_KEY_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_key
# Location of the client certificate for etcd.
- name: ETCD_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_cert
- name: IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: NODENAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: FELIX_HEALTHENABLED
value: "true"
securityContext:
privileged: true
resources:
limits:
cpu: {{ calico_node_cpu_limit }}
memory: {{ calico_node_memory_limit }}
requests:
cpu: {{ calico_node_cpu_requests }}
memory: {{ calico_node_memory_requests }}
livenessProbe:
httpGet:
path: /liveness
port: 9099
periodSeconds: 10
initialDelaySeconds: 10
failureThreshold: 6
readinessProbe:
httpGet:
path: /readiness
port: 9099
periodSeconds: 10
volumeMounts:
- mountPath: /lib/modules
name: lib-modules
readOnly: true
- mountPath: /var/run/calico
name: var-run-calico
readOnly: false
- mountPath: /calico-secrets
name: etcd-certs
volumes:
# Used by calico/node.
- name: lib-modules
hostPath:
path: /lib/modules
- name: var-run-calico
hostPath:
path: /var/run/calico
# Used to install CNI.
- name: cni-bin-dir
hostPath:
path: /opt/cni/bin
- name: cni-net-dir
hostPath:
path: /etc/cni/net.d
# Mount in the etcd TLS secrets.
- name: etcd-certs
hostPath:
path: "{{ calico_cert_dir }}"
updateStrategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate

20
roles/network_plugin/calico/templates/calico.env.j2
View file

@ -1,20 +0,0 @@
ETCD_ENDPOINTS="{{ etcd_access_endpoint }}"
ETCD_CA_CERT_FILE="{{ calico_cert_dir }}/ca_cert.crt"
ETCD_CERT_FILE="{{ calico_cert_dir }}/cert.crt"
ETCD_KEY_FILE="{{ calico_cert_dir }}/key.pem"
CALICO_IP="{{ip | default(ansible_default_ipv4.address) }}"
CALICO_IP6=""
{% if calico_network_backend is defined and calico_network_backend == 'none' %}
CLUSTER_TYPE="kubespray"
{% else %}
CLUSTER_TYPE="kubespray,bgp"
{% endif %}
{% if calico_network_backend is defined %}
CALICO_NETWORKING_BACKEND="{{calico_network_backend }}"
{% endif %}
{% if inventory_hostname in groups['k8s-cluster'] and peer_with_router|default(false)%}
CALICO_AS="{{ local_as }}"
{% endif %}
CALICO_NO_DEFAULT_POOLS="true"
CALICO_LIBNETWORK_ENABLED="true"
CALICO_HOSTNAME="{{ ansible_hostname }}"

2
roles/network_plugin/calico/templates/cni-calico.conf.j2
View file

@ -1,8 +1,6 @@
{
"name": "calico-k8s-network",
{% if not legacy_calicoctl %}
"hostname": "{{ ansible_hostname }}",
{% endif %}
"type": "calico",
"etcd_endpoints": "{{ etcd_access_endpoint }}",
"etcd_cert_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem",