Calico is not a network overlay

This commit is contained in:
Smaine Kahlouch 2015-10-27 15:42:46 +01:00
parent b98227e9a4
commit f216302f95
24 changed files with 86 additions and 45 deletions

View file

@ -1,7 +1,7 @@
kubernetes-ansible kubernetes-ansible
======== ========
Install and configure a kubernetes cluster including network overlay and optionnal addons. Install and configure a kubernetes cluster including network plugin and optionnal addons.
Based on [CiscoCloud](https://github.com/CiscoCloud/kubernetes-ansible) work. Based on [CiscoCloud](https://github.com/CiscoCloud/kubernetes-ansible) work.
### Requirements ### Requirements
@ -32,6 +32,49 @@ Please ensure that you have enough disk space there (about **1G**).
### Variables ### Variables
The main variables to change are located in the directory ```environments/[env_name]/group_vars/k8s-cluster.yml```. The main variables to change are located in the directory ```environments/[env_name]/group_vars/k8s-cluster.yml```.
### Inventory
Below is an example of an inventory.
Note : The bgp vars (local_as, peers) are not mandatory if the var "peer_with_router" is set to false
```
[downloader]
10.99.0.26
[kube-master]
# NB : the br_addr must be in the {{ calico_pool }} subnet
# it will assign a /24 subnet per node
10.99.0.26 br_addr=10.99.64.1
[etcd]
10.99.0.26
[kube-node]
10.99.0.4
10.99.0.5
10.99.0.6
10.99.0.36
10.99.0.37
[itx2]
10.99.0.26 br_addr=10.99.16.1
10.99.0.4 br_addr=10.99.65.1 local_as=xxxxxxxx
10.99.0.5 br_addr=10.99.66.1 local_as=xxxxxxxx
10.99.0.6 br_addr=10.99.69.1 local_as=xxxxxxxx
[rmv]
10.99.0.36 br_addr=10.99.67.1 local_as=xxxxxxxx
10.99.0.37 br_addr=10.99.68.1 local_as=xxxxxxxx
[k8s-cluster:children]
kube-node
kube-master
[itx2:vars]
peers=[{"router_id": "10.99.0.2", "as": "65xxx"}, {"router_id": "10.99.0.3", "as": "65xxx"}]
[rmv:vars]
peers=[{"router_id": "10.99.0.34", "as": "65xxx"}, {"router_id": "10.99.0.35", "as": "65xxx"}]
```
### Playbook ### Playbook
``` ```
--- ---
@ -44,7 +87,7 @@ The main variables to change are located in the directory ```environments/[env_n
roles: roles:
- { role: etcd, tags: etcd } - { role: etcd, tags: etcd }
- { role: docker, tags: docker } - { role: docker, tags: docker }
- { role: overlay_network, tags: ['calico', 'flannel', 'network'] } - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
- { role: dnsmasq, tags: dnsmasq } - { role: dnsmasq, tags: dnsmasq }
- hosts: kube-master - hosts: kube-master
@ -72,13 +115,13 @@ Kubernetes
------------------------- -------------------------
### Network Overlay ### Network Overlay
You can choose between 2 network overlays. Only one must be chosen. You can choose between 2 network plugins. Only one must be chosen.
* **flannel**: gre/vxlan (layer 2) networking. ([official docs]('https://github.com/coreos/flannel')) * **flannel**: gre/vxlan (layer 2) networking. ([official docs]('https://github.com/coreos/flannel'))
* **calico**: bgp (layer 3) networking. ([official docs]('http://docs.projectcalico.org/en/0.13/')) * **calico**: bgp (layer 3) networking. ([official docs]('http://docs.projectcalico.org/en/0.13/'))
The choice is defined with the variable '**overlay_network_plugin**' The choice is defined with the variable '**kube_network_plugin**'
### Expose a service ### Expose a service
There are several loadbalancing solutions. There are several loadbalancing solutions.

View file

@ -8,7 +8,7 @@
roles: roles:
- { role: etcd, tags: etcd } - { role: etcd, tags: etcd }
- { role: docker, tags: docker } - { role: docker, tags: docker }
- { role: overlay_network, tags: ['calico', 'flannel', 'network'] } - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
- { role: dnsmasq, tags: dnsmasq } - { role: dnsmasq, tags: dnsmasq }
- hosts: kube-master - hosts: kube-master

View file

@ -11,7 +11,7 @@ kube_users:
cluster_name: cluster.local cluster_name: cluster.local
# #
# set this variable to calico if needed. keep it empty if flannel is used # set this variable to calico if needed. keep it empty if flannel is used
overlay_network_plugin: calico kube_network_plugin: calico
# Kubernetes internal network for services, unused block of space. # Kubernetes internal network for services, unused block of space.
kube_service_addresses: 10.233.0.0/18 kube_service_addresses: 10.233.0.0/18
@ -19,17 +19,16 @@ kube_service_addresses: 10.233.0.0/18
# internal network. When used, it will assign IP # internal network. When used, it will assign IP
# addresses from this range to individual pods. # addresses from this range to individual pods.
# This network must be unused in your network infrastructure! # This network must be unused in your network infrastructure!
overlay_network_subnet: 10.233.64.0/18 kube_pods_subnet: 10.233.64.0/18
# internal network total size (optional). This is the prefix of the # internal network total size (optional). This is the prefix of the
# entire overlay network. So the entirety of 4.0.0.0/16 must be # entire network. Must be unused in your environment.
# unused in your environment. # kube_network_prefix: 18
# overlay_network_prefix: 18
# internal network node size allocation (optional). This is the size allocated # internal network node size allocation (optional). This is the size allocated
# to each node on your network. With these defaults you should have # to each node on your network. With these defaults you should have
# room for 4096 nodes with 254 pods per node. # room for 4096 nodes with 254 pods per node.
overlay_network_host_prefix: 24 kube_network_node_prefix: 24
# With calico it is possible to distributed routes with border routers of the datacenter. # With calico it is possible to distributed routes with border routers of the datacenter.
peer_with_router: false peer_with_router: false

View file

@ -9,9 +9,9 @@
# Kubernetes cluster name, also will be used as DNS domain # Kubernetes cluster name, also will be used as DNS domain
# cluster_name: cluster.local # cluster_name: cluster.local
#
# set this variable to calico if needed. keep it empty if flannel is used # set this variable to calico if needed. keep it empty if flannel is used
# overlay_network_plugin: calico # kube_network_plugin: calico
# Kubernetes internal network for services, unused block of space. # Kubernetes internal network for services, unused block of space.
# kube_service_addresses: 10.233.0.0/18 # kube_service_addresses: 10.233.0.0/18
@ -19,17 +19,16 @@
# internal network. When used, it will assign IP # internal network. When used, it will assign IP
# addresses from this range to individual pods. # addresses from this range to individual pods.
# This network must be unused in your network infrastructure! # This network must be unused in your network infrastructure!
# overlay_network_subnet: 10.233.64.0/18 # kube_pods_subnet: 10.233.64.0/18
# internal network total size (optional). This is the prefix of the # internal network total size (optional). This is the prefix of the
# entire overlay network. So the entirety of 4.0.0.0/16 must be # entire network. Must be unused in your environment.
# unused in your environment. # kube_network_prefix: 18
# overlay_network_prefix: 18
# internal network node size allocation (optional). This is the size allocated # internal network node size allocation (optional). This is the size allocated
# to each node on your network. With these defaults you should have # to each node on your network. With these defaults you should have
# room for 4096 nodes with 254 pods per node. # room for 4096 nodes with 254 pods per node.
# overlay_network_host_prefix: 24 # kube_network_node_prefix: 24
# With calico it is possible to distributed routes with border routers of the datacenter. # With calico it is possible to distributed routes with border routers of the datacenter.
# peer_with_router: false # peer_with_router: false

View file

@ -1,11 +1,11 @@
--- ---
- name: Write script for calico/docker bridge configuration - name: Write script for calico/docker bridge configuration
template: src=create_cbr.j2 dest=/etc/network/if-up.d/create_cbr mode=u+x template: src=create_cbr.j2 dest=/etc/network/if-up.d/create_cbr mode=u+x
when: overlay_network_plugin is defined and overlay_network_plugin == "calico" when: kube_network_plugin is defined and kube_network_plugin == "calico"
- name: Configure calico/docker bridge - name: Configure calico/docker bridge
shell: /etc/network/if-up.d/create_cbr shell: /etc/network/if-up.d/create_cbr
when: overlay_network_plugin is defined and overlay_network_plugin == "calico" when: kube_network_plugin is defined and kube_network_plugin == "calico"
- name: Configure docker to use cbr0 bridge - name: Configure docker to use cbr0 bridge
lineinfile: lineinfile:
@ -14,7 +14,7 @@
line='DOCKER_OPTS="--bridge=cbr0 --iptables=false --ip-masq=false"' line='DOCKER_OPTS="--bridge=cbr0 --iptables=false --ip-masq=false"'
notify: notify:
- restart docker - restart docker
when: overlay_network_plugin is defined and overlay_network_plugin == "calico" when: kube_network_plugin is defined and kube_network_plugin == "calico"
- name: enable docker - name: enable docker
service: service:

View file

@ -13,7 +13,7 @@
with_items: with_items:
- aufs-tools - aufs-tools
- cgroupfs-mount - cgroupfs-mount
- docker-engine=1.8.2-0~{{ ansible_distribution_release }} - docker-engine=1.8.3-0~{{ ansible_distribution_release }}
- name: Copy default docker configuration - name: Copy default docker configuration
template: src=default-docker.j2 dest=/etc/default/docker template: src=default-docker.j2 dest=/etc/default/docker

View file

@ -9,6 +9,6 @@ fi
# Configure calico bridge ip # Configure calico bridge ip
br_ips=$(ip addr list cbr0 |grep "inet " |cut -d' ' -f6) br_ips=$(ip addr list cbr0 |grep "inet " |cut -d' ' -f6)
if ! [[ "${br_ips}" =~ "{{ br_addr }}/{{ overlay_network_host_prefix }}" ]];then if ! [[ "${br_ips}" =~ "{{ br_addr }}/{{ kube_network_node_prefix }}" ]];then
ip a add {{ br_addr }}/{{ overlay_network_host_prefix }} dev cbr0 ip a add {{ br_addr }}/{{ kube_network_node_prefix }} dev cbr0
fi fi

View file

@ -4,7 +4,7 @@
#DOCKER="/usr/local/bin/docker" #DOCKER="/usr/local/bin/docker"
# Use DOCKER_OPTS to modify the daemon startup options. # Use DOCKER_OPTS to modify the daemon startup options.
{% if overlay_network_plugin is defined and overlay_network_plugin == "calico" %} {% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
DOCKER_OPTS="--bridge=cbr0 --iptables=false --ip-masq=false" DOCKER_OPTS="--bridge=cbr0 --iptables=false --ip-masq=false"
{% endif %} {% endif %}

View file

@ -1,7 +1,7 @@
[Unit] [Unit]
Description=Kubernetes Kube-Proxy Server Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes Documentation=https://github.com/GoogleCloudPlatform/kubernetes
{% if overlay_network_plugin|default('') %} {% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
After=docker.service calico-node.service After=docker.service calico-node.service
{% else %} {% else %}
After=docker.service After=docker.service

View file

@ -16,6 +16,6 @@ Environment="KUBELET_ARGS=--cluster_dns={{ dns_server }} --cluster_domain={{ dns
{% else %} {% else %}
Environment="KUBELET_ARGS=--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}" Environment="KUBELET_ARGS=--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"
{% endif %} {% endif %}
{% if overlay_network_plugin|default('') %} {% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
Environment="KUBELET_NETWORK_PLUGIN=--network_plugin={{ overlay_network_plugin }}" Environment="KUBELET_NETWORK_PLUGIN=--network_plugin={{ kube_network_plugin }}"
{% endif %} {% endif %}

View file

@ -1,7 +1,7 @@
[Unit] [Unit]
Description=Kubernetes Kube-Proxy Server Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes Documentation=https://github.com/GoogleCloudPlatform/kubernetes
{% if overlay_network_plugin|default('') %} {% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
After=docker.service calico-node.service After=docker.service calico-node.service
{% else %} {% else %}
After=docker.service After=docker.service

View file

@ -1,7 +1,7 @@
[Unit] [Unit]
Description=Kubernetes Kubelet Server Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes Documentation=https://github.com/GoogleCloudPlatform/kubernetes
{% if overlay_network_plugin|default('') %} {% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
After=docker.service calico-node.service After=docker.service calico-node.service
{% else %} {% else %}
After=docker.service After=docker.service

View file

@ -31,7 +31,7 @@
run_once: true run_once: true
- name: Calico | Configure calico-node desired pool - name: Calico | Configure calico-node desired pool
shell: calicoctl pool add {{ overlay_network_subnet }} shell: calicoctl pool add {{ kube_pods_subnet }}
environment: environment:
ETCD_AUTHORITY: "{{ groups['kube-master'][0] }}:4001" ETCD_AUTHORITY: "{{ groups['kube-master'][0] }}:4001"
run_once: true run_once: true

View file

@ -0,0 +1,13 @@
---
- name: "Test if network plugin is defined"
fail: msg="ERROR, One network_plugin variable must be defined (Flannel or Calico)"
when: ( kube_network_plugin is defined and kube_network_plugin == "calico" and kube_network_plugin == "flannel" ) or
kube_network_plugin is not defined
- include: flannel.yml
when: kube_network_plugin == "flannel"
- include: calico.yml
when: kube_network_plugin == "calico"
- meta: flush_handlers

View file

@ -0,0 +1 @@
{ "Network": "{{ kube_service_addresses }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "vxlan" } }

View file

@ -1,13 +0,0 @@
---
- name: "Test if overlay network is defined"
fail: msg="ERROR, One overlay_network variable must be defined (Flannel or Calico)"
when: ( overlay_network_plugin is defined and overlay_network_plugin == "calico" and overlay_network_plugin == "flannel" ) or
overlay_network_plugin is not defined
- include: flannel.yml
when: overlay_network_plugin == "flannel"
- include: calico.yml
when: overlay_network_plugin == "calico"
- meta: flush_handlers

View file

@ -1 +0,0 @@
{ "Network": "{{ kube_service_addresses }}", "SubnetLen": {{ overlay_network_host_prefix }}, "Backend": { "Type": "vxlan" } }