From f13e76d022d6143a3307a9a88573207ed6858330 Mon Sep 17 00:00:00 2001 From: melkosoft Date: Fri, 16 Feb 2018 19:37:47 -0800 Subject: [PATCH] Added cilium support (#2236) * Added cilium support * Fix typo in debian test config * Remove empty lines * Changed cilium version from to * Add missing changes for cilium * Add cilium to CI pipeline * Fix wrong file name * Check kernel version for cilium * fixed ci error * fixed cilium-ds.j2 template * added waiting for cilium pods to run * Fixed missing EOF * Fixed trailing spaces * Fixed trailing spaces * Fixed trailing spaces * Fixed too many blank lines * Updated tolerations,annotations in cilium DS template * Set cilium_version to iptables-1.9 to see if bug is fixed in CI * Update cilium image tag to v1.0.0-rc4 * Update Cilium test case CI vars filenames * Add optional prometheus flag, adjust initial readiness delay * Update README.md with cilium info --- .gitlab-ci.yml | 15 ++ README.md | 5 +- inventory/sample/group_vars/k8s-cluster.yml | 2 +- roles/download/defaults/main.yml | 9 + .../network_plugin/cilium/tasks/main.yml | 20 +++ .../network_plugin/meta/main.yml | 5 + .../kube-controller-manager.manifest.j2 | 2 +- .../node/templates/kubelet.kubeadm.env.j2 | 2 +- .../node/templates/kubelet.rkt.service.j2 | 2 +- .../node/templates/kubelet.standard.env.j2 | 2 +- roles/kubernetes/preinstall/tasks/main.yml | 3 +- .../preinstall/tasks/verify-settings.yml | 6 + roles/kubespray-defaults/defaults/main.yaml | 2 +- roles/network_plugin/cilium/defaults/main.yml | 25 +++ roles/network_plugin/cilium/handlers/main.yml | 14 ++ roles/network_plugin/cilium/tasks/main.yml | 55 ++++++ .../cilium/templates/cilium-config.yml.j2 | 29 ++++ .../cilium/templates/cilium-cr.yml.j2 | 62 +++++++ .../cilium/templates/cilium-crb.yml.j2 | 15 ++ .../cilium/templates/cilium-ds.yml.j2 | 163 ++++++++++++++++++ .../cilium/templates/cilium-sa.yml.j2 | 6 + roles/network_plugin/meta/main.yml | 5 + tests/files/gce_centos7-cilium.yml | 11 ++ tests/files/gce_coreos-cilium.yml | 13 ++ tests/files/gce_rhel7-cilium.yml | 10 ++ tests/files/gce_ubuntu-cilium-sep.yml | 11 ++ 26 files changed, 486 insertions(+), 8 deletions(-) create mode 100755 roles/kubernetes-apps/network_plugin/cilium/tasks/main.yml create mode 100755 roles/network_plugin/cilium/defaults/main.yml create mode 100644 roles/network_plugin/cilium/handlers/main.yml create mode 100755 roles/network_plugin/cilium/tasks/main.yml create mode 100755 roles/network_plugin/cilium/templates/cilium-config.yml.j2 create mode 100755 roles/network_plugin/cilium/templates/cilium-cr.yml.j2 create mode 100755 roles/network_plugin/cilium/templates/cilium-crb.yml.j2 create mode 100755 roles/network_plugin/cilium/templates/cilium-ds.yml.j2 create mode 100755 roles/network_plugin/cilium/templates/cilium-sa.yml.j2 create mode 100644 tests/files/gce_centos7-cilium.yml create mode 100644 tests/files/gce_coreos-cilium.yml create mode 100644 tests/files/gce_rhel7-cilium.yml create mode 100644 tests/files/gce_ubuntu-cilium-sep.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d3016566f..3d49fd26d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -257,6 +257,10 @@ before_script: # stage: deploy-special MOVED_TO_GROUP_VARS: "true" +.ubuntu_cilium_sep_variables: &ubuntu_cilium_sep_variables +# stage: deploy-special + MOVED_TO_GROUP_VARS: "true" + .rhel7_weave_variables: &rhel7_weave_variables # stage: deploy-part1 MOVED_TO_GROUP_VARS: "true" @@ -456,6 +460,17 @@ gce_ubuntu-contiv-sep: except: ['triggers'] only: ['master', /^pr-.*$/] +gce_ubuntu-cilium-sep: + stage: deploy-special + <<: *job + <<: *gce + variables: + <<: *gce_variables + <<: *ubuntu_cilium_sep_variables + when: manual + except: ['triggers'] + only: ['master', /^pr-.*$/] + gce_rhel7-weave: stage: deploy-part2 <<: *job diff --git a/README.md b/README.md index f784fc2cf..df80c27ff 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ Versions of supported components - [flanneld](https://github.com/coreos/flannel/releases) v0.8.0 - [calico](https://docs.projectcalico.org/v2.5/releases/) v2.5.0 - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) +- [cilium](https://github.com/cilium/cilium) v1.0.0-rc4 - [contiv](https://github.com/contiv/install/releases) v1.0.3 - [weave](http://weave.works/) v2.0.1 - [docker](https://www.docker.com/) v1.13 (see note) @@ -107,7 +108,7 @@ Requirements Network Plugins --------------- -You can choose between 4 network plugins. (default: `calico`, except Vagrant uses `flannel`) +You can choose between 6 network plugins. (default: `calico`, except Vagrant uses `flannel`) - [flannel](docs/flannel.md): gre/vxlan (layer 2) networking. @@ -115,6 +116,8 @@ You can choose between 4 network plugins. (default: `calico`, except Vagrant use - [canal](https://github.com/projectcalico/canal): a composition of calico and flannel plugins. +- [cilium](http://docs.cilium.io/en/latest/): layer 3/4 networking (as well as layer 7 to protect and secure application protocols), supports dynamic insertion of BPF bytecode into the Linux kernel to implement security services, networking and visibility logic. + - [contiv](docs/contiv.md): supports vlan, vxlan, bgp and Cisco SDN networking. This plugin is able to apply firewall policies, segregate containers in multiple network and bridging pods onto physical networks. diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index 61c54b5d1..cf4f08a89 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -62,7 +62,7 @@ kube_users: # kube_oidc_groups_claim: groups -# Choose network plugin (calico, contiv, weave or flannel) +# Choose network plugin (cilium, calico, contiv, weave or flannel) # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 30ab95808..406ec8b95 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -41,6 +41,7 @@ vault_version: 0.8.1 weave_version: 2.2.0 pod_infra_version: 3.0 contiv_version: 1.1.7 +cilium_version: "v1.0.0-rc4" # Download URLs istioctl_download_url: "https://storage.googleapis.com/istio-release/releases/{{ istio_version }}/istioctl/istioctl-linux" @@ -88,6 +89,8 @@ contiv_image_repo: "contiv/netplugin" contiv_image_tag: "{{ contiv_version }}" contiv_auth_proxy_image_repo: "contiv/auth_proxy" contiv_auth_proxy_image_tag: "{{ contiv_version }}" +cilium_image_repo: "docker.io/cilium/cilium" +cilium_image_tag: "{{ cilium_version }}" nginx_image_repo: nginx nginx_image_tag: 1.13 @@ -174,6 +177,12 @@ downloads: repo: "{{ hyperkube_image_repo }}" tag: "{{ hyperkube_image_tag }}" sha256: "{{ hyperkube_digest_checksum|default(None) }}" + cilium: + enabled: "{{ kube_network_plugin == 'cilium' }}" + container: true + repo: "{{ cilium_image_repo }}" + tag: "{{ cilium_image_tag }}" + sha256: "{{ cilium_digest_checksum|default(None) }}" flannel: enabled: "{{ kube_network_plugin == 'flannel' or kube_network_plugin == 'canal' }}" container: true diff --git a/roles/kubernetes-apps/network_plugin/cilium/tasks/main.yml b/roles/kubernetes-apps/network_plugin/cilium/tasks/main.yml new file mode 100755 index 000000000..2359fe2d4 --- /dev/null +++ b/roles/kubernetes-apps/network_plugin/cilium/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Cilium | Start Resources + kube: + name: "{{item.item.name}}" + namespace: "{{ system_namespace }}" + kubectl: "{{bin_dir}}/kubectl" + resource: "{{item.item.type}}" + filename: "{{kube_config_dir}}/{{item.item.file}}" + state: "latest" + with_items: "{{ cilium_node_manifests.results }}" + when: inventory_hostname == groups['kube-master'][0] and not item|skipped + +- name: Cilium | Wait for pods to run + command: "{{bin_dir}}/kubectl -n {{system_namespace}} get pods -l k8s-app=cilium -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'" + register: pods_not_ready + until: pods_not_ready.stdout.find("cilium")==-1 + retries: 30 + delay: 10 + ignore_errors: yes + when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/network_plugin/meta/main.yml b/roles/kubernetes-apps/network_plugin/meta/main.yml index d194c5719..7876401b8 100644 --- a/roles/kubernetes-apps/network_plugin/meta/main.yml +++ b/roles/kubernetes-apps/network_plugin/meta/main.yml @@ -1,5 +1,10 @@ --- dependencies: + - role: kubernetes-apps/network_plugin/cilium + when: kube_network_plugin == 'cilium' + tags: + - cilium + - role: kubernetes-apps/network_plugin/calico when: kube_network_plugin == 'calico' tags: diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 03a406329..2b4282a2e 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -52,7 +52,7 @@ spec: {% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %} - --configure-cloud-routes=true {% endif %} -{% if kube_network_plugin is defined and kube_network_plugin in ["cloud", "flannel", "canal"] %} +{% if kube_network_plugin is defined and kube_network_plugin in ["cloud", "flannel", "canal", "cilium"] %} - --allocate-node-cidrs=true - --cluster-cidr={{ kube_pods_subnet }} - --service-cluster-ip-range={{ kube_service_addresses }} diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index 68c14c0ce..e49b15f48 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -60,7 +60,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_reserve }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" -{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv"] %} +{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %} KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet" diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 index f78a55256..e1406e7e1 100644 --- a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 @@ -39,7 +39,7 @@ ExecStart=/usr/bin/rkt run \ {% if local_volume_provisioner_enabled == true %} --volume local-volume-base-dir,kind=host,source={{ local_volume_base_dir }},readOnly=false,recursive=true \ {% endif %} -{% if kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv"] %} +{% if kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium"] %} --volume etc-cni,kind=host,source=/etc/cni,readOnly=true \ --volume opt-cni,kind=host,source=/opt/cni,readOnly=true \ --volume var-lib-cni,kind=host,source=/var/lib/cni,readOnly=false \ diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2 index 43dcb9451..cb7d83d35 100644 --- a/roles/kubernetes/node/templates/kubelet.standard.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2 @@ -83,7 +83,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% endif %} KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kubelet_reserve }} {{ node_labels }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}" -{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv"] %} +{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave", "contiv", "cilium"] %} KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %} DOCKER_SOCKET="--docker-endpoint=unix:/var/run/weave/weave.sock" diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index daf3dbbed..289065c71 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -89,10 +89,11 @@ - "/etc/cni/net.d" - "/opt/cni/bin" when: - - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv"] + - kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium"] - inventory_hostname in groups['k8s-cluster'] tags: - network + - cilium - calico - weave - canal diff --git a/roles/kubernetes/preinstall/tasks/verify-settings.yml b/roles/kubernetes/preinstall/tasks/verify-settings.yml index 6f77b6ad8..8f0a2e854 100644 --- a/roles/kubernetes/preinstall/tasks/verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/verify-settings.yml @@ -88,4 +88,10 @@ assert: that: rbac_enabled and kube_api_anonymous_auth when: kube_apiserver_insecure_port == 0 + ignore_errors: "{{ ignore_assert_errors }}" + +- name: Stop if kernel version is too low + assert: + that: ansible_kernel.split('-')[0]|version_compare('4.8', '>=') + when: kube_network_plugin == 'cilium' ignore_errors: "{{ ignore_assert_errors }}" \ No newline at end of file diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 3be3e9d66..6883370ee 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -82,7 +82,7 @@ kube_users: pass: "{{kube_api_pwd}}" role: admin -# Choose network plugin (calico, weave or flannel) +# Choose network plugin (cilium, calico, weave or flannel) # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml new file mode 100755 index 000000000..389fe5bd6 --- /dev/null +++ b/roles/network_plugin/cilium/defaults/main.yml @@ -0,0 +1,25 @@ +--- +# Log-level +cilium_debug: false +cilium_disable_ipv4: false + +# Etcd SSL dirs +cilium_cert_dir: /etc/cilium/certs +etcd_cert_dir: /etc/ssl/etcd/ssl + +# Cilium Network Policy directory +cilium_policy_dir: /etc/kubernetes/policy + +# Limits for apps +cilium_memory_limit: 500M +cilium_cpu_limit: 200m +cilium_memory_requests: 64M +cilium_cpu_requests: 50m + +# Optional features +cilium_enable_prometheus: false + +rbac_resources: + - sa + - clusterrole + - clusterrolebinding diff --git a/roles/network_plugin/cilium/handlers/main.yml b/roles/network_plugin/cilium/handlers/main.yml new file mode 100644 index 000000000..039c3469f --- /dev/null +++ b/roles/network_plugin/cilium/handlers/main.yml @@ -0,0 +1,14 @@ +--- +- name: restart kubelet + command: /bin/true + notify: + - Kubelet | reload systemd + - Kubelet | reload kubelet + +- name: Kubelet | reload systemd + command: systemctl daemon-reload + +- name: Kubelet | reload kubelet + service: + name: kubelet + state: restarted \ No newline at end of file diff --git a/roles/network_plugin/cilium/tasks/main.yml b/roles/network_plugin/cilium/tasks/main.yml new file mode 100755 index 000000000..12408a00a --- /dev/null +++ b/roles/network_plugin/cilium/tasks/main.yml @@ -0,0 +1,55 @@ +--- +- name: Cilium | Ensure BFPFS mounted + mount: + fstype: bpf + path: /sys/fs/bpf + src: bpffs + state: mounted + +- name: Cilium | Create Cilium certs directory + file: + dest: "{{ cilium_cert_dir }}" + state: directory + mode: 0750 + owner: root + group: root + +- name: Cilium | Link etcd certificates for cilium + file: + src: "{{ etcd_cert_dir }}/{{ item.s }}" + dest: "{{ cilium_cert_dir }}/{{ item.d }}" + state: hard + force: yes + with_items: + - {s: "ca.pem", d: "ca_cert.crt"} + - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"} + - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"} + +- name: Cilium | Create Cilium node manifests + template: + src: "{{item.file}}.j2" + dest: "{{kube_config_dir}}/{{item.file}}" + with_items: + - {name: cilium, file: cilium-config.yml, type: cm} + - {name: cilium, file: cilium-crb.yml, type: clusterrolebinding} + - {name: cilium, file: cilium-cr.yml, type: clusterrole} + - {name: cilium, file: cilium-ds.yml, type: ds} + - {name: cilium, file: cilium-sa.yml, type: sa} + register: cilium_node_manifests + when: + - inventory_hostname in groups['kube-master'] + - rbac_enabled or item.type not in rbac_resources + +- name: Cilium | Set CNI directory permissions + file: + path: /opt/cni/bin + state: directory + owner: kube + recurse: true + mode: 0755 + register: cni_bin_dir + +- name: Cilium | Create network policy directory + file: + path: "{{ cilium_policy_dir }}" + state: directory diff --git a/roles/network_plugin/cilium/templates/cilium-config.yml.j2 b/roles/network_plugin/cilium/templates/cilium-config.yml.j2 new file mode 100755 index 000000000..a96bb8531 --- /dev/null +++ b/roles/network_plugin/cilium/templates/cilium-config.yml.j2 @@ -0,0 +1,29 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: cilium-config + namespace: {{ system_namespace }} +data: + # This etcd-config contains the etcd endpoints of your cluster. If you use + # TLS please make sure you uncomment the ca-file line and add the respective + # certificate has a k8s secret, see explanation bellow in the comment labeled + # "ETCD-CERT" + etcd-config: |- + --- + endpoints: +{% for ip_addr in etcd_access_addresses.split(',') %} + - {{ ip_addr }} +{% endfor %} + # + # In case you want to use TLS in etcd, uncomment the following line + # and add the certificate as explained in the comment labeled "ETCD-CERT" + ca-file: "{{ cilium_cert_dir }}/ca_cert.crt" + # + # In case you want client to server authentication, uncomment the following + # lines and add the certificate and key in cilium-etcd-secrets bellow + key-file: "{{ cilium_cert_dir }}/key.pem" + cert-file: "{{ cilium_cert_dir }}/cert.crt" + + # If you want to run cilium in debug mode change this value to true + debug: "{{ cilium_debug }}" + disable-ipv4: "{{ cilium_disable_ipv4 }}" diff --git a/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 new file mode 100755 index 000000000..8eae0e8ed --- /dev/null +++ b/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 @@ -0,0 +1,62 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: cilium +rules: +- apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - nodes + - endpoints + - componentstatuses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch + - update +- apiGroups: + - extensions + resources: + - networkpolicies #FIXME remove this when we drop support for k8s NP-beta GH-1202 + - thirdpartyresources + - ingresses + verbs: + - create + - get + - list + - watch +- apiGroups: + - "apiextensions.k8s.io" + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + verbs: + - "*" diff --git a/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 b/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 new file mode 100755 index 000000000..dcfe4d471 --- /dev/null +++ b/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 @@ -0,0 +1,15 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: cilium + namespace: {{ system_namespace }} +- kind: Group + name: system:nodes diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 new file mode 100755 index 000000000..9f48a62db --- /dev/null +++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 @@ -0,0 +1,163 @@ +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: cilium + namespace: {{ system_namespace }} +spec: + template: + metadata: + labels: + k8s-app: cilium + kubernetes.io/cluster-service: "true" + annotations: + # This annotation plus the CriticalAddonsOnly toleration makes + # cilium to be a critical pod in the cluster, which ensures cilium + # gets priority scheduling. + # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ + scheduler.alpha.kubernetes.io/critical-pod: '' + scheduler.alpha.kubernetes.io/tolerations: >- + [{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}] +{% if cilium_enable_prometheus %} + prometheus.io/scrape: "true" + prometheus.io/port: "9090" +{% endif %} + spec: +{% if rbac_enabled %} + serviceAccountName: cilium +{% endif %} + containers: + - image: {{ cilium_image_repo }}:{{ cilium_image_tag }} + imagePullPolicy: Always + name: cilium-agent + command: [ "cilium-agent" ] + args: + - "--debug=$(CILIUM_DEBUG)" + - "-t" + - "vxlan" + - "--kvstore" + - "etcd" + - "--kvstore-opt" + - "etcd.config=/var/lib/etcd-config/etcd.config" + - "--disable-ipv4=$(DISABLE_IPV4)" +{% if cilium_enable_prometheus %} + ports: + - name: prometheus + containerPort: 9090 +{% endif %} + lifecycle: + postStart: + exec: + command: + - "/cni-install.sh" + preStop: + exec: + command: + - "/cni-uninstall.sh" + env: + - name: "K8S_NODE_NAME" + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: "CILIUM_DEBUG" + valueFrom: + configMapKeyRef: + name: cilium-config + key: debug + - name: "DISABLE_IPV4" + valueFrom: + configMapKeyRef: + name: cilium-config + key: disable-ipv4 +{% if cilium_enable_prometheus %} + # Note: this variable is a no-op if not defined, and is used in the + # prometheus examples. + - name: "CILIUM_PROMETHEUS_SERVE_ADDR" + valueFrom: + configMapKeyRef: + name: cilium-metrics-config + optional: true + key: prometheus-serve-addr +{% endif %} + livenessProbe: + exec: + command: + - cilium + - status + # The initial delay for the liveness probe is intentionally large to + # avoid an endless kill & restart cycle if in the event that the initial + # bootstrapping takes longer than expected. + initialDelaySeconds: 120 + failureThreshold: 10 + periodSeconds: 10 + readinessProbe: + exec: + command: + - cilium + - status + initialDelaySeconds: 5 + periodSeconds: 5 + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + - name: cilium-run + mountPath: /var/run/cilium + - name: cni-path + mountPath: /host/opt/cni/bin + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: docker-socket + mountPath: /var/run/docker.sock + readOnly: true + - name: etcd-config-path + mountPath: /var/lib/etcd-config + readOnly: true + - name: cilium-certs + mountPath: {{ cilium_cert_dir }} + readOnly: true + securityContext: + capabilities: + add: + - "NET_ADMIN" + privileged: true + hostNetwork: true + volumes: + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + # To keep state between restarts / upgrades + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + # To read docker events from the node + - name: docker-socket + hostPath: + path: /var/run/docker.sock + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + - name: cilium-certs + hostPath: + path: {{ cilium_cert_dir }} + # To read the etcd config stored in config maps + - name: etcd-config-path + configMap: + name: cilium-config + items: + - key: etcd-config + path: etcd.config + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + # Mark cilium's pod as critical for rescheduling + - key: CriticalAddonsOnly + operator: "Exists" diff --git a/roles/network_plugin/cilium/templates/cilium-sa.yml.j2 b/roles/network_plugin/cilium/templates/cilium-sa.yml.j2 new file mode 100755 index 000000000..d6ef2a431 --- /dev/null +++ b/roles/network_plugin/cilium/templates/cilium-sa.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cilium + namespace: {{ system_namespace }} diff --git a/roles/network_plugin/meta/main.yml b/roles/network_plugin/meta/main.yml index 7457f4eb6..4a1a73062 100644 --- a/roles/network_plugin/meta/main.yml +++ b/roles/network_plugin/meta/main.yml @@ -1,5 +1,10 @@ --- dependencies: + - role: network_plugin/cilium + when: kube_network_plugin == 'cilium' + tags: + - cilium + - role: network_plugin/calico when: kube_network_plugin == 'calico' tags: diff --git a/tests/files/gce_centos7-cilium.yml b/tests/files/gce_centos7-cilium.yml new file mode 100644 index 000000000..ca682f7ed --- /dev/null +++ b/tests/files/gce_centos7-cilium.yml @@ -0,0 +1,11 @@ +# Instance settings +cloud_image_family: centos-7 +cloud_region: us-central1-c +cloud_machine_type: "n1-standard-1" +mode: default + +# Deployment settings +kube_network_plugin: cilium +deploy_netchecker: true +kubedns_min_replicas: 1 +cloud_provider: gce diff --git a/tests/files/gce_coreos-cilium.yml b/tests/files/gce_coreos-cilium.yml new file mode 100644 index 000000000..a09003970 --- /dev/null +++ b/tests/files/gce_coreos-cilium.yml @@ -0,0 +1,13 @@ +# Instance settings +cloud_image_family: coreos-stable +cloud_region: us-central1-c +mode: default +startup_script: 'systemctl disable locksmithd && systemctl stop locksmithd' + +# Deployment settings +kube_network_plugin: cilium +bootstrap_os: coreos +resolvconf_mode: host_resolvconf # this is required as long as the coreos stable channel uses docker < 1.12 +deploy_netchecker: true +kubedns_min_replicas: 1 +cloud_provider: gce diff --git a/tests/files/gce_rhel7-cilium.yml b/tests/files/gce_rhel7-cilium.yml new file mode 100644 index 000000000..d67658a6c --- /dev/null +++ b/tests/files/gce_rhel7-cilium.yml @@ -0,0 +1,10 @@ +# Instance settings +cloud_image_family: rhel-7 +cloud_region: us-central1-b +mode: default + +# Deployment settings +kube_network_plugin: cilium +deploy_netchecker: true +kubedns_min_replicas: 1 +cloud_provider: gce diff --git a/tests/files/gce_ubuntu-cilium-sep.yml b/tests/files/gce_ubuntu-cilium-sep.yml new file mode 100644 index 000000000..e7150a27e --- /dev/null +++ b/tests/files/gce_ubuntu-cilium-sep.yml @@ -0,0 +1,11 @@ +# Instance settings +cloud_image_family: ubuntu-1604-lts +cloud_region: us-central1-b +mode: separate + +# Deployment settings +kube_network_plugin: cilium +deploy_netchecker: true +kubedns_min_replicas: 1 +cloud_provider: gce +