diff --git a/README.md b/README.md index bf2989919..e1213f4a0 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Supported Components - Network Plugin - [calico](https://github.com/projectcalico/calico) v2.6.8 - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) - - [cilium](https://github.com/cilium/cilium) v1.0.0-rc8 + - [cilium](https://github.com/cilium/cilium) v1.1.2 - [contiv](https://github.com/contiv/install) v1.1.7 - [flanneld](https://github.com/coreos/flannel) v0.10.0 - [weave](https://github.com/weaveworks/weave) v2.4.0 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 557f10be4..b06e23b23 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -40,7 +40,7 @@ vault_version: 0.10.1 weave_version: "2.4.0" pod_infra_version: 3.0 contiv_version: 1.1.7 -cilium_version: "v1.0.0-rc8" +cilium_version: "v1.1.2" # Download URLs kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/amd64/kubeadm" diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml index 389fe5bd6..dea905b3b 100755 --- a/roles/network_plugin/cilium/defaults/main.yml +++ b/roles/network_plugin/cilium/defaults/main.yml @@ -12,9 +12,9 @@ cilium_policy_dir: /etc/kubernetes/policy # Limits for apps cilium_memory_limit: 500M -cilium_cpu_limit: 200m +cilium_cpu_limit: 500m cilium_memory_requests: 64M -cilium_cpu_requests: 50m +cilium_cpu_requests: 100m # Optional features cilium_enable_prometheus: false diff --git a/roles/network_plugin/cilium/templates/cilium-config.yml.j2 b/roles/network_plugin/cilium/templates/cilium-config.yml.j2 index c5051e2ca..cf5758465 100755 --- a/roles/network_plugin/cilium/templates/cilium-config.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-config.yml.j2 @@ -1,29 +1,49 @@ -kind: ConfigMap +--- apiVersion: v1 +kind: ConfigMap metadata: name: cilium-config namespace: kube-system data: # This etcd-config contains the etcd endpoints of your cluster. If you use - # TLS please make sure you uncomment the ca-file line and add the respective - # certificate has a k8s secret, see explanation bellow in the comment labeled - # "ETCD-CERT" + # TLS please make sure you follow the tutorial in https://cilium.link/etcd-config etcd-config: |- --- - endpoints: + endpoints: {% for ip_addr in etcd_access_addresses.split(',') %} - - {{ ip_addr }} + - {{ ip_addr }} {% endfor %} - # - # In case you want to use TLS in etcd, uncomment the following line - # and add the certificate as explained in the comment labeled "ETCD-CERT" + + # In case you want to use TLS in etcd, uncomment the 'ca-file' line + # and create a kubernetes secret by following the tutorial in + # https://cilium.link/etcd-config ca-file: "{{ cilium_cert_dir }}/ca_cert.crt" - # + # In case you want client to server authentication, uncomment the following - # lines and add the certificate and key in cilium-etcd-secrets bellow + # lines and create a kubernetes secret by following the tutorial in + # https://cilium.link/etcd-config key-file: "{{ cilium_cert_dir }}/key.pem" cert-file: "{{ cilium_cert_dir }}/cert.crt" # If you want to run cilium in debug mode change this value to true debug: "{{ cilium_debug }}" disable-ipv4: "{{ cilium_disable_ipv4 }}" + # If you want to clean cilium state; change this value to true + clean-cilium-state: "false" + legacy-host-allows-world: "false" + + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation-level: "none" + + # Regular expression matching compatible Istio sidecar istio-proxy + # container image names + sidecar-istio-proxy-image: "cilium/istio_proxy" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + tunnel: "vxlan" diff --git a/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 index 11fd01087..2e5efff86 100755 --- a/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 @@ -1,64 +1,66 @@ --- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: cilium rules: -- apiGroups: - - "networking.k8s.io" - resources: - - networkpolicies - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - - services - - nodes - - endpoints - - componentstatuses - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - pods - - nodes - verbs: - - get - - list - - watch - - update -- apiGroups: - - extensions - resources: - - networkpolicies #FIXME remove this when we drop support for k8s NP-beta GH-1202 - - thirdpartyresources - - ingresses - verbs: - - create - - get - - list - - watch -- apiGroups: - - "apiextensions.k8s.io" - resources: - - customresourcedefinitions - verbs: - - create - - get - - list - - watch - - update -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies - - ciliumendpoints - verbs: - - "*" + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - namespaces + - services + - nodes + - endpoints + - componentstatuses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch + - update + - apiGroups: + - extensions + resources: + - networkpolicies # FIXME remove this when we drop support for k8s NP-beta GH-1202 + - thirdpartyresources + - ingresses + verbs: + - create + - get + - list + - watch + - apiGroups: + - "apiextensions.k8s.io" + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumnetworkpolicies/status + - ciliumendpoints + - ciliumendpoints/status + verbs: + - "*" diff --git a/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 b/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 index 04d603d57..35994bc68 100755 --- a/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 @@ -1,6 +1,6 @@ --- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: cilium roleRef: @@ -8,8 +8,8 @@ roleRef: kind: ClusterRole name: cilium subjects: -- kind: ServiceAccount - name: cilium - namespace: kube-system -- kind: Group - name: system:nodes + - kind: ServiceAccount + name: cilium + namespace: kube-system + - kind: Group + name: system:nodes diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 index 8eaa24f32..1ec322916 100755 --- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 @@ -1,10 +1,21 @@ --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: DaemonSet metadata: name: cilium namespace: kube-system spec: + updateStrategy: + type: "RollingUpdate" + rollingUpdate: + # Specifies the maximum number of Pods that can be unavailable during the update process. + # The current default value is 1 or 100% for daemonsets; Adding an explicit value here + # to avoid confusion, as the default value is specific to the type (daemonset/deployment). + maxUnavailable: "100%" + selector: + matchLabels: + k8s-app: cilium + kubernetes.io/cluster-service: "true" template: metadata: labels: @@ -26,145 +37,185 @@ spec: {% if rbac_enabled %} serviceAccountName: cilium {% endif %} + initContainers: + - name: clean-cilium-state + image: docker.io/library/busybox:1.28.4 + imagePullPolicy: IfNotPresent + command: ['sh', '-c', 'if [ "${CLEAN_CILIUM_STATE}" = "true" ]; then rm -rf /var/run/cilium/state; rm -rf /sys/fs/bpf/tc/globals/cilium_*; fi'] + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + - name: cilium-run + mountPath: /var/run/cilium + env: + - name: "CLEAN_CILIUM_STATE" + valueFrom: + configMapKeyRef: + name: cilium-config + optional: true + key: clean-cilium-state containers: - - image: {{ cilium_image_repo }}:{{ cilium_image_tag }} - imagePullPolicy: Always - name: cilium-agent - command: [ "cilium-agent" ] - args: - - "--debug=$(CILIUM_DEBUG)" - - "-t" - - "vxlan" - - "--kvstore" - - "etcd" - - "--kvstore-opt" - - "etcd.config=/var/lib/etcd-config/etcd.config" - - "--disable-ipv4=$(DISABLE_IPV4)" + - image: {{ cilium_image_repo }}:{{ cilium_image_tag }} + imagePullPolicy: Always + name: cilium-agent + command: ["cilium-agent"] + args: + - "--debug=$(CILIUM_DEBUG)" + - "--kvstore=etcd" + - "--kvstore-opt=etcd.config=/var/lib/etcd-config/etcd.config" + - "--disable-ipv4=$(DISABLE_IPV4)" {% if cilium_enable_prometheus %} - ports: - - name: prometheus - containerPort: 9090 + ports: + - name: prometheus + containerPort: 9090 {% endif %} - lifecycle: - postStart: + lifecycle: + postStart: + exec: + command: + - "/cni-install.sh" + preStop: + exec: + command: + - "/cni-uninstall.sh" + env: + - name: "K8S_NODE_NAME" + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: "CILIUM_DEBUG" + valueFrom: + configMapKeyRef: + name: cilium-config + key: debug + - name: "DISABLE_IPV4" + valueFrom: + configMapKeyRef: + name: cilium-config + key: disable-ipv4 +{% if cilium_enable_prometheus %} + # Note: this variable is a no-op if not defined, and is used in the + # prometheus examples. + - name: "CILIUM_PROMETHEUS_SERVE_ADDR" + valueFrom: + configMapKeyRef: + name: cilium-metrics-config + optional: true + key: prometheus-serve-addr +{% endif %} + - name: "CILIUM_LEGACY_HOST_ALLOWS_WORLD" + valueFrom: + configMapKeyRef: + name: cilium-config + optional: true + key: legacy-host-allows-world + - name: "CILIUM_SIDECAR_ISTIO_PROXY_IMAGE" + valueFrom: + configMapKeyRef: + name: cilium-config + key: sidecar-istio-proxy-image + optional: true + - name: "CILIUM_TUNNEL" + valueFrom: + configMapKeyRef: + key: tunnel + name: cilium-config + optional: true + - name: "CILIUM_MONITOR_AGGREGATION_LEVEL" + valueFrom: + configMapKeyRef: + key: monitor-aggregation-level + name: cilium-config + optional: true + resources: + limits: + cpu: {{ cilium_cpu_limit }} + memory: {{ cilium_memory_limit }} + requests: + cpu: {{ cilium_cpu_requests }} + memory: {{ cilium_memory_requests }} + livenessProbe: exec: command: - - "/cni-install.sh" - preStop: + - cilium + - status + # The initial delay for the liveness probe is intentionally large to + # avoid an endless kill & restart cycle if in the event that the initial + # bootstrapping takes longer than expected. + initialDelaySeconds: 120 + failureThreshold: 10 + periodSeconds: 10 + readinessProbe: exec: command: - - "/cni-uninstall.sh" - env: - - name: "K8S_NODE_NAME" - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: "CILIUM_DEBUG" - valueFrom: - configMapKeyRef: - name: cilium-config - key: debug - - name: "DISABLE_IPV4" - valueFrom: - configMapKeyRef: - name: cilium-config - key: disable-ipv4 -{% if cilium_enable_prometheus %} - # Note: this variable is a no-op if not defined, and is used in the - # prometheus examples. - - name: "CILIUM_PROMETHEUS_SERVE_ADDR" - valueFrom: - configMapKeyRef: - name: cilium-metrics-config - optional: true - key: prometheus-serve-addr -{% endif %} - resources: - limits: - cpu: {{ cilium_cpu_limit }} - memory: {{ cilium_memory_limit }} - requests: - cpu: {{ cilium_cpu_requests }} - memory: {{ cilium_memory_requests }} - livenessProbe: - exec: - command: - - cilium - - status - # The initial delay for the liveness probe is intentionally large to - # avoid an endless kill & restart cycle if in the event that the initial - # bootstrapping takes longer than expected. - initialDelaySeconds: 120 - failureThreshold: 10 - periodSeconds: 10 - readinessProbe: - exec: - command: - - cilium - - status - initialDelaySeconds: 5 - periodSeconds: 5 - volumeMounts: - - name: bpf-maps - mountPath: /sys/fs/bpf - - name: cilium-run - mountPath: /var/run/cilium - - name: cni-path - mountPath: /host/opt/cni/bin - - name: etc-cni-netd - mountPath: /host/etc/cni/net.d - - name: docker-socket - mountPath: /var/run/docker.sock - readOnly: true - - name: etcd-config-path - mountPath: /var/lib/etcd-config - readOnly: true - - name: cilium-certs - mountPath: {{ cilium_cert_dir }} - readOnly: true - securityContext: - capabilities: - add: - - "NET_ADMIN" - privileged: true + - cilium + - status + initialDelaySeconds: 5 + periodSeconds: 5 + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + - name: cilium-run + mountPath: /var/run/cilium + - name: cni-path + mountPath: /host/opt/cni/bin + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: docker-socket + mountPath: /var/run/docker.sock + readOnly: true + - name: etcd-config-path + mountPath: /var/lib/etcd-config + readOnly: true + - name: cilium-certs + mountPath: {{ cilium_cert_dir }} + readOnly: true + securityContext: + capabilities: + add: + - "NET_ADMIN" + privileged: true hostNetwork: true volumes: - # To keep state between restarts / upgrades + # To keep state between restarts / upgrades - name: cilium-run hostPath: path: /var/run/cilium - # To keep state between restarts / upgrades + # To keep state between restarts / upgrades - name: bpf-maps hostPath: path: /sys/fs/bpf - # To read docker events from the node + # To read docker events from the node - name: docker-socket hostPath: path: /var/run/docker.sock - # To install cilium cni plugin in the host + # To install cilium cni plugin in the host - name: cni-path hostPath: path: /opt/cni/bin - # To install cilium cni configuration in the host + # To install cilium cni configuration in the host - name: etc-cni-netd hostPath: - path: /etc/cni/net.d - - name: cilium-certs - hostPath: - path: {{ cilium_cert_dir }} - # To read the etcd config stored in config maps + path: /etc/cni/net.d + # To read the etcd config stored in config maps - name: etcd-config-path configMap: name: cilium-config items: - - key: etcd-config - path: etcd.config + - key: etcd-config + path: etcd.config + # To read the k8s etcd secrets in case the user might want to use TLS + - name: cilium-certs + hostPath: + path: {{ cilium_cert_dir }} + + restartPolicy: Always tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - # Mark cilium's pod as critical for rescheduling - - key: CriticalAddonsOnly - operator: "Exists" + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + # Mark cilium's pod as critical for rescheduling + - key: CriticalAddonsOnly + operator: "Exists"