From a096761306d5ebafe550e0906d341ffbde753bd9 Mon Sep 17 00:00:00 2001 From: wangxf Date: Tue, 29 Jan 2019 03:03:49 +0800 Subject: [PATCH] [PR-Calico]Support calico 3.4.0 (#4102) * Suport calico 3.4.0 Signed-off-by: wangxf1987 * Remove symlink + cni conflist template when 3.3.0+, handle Canal, addition of install-cni: sidecar(3.3.0) or initontainer(3.4.0), KUBECONFIG_FILEPATH, calico_cert_dir, advertise cluster ips * scheduler.alpha.kubernetes.io/critical-pod deprecated since 1.12 --- .../group_vars/k8s-cluster/k8s-net-calico.yml | 3 + roles/download/defaults/main.yml | 8 +-- .../templates/calico-kube-controllers.yml.j2 | 17 ++++- roles/kubernetes/kubeadm/tasks/main.yml | 4 +- roles/network_plugin/calico/defaults/main.yml | 2 +- roles/network_plugin/calico/tasks/install.yml | 2 +- .../calico/templates/calico-cr.yml.j2 | 13 ++++ .../calico/templates/calico-node.yml.j2 | 65 +++++++++++++++++-- .../calico/templates/cni-calico.conflist.j2 | 8 +-- roles/network_plugin/canal/tasks/main.yml | 2 +- .../canal/templates/canal-node.yaml.j2 | 50 ++++++++++++++ .../canal/templates/cni-canal.conflist.j2 | 2 +- 12 files changed, 156 insertions(+), 20 deletions(-) diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml b/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml index 045bd21e0..21fd7994f 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml @@ -18,3 +18,6 @@ # not be specified in calico CNI config, so Calico will use built-in # defaults. The value should be a number, not a string. # calico_mtu: 1500 + +# Advertise Cluster IPs +# calico_advertise_cluster_ips: true diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 41b78266b..c7ba10706 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -44,10 +44,10 @@ kube_image_repo: "gcr.io/google-containers" # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download -calico_version: "v3.1.3" -calico_ctl_version: "v3.1.3" -calico_cni_version: "v3.1.3" -calico_policy_version: "v3.1.3" +calico_version: "v3.4.0" +calico_ctl_version: "v3.4.0" +calico_cni_version: "v3.4.0" +calico_policy_version: "v3.4.0" calico_rr_version: "v0.6.1" flannel_version: "v0.10.0" diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 index 1eb8e1d4d..1e7b8240d 100644 --- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 +++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: calico-kube-controllers @@ -9,7 +9,7 @@ metadata: spec: replicas: 1 strategy: - type: RollingUpdate + type: Recreate selector: matchLabels: kubernetes.io/cluster-service: "true" @@ -22,11 +22,15 @@ spec: kubernetes.io/cluster-service: "true" k8s-app: calico-kube-controllers spec: + nodeSelector: + beta.kubernetes.io/os: linux hostNetwork: true serviceAccountName: calico-kube-controllers tolerations: - - effect: NoSchedule + - key: CriticalAddonsOnly operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule {% if kube_version is version('v1.11.1', '>=') %} priorityClassName: system-cluster-critical {% endif %} @@ -50,6 +54,13 @@ spec: value: "{{ calico_cert_dir }}/cert.crt" - name: ETCD_KEY_FILE value: "{{ calico_cert_dir }}/key.pem" +{% if calico_version is version('v3.3.0', '>=') %} + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r +{% endif %} volumeMounts: - mountPath: {{ calico_cert_dir }} name: etcd-certs diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 2fbfac851..ba3991515 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -99,7 +99,9 @@ dest: "{{ kube_config_dir }}/node-kubeconfig.yaml" state: link force: yes - when: kube_network_plugin in ['calico','canal'] + when: + - kube_network_plugin in ['calico','canal'] + - calico_version is version('v3.3.0', '<') # FIXME(jjo): need to post-remove kube-proxy until https://github.com/kubernetes/kubeadm/issues/776 # is fixed diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml index 45a2bda06..f53394489 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico/defaults/main.yml @@ -32,7 +32,7 @@ calico_node_cpu_requests: 150m calicoctl_memory_limit: 170M calicoctl_cpu_limit: 100m calicoctl_memory_requests: 32M -calicoctl_cpu_requests: 50m +calicoctl_cpu_requests: 250m # Enable Prometheus Metrics endpoint for felix calico_felix_prometheusmetricsenabled: "false" diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index 583ac0eb3..69611c33f 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -2,7 +2,7 @@ - name: Calico | Write Calico cni config template: src: "cni-calico.conflist.j2" - dest: "/etc/cni/net.d/10-calico.conflist" + dest: "/etc/cni/net.d/{% if calico_version is version('v3.3.0', '>=') %}calico.conflist.template{% else %}10-calico.conflist{% endif %}" owner: kube - name: Calico | Create calico certs directory diff --git a/roles/network_plugin/calico/templates/calico-cr.yml.j2 b/roles/network_plugin/calico/templates/calico-cr.yml.j2 index 41d4f2b06..b32a16745 100644 --- a/roles/network_plugin/calico/templates/calico-cr.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-cr.yml.j2 @@ -9,8 +9,21 @@ rules: resources: - pods - nodes + - namespaces verbs: - get + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + - watch + - list + - apiGroups: [""] + resources: + - nodes/status + verbs: + - patch - apiGroups: - policy resourceNames: diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2 index 329ad87dd..096f8f539 100644 --- a/roles/network_plugin/calico/templates/calico-node.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-node.yml.j2 @@ -31,14 +31,62 @@ spec: hostNetwork: true serviceAccountName: calico-node tolerations: - - operator: Exists + - effect: NoExecute + operator: Exists + - effect: NoSchedule + operator: Exists # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - key: CriticalAddonsOnly operator: "Exists" # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 +{% if calico_version is version('v3.4.0', '>=') %} + initContainers: + # This container installs the Calico CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # CNI binaries are already on the host + - name: UPDATE_CNI_BINARIES + value: "false" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG_FILE + value: "/host/etc/cni/net.d/calico.conflist.template" + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/etc/cni/net.d + name: cni-net-dir +{% endif %} containers: +{% if calico_version is version('v3.3.0', '>=') and calico_version is version('v3.4.0', '<') %} + - name: install-cni + image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # CNI binaries are already on the host + - name: UPDATE_CNI_BINARIES + value: "false" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG_FILE + value: "/host/etc/cni/net.d/calico.conflist.template" + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/etc/cni/net.d + name: cni-net-dir +{% endif %} # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each # host. @@ -107,6 +155,10 @@ spec: value: "{{ calico_felix_prometheusgometricsenabled }}" - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED value: "{{ calico_felix_prometheusprocessmetricsenabled }}" +{% if calico_version is version('v3.4.0', '>=') and calico_advertise_cluster_ips|default(false) %} + - name: CALICO_ADVERTISE_CLUSTER_IPS + value: "{{ kube_service_addresses }}" +{% endif %} # Location of the CA certificate for etcd. - name: ETCD_CA_CERT_FILE valueFrom: @@ -162,10 +214,18 @@ spec: initialDelaySeconds: 10 failureThreshold: 6 readinessProbe: +{% if calico_version is version('v3.3.0', '<') %} httpGet: host: 127.0.0.1 path: /readiness port: 9099 +{% else %} + exec: + command: + - /bin/calico-node + - -bird-ready + - -felix-ready +{% endif %} periodSeconds: 10 volumeMounts: - mountPath: /lib/modules @@ -193,9 +253,6 @@ spec: hostPath: path: /var/lib/calico # Used to install CNI. - - name: cni-bin-dir - hostPath: - path: /opt/cni/bin - name: cni-net-dir hostPath: path: /etc/cni/net.d diff --git a/roles/network_plugin/calico/templates/cni-calico.conflist.j2 b/roles/network_plugin/calico/templates/cni-calico.conflist.j2 index e84ab7d61..9ce2483cc 100644 --- a/roles/network_plugin/calico/templates/cni-calico.conflist.j2 +++ b/roles/network_plugin/calico/templates/cni-calico.conflist.j2 @@ -10,9 +10,9 @@ {% endif %} "type": "calico", "etcd_endpoints": "{{ etcd_access_addresses }}", - "etcd_cert_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem", - "etcd_key_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem", - "etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem", + "etcd_cert_file": "{{ calico_cert_dir }}/cert.crt", + "etcd_key_file": "{{ calico_cert_dir }}/key.pem", + "etcd_ca_cert_file": "{{ calico_cert_dir }}/ca_cert.crt", "log_level": "info", "ipam": { "type": "calico-ipam", @@ -28,7 +28,7 @@ "mtu": {{ calico_mtu }}, {%- endif %} "kubernetes": { - "kubeconfig": "{{ kube_config_dir }}/node-kubeconfig.yaml" + "kubeconfig": "{% if calico_version is version('v3.3.0', '>=') %}__KUBECONFIG_FILEPATH__{% else %}{{ kube_config_dir }}/node-kubeconfig.yaml{% endif %}" } }, { diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml index d59c818fe..db4764f2a 100644 --- a/roles/network_plugin/canal/tasks/main.yml +++ b/roles/network_plugin/canal/tasks/main.yml @@ -2,7 +2,7 @@ - name: Canal | Write Canal cni config template: src: "cni-canal.conflist.j2" - dest: "/etc/cni/net.d/10-canal.conflist" + dest: "/etc/cni/net.d/{% if calico_version is version('v3.3.0', '>=') %}canal.conflist.template{% else %}10-canal.conflist{% endif %}" owner: kube - name: Canal | Create canal certs directory diff --git a/roles/network_plugin/canal/templates/canal-node.yaml.j2 b/roles/network_plugin/canal/templates/canal-node.yaml.j2 index f144c39ed..21e6f5fd1 100644 --- a/roles/network_plugin/canal/templates/canal-node.yaml.j2 +++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2 @@ -57,7 +57,49 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate +{% if calico_version is version('v3.4.0', '>=') %} + initContainers: + # This container installs the Calico CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-canal.conflist" + # CNI binaries are already on the host + - name: UPDATE_CNI_BINARIES + value: "false" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG_FILE + value: "/host/etc/cni/net.d/canal.conflist.template" + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/etc/cni/net.d + name: cni-net-dir +{% endif %} containers: +{% if calico_version is version('v3.3.0', '>=') and calico_version is version('v3.4.0', '<') %} + - name: install-cni + image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-canal.conflist" + # CNI binaries are already on the host + - name: UPDATE_CNI_BINARIES + value: "false" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG_FILE + value: "/host/etc/cni/net.d/canal.conflist.template" + volumeMounts: + - mountPath: /host/etc/cni/net.d + name: cni-net-dir +{% endif %} # Runs the flannel daemon to enable vxlan networking between # container hosts. - name: flannel @@ -209,9 +251,17 @@ spec: initialDelaySeconds: 10 failureThreshold: 6 readinessProbe: +{% if calico_version is version('v3.3.0', '<')%} httpGet: path: /readiness port: 9099 +{% else %} + exec: + command: + - /bin/calico-node + - -bird-ready + - -felix-ready +{% endif %} periodSeconds: 10 volumeMounts: - mountPath: /lib/modules diff --git a/roles/network_plugin/canal/templates/cni-canal.conflist.j2 b/roles/network_plugin/canal/templates/cni-canal.conflist.j2 index 13b58a0db..fc363ee81 100644 --- a/roles/network_plugin/canal/templates/cni-canal.conflist.j2 +++ b/roles/network_plugin/canal/templates/cni-canal.conflist.j2 @@ -15,7 +15,7 @@ "type": "k8s" }, "kubernetes": { - "kubeconfig": "{{ kube_config_dir }}/node-kubeconfig.yaml" + "kubeconfig": "{% if calico_version is version('v3.3.0', '>=') %}__KUBECONFIG_FILEPATH__{% else %}{{ kube_config_dir }}/node-kubeconfig.yaml{% endif %}" } } },