From 82119ca9233c89e5663ac8d31eb8a7e51f93b28a Mon Sep 17 00:00:00 2001 From: grialeyur <49140147+grialeyur@users.noreply.github.com> Date: Thu, 25 Apr 2019 16:00:48 +0400 Subject: [PATCH] Add support calico kubernetes datastore and typha. (#4498) * Add support calico kubernetes datastore and typha. * Add typha_enabled to kubespray-defaults. --- .../group_vars/k8s-cluster/k8s-net-calico.yml | 9 ++ roles/download/defaults/main.yml | 12 ++ .../network_plugin/calico/tasks/main.yml | 2 + roles/kubespray-defaults/defaults/main.yaml | 2 + roles/network_plugin/calico/defaults/main.yml | 7 + roles/network_plugin/calico/tasks/install.yml | 22 +++ .../calico/templates/calico-config.yml.j2 | 7 + .../calico/templates/calico-cr.yml.j2 | 75 ++++++++++ .../calico/templates/calico-node.yml.j2 | 69 ++++++--- .../calico/templates/calico-typha.yml.j2 | 123 ++++++++++++++++ .../calico/templates/cni-calico.conflist.j2 | 11 +- .../calico/templates/kdd-crds.yml.j2 | 133 ++++++++++++++++++ 12 files changed, 453 insertions(+), 19 deletions(-) create mode 100644 roles/network_plugin/calico/templates/calico-typha.yml.j2 create mode 100644 roles/network_plugin/calico/templates/kdd-crds.yml.j2 diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml b/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml index 9d2654c76..1c938de19 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml @@ -24,3 +24,12 @@ # Advertise Cluster IPs # calico_advertise_cluster_ips: true + +# Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore) +# calico_datastore: "etcd" + +# Use typha (only with kdd) +# typha_enabled: false + +# Number of typha replicas +# typha_replicas: 1 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index c483a5e3f..a5080d97e 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -49,6 +49,7 @@ calico_ctl_version: "v3.4.4" calico_cni_version: "v3.4.0" calico_policy_version: "v3.4.0" calico_rr_version: "v0.6.1" +calico_typha_version: "v3.4.4" flannel_version: "v0.11.0" flannel_cni_version: "v0.3.0" @@ -181,6 +182,8 @@ calico_policy_image_repo: "docker.io/calico/kube-controllers" calico_policy_image_tag: "{{ calico_policy_version }}" calico_rr_image_repo: "docker.io/calico/routereflector" calico_rr_image_tag: "{{ calico_rr_version }}" +calico_typha_image_repo: "docker.io/calico/typha" +calico_typha_image_tag: "{{ calico_typha_version }}" pod_infra_image_repo: "gcr.io/google_containers/pause-{{ image_arch }}" pod_infra_image_tag: "{{ pod_infra_version }}" install_socat_image_repo: "docker.io/xueshanf/install-socat" @@ -434,6 +437,15 @@ downloads: groups: - calico-rr + calico_typha: + enabled: "{{ typha_enabled == 'calico' }}" + container: true + repo: "{{ calico_typha_image_repo }}" + tag: "{{ calico_typha_image_tag }}" + sha256: "{{ calico_typha_digest_checksum|default(None) }}" + groups: + - k8s-cluster + weave_kube: enabled: "{{ kube_network_plugin == 'weave' }}" container: true diff --git a/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml b/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml index 1be16776f..d82da98ae 100644 --- a/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml +++ b/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml @@ -9,6 +9,8 @@ state: "latest" with_items: - "{{ calico_node_manifests.results }}" + - "{{ calico_node_kdd_manifest.results }}" + - "{{ calico_node_typha_manifest.results }}" when: - inventory_hostname == groups['kube-master'][0] and not item is skipped loop_control: diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 9084c5224..5cdd143e3 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -501,3 +501,5 @@ pip_extra_args: |- etcd_config_dir: /etc/ssl/etcd etcd_cert_dir: "{{ etcd_config_dir }}/ssl" + +typha_enabled: false diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml index 3ee4b1b29..d010bcebe 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico/defaults/main.yml @@ -65,3 +65,10 @@ calico_upgrade_needed: false kube_etcd_cacert_file: ca.pem kube_etcd_cert_file: node-{{ inventory_hostname }}.pem kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem + +# Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore) +calico_datastore: "etcd" +# Use typha (only with kdd) +typha_enabled: false +# Number of typha replicas +typha_replicas: 1 diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index 67b006349..321669add 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -350,3 +350,25 @@ when: - inventory_hostname in groups['kube-master'] - rbac_enabled or item.type not in rbac_resources + +- name: Calico | Create calico manifests for kdd + template: + src: "{{item.file}}.j2" + dest: "{{kube_config_dir}}/{{item.file}}" + with_items: + - {name: calico, file: kdd-crds.yml, type: kdd} + register: calico_node_kdd_manifest + when: + - inventory_hostname in groups['kube-master'] + - calico_datastore == "kdd" + +- name: Calico | Create calico manifests for typha + template: + src: "{{item.file}}.j2" + dest: "{{kube_config_dir}}/{{item.file}}" + with_items: + - {name: calico, file: calico-typha.yml, type: typha} + register: calico_node_typha_manifest + when: + - inventory_hostname in groups['kube-master'] + - typha_enabled and calico_datastore == "kdd" diff --git a/roles/network_plugin/calico/templates/calico-config.yml.j2 b/roles/network_plugin/calico/templates/calico-config.yml.j2 index dc942d850..550e7b7e1 100644 --- a/roles/network_plugin/calico/templates/calico-config.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-config.yml.j2 @@ -4,10 +4,17 @@ metadata: name: calico-config namespace: kube-system data: +{% if calico_datastore == "etcd" %} etcd_endpoints: "{{ etcd_access_addresses }}" etcd_ca: "/calico-secrets/ca_cert.crt" etcd_cert: "/calico-secrets/cert.crt" etcd_key: "/calico-secrets/key.pem" +{% elif calico_datastore == "kdd" and typha_enabled == "true" %} + # To enable Typha, set this to "calico-typha" *and* set a non-zero value for Typha replicas + # below. We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is + # essential. + typha_service_name: "calico-typha" +{% endif %} {% if calico_network_backend is defined and calico_network_backend == 'none' %} cluster_type: "kubespray" calico_backend: "none" diff --git a/roles/network_plugin/calico/templates/calico-cr.yml.j2 b/roles/network_plugin/calico/templates/calico-cr.yml.j2 index b32a16745..64d953585 100644 --- a/roles/network_plugin/calico/templates/calico-cr.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-cr.yml.j2 @@ -19,11 +19,16 @@ rules: verbs: - watch - list +{% if calico_datastore == "kdd" %} + # Used to discover Typhas. + - get +{% endif %} - apiGroups: [""] resources: - nodes/status verbs: - patch +{% if calico_datastore == "etcd" %} - apiGroups: - policy resourceNames: @@ -32,3 +37,73 @@ rules: - podsecuritypolicies verbs: - use +{% elif calico_datastore == "kdd" %} + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - clusterinformations + - hostendpoints + verbs: + - get + - list + - watch + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only requried for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update +{% endif %} diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2 index d8ea78a44..cd4841867 100644 --- a/roles/network_plugin/calico/templates/calico-node.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-node.yml.j2 @@ -61,6 +61,13 @@ spec: # Prevents the container from sleeping forever. - name: SLEEP value: "false" +{% if calico_datastore == "kdd" %} + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName +{% endif %} volumeMounts: - mountPath: /host/etc/cni/net.d name: cni-net-dir @@ -98,11 +105,51 @@ spec: image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }} env: # The location of the Calico etcd cluster. +{% if calico_datastore == "etcd" %} - name: ETCD_ENDPOINTS valueFrom: configMapKeyRef: name: calico-config key: etcd_endpoints + # Location of the CA certificate for etcd. + - name: ETCD_CA_CERT_FILE + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_ca + # Location of the client key for etcd. + - name: ETCD_KEY_FILE + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_key + # Location of the client certificate for etcd. + - name: ETCD_CERT_FILE + valueFrom: + configMapKeyRef: + name: calico-config + key: etcd_cert +{% elif calico_datastore == "kdd" %} + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + {% if typha_enabled == "true" %} + # Typha support: controlled by the ConfigMap. + - name: FELIX_TYPHAK8SSERVICENAME + valueFrom: + configMapKeyRef: + name: calico-config + key: typha_service_name + {% endif %} + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName +{% endif %} # Choose the backend to use. - name: CALICO_NETWORKING_BACKEND valueFrom: @@ -167,24 +214,6 @@ spec: - name: CALICO_ADVERTISE_CLUSTER_IPS value: "{{ kube_service_addresses }}" {% endif %} - # Location of the CA certificate for etcd. - - name: ETCD_CA_CERT_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_ca - # Location of the client key for etcd. - - name: ETCD_KEY_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_key - # Location of the client certificate for etcd. - - name: ETCD_CERT_FILE - valueFrom: - configMapKeyRef: - name: calico-config - key: etcd_cert {% if calico_ip_auto_method is defined %} - name: IP_AUTODETECTION_METHOD value: "{{ calico_ip_auto_method }}" @@ -243,8 +272,10 @@ spec: - mountPath: /var/lib/calico name: var-lib-calico readOnly: false +{% if calico_datastore == "etcd" %} - mountPath: /calico-secrets name: etcd-certs +{% endif %} - name: xtables-lock mountPath: /run/xtables.lock readOnly: false @@ -266,10 +297,12 @@ spec: - name: cni-bin-dir hostPath: path: /opt/cni/bin +{% if calico_datastore == "etcd" %} # Mount in the etcd TLS secrets. - name: etcd-certs hostPath: path: "{{ calico_cert_dir }}" +{% endif %} # Mount the global iptables lock file, used by calico/node - name: xtables-lock hostPath: diff --git a/roles/network_plugin/calico/templates/calico-typha.yml.j2 b/roles/network_plugin/calico/templates/calico-typha.yml.j2 new file mode 100644 index 000000000..19e5ec894 --- /dev/null +++ b/roles/network_plugin/calico/templates/calico-typha.yml.j2 @@ -0,0 +1,123 @@ +# This manifest creates a Service, which will be backed by Calico's Typha daemon. +# Typha sits in between Felix and the API server, reducing Calico's load on the API server. + +apiVersion: v1 +kind: Service +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + ports: + - port: 5473 + protocol: TCP + targetPort: calico-typha + name: calico-typha + selector: + k8s-app: calico-typha + +--- + +# This manifest creates a Deployment of Typha to back the above service. + +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the + # typha_service_name variable in the calico-config ConfigMap above. + # + # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential + # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In + # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. + replicas: {{ typha_replicas }} + revisionHistoryLimit: 2 + template: + metadata: + labels: + k8s-app: calico-typha + annotations: + # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical + # add-on, ensuring it gets priority scheduling and that its resources are reserved + # if it ever gets evicted. + scheduler.alpha.kubernetes.io/critical-pod: '' + cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' + spec: + nodeSelector: + beta.kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + # Since Calico can't network a pod until Typha is up, we need to run Typha itself + # as a host-networked pod. + serviceAccountName: calico-node + containers: +# - image: calico/typha:v3.4.4 + - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }} + name: calico-typha + ports: + - containerPort: 5473 + name: calico-typha + protocol: TCP + env: + # Enable "info" logging by default. Can be set to "debug" to increase verbosity. + - name: TYPHA_LOGSEVERITYSCREEN + value: "info" + # Disable logging to file and syslog since those don't make sense in Kubernetes. + - name: TYPHA_LOGFILEPATH + value: "none" + - name: TYPHA_LOGSEVERITYSYS + value: "none" + # Monitor the Kubernetes API to find the number of running instances and rebalance + # connections. + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" + - name: TYPHA_DATASTORETYPE + value: "kubernetes" + - name: TYPHA_HEALTHENABLED + value: "true" + # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, + # this opens a port on the host, which may need to be secured. + #- name: TYPHA_PROMETHEUSMETRICSENABLED + # value: "true" + #- name: TYPHA_PROMETHEUSMETRICSPORT + # value: "9093" + livenessProbe: + exec: + command: + - calico-typha + - check + - liveness + periodSeconds: 30 + initialDelaySeconds: 30 + readinessProbe: + exec: + command: + - calico-typha + - check + - readiness + periodSeconds: 10 + +--- + +# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-typha diff --git a/roles/network_plugin/calico/templates/cni-calico.conflist.j2 b/roles/network_plugin/calico/templates/cni-calico.conflist.j2 index be76da0ab..f9c50e133 100644 --- a/roles/network_plugin/calico/templates/cni-calico.conflist.j2 +++ b/roles/network_plugin/calico/templates/cni-calico.conflist.j2 @@ -9,16 +9,25 @@ "nodename": "{{ calico_baremetal_nodename }}", {% endif %} "type": "calico", + "log_level": "info", +{% if calico_datastore == "etcd" %} "etcd_endpoints": "{{ etcd_access_addresses }}", "etcd_cert_file": "{{ calico_cert_dir }}/cert.crt", "etcd_key_file": "{{ calico_cert_dir }}/key.pem", "etcd_ca_cert_file": "{{ calico_cert_dir }}/ca_cert.crt", - "log_level": "info", "ipam": { "type": "calico-ipam", "assign_ipv4": "true", "ipv4_pools": ["{{ calico_pool_cidr | default(kube_pods_subnet) }}"] }, +{% elif calico_datastore == "kdd" %} + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "ipam": { + "type": "host-local", + "subnet": "usePodCidr" + }, +{% endif %} {% if enable_network_policy %} "policy": { "type": "k8s" diff --git a/roles/network_plugin/calico/templates/kdd-crds.yml.j2 b/roles/network_plugin/calico/templates/kdd-crds.yml.j2 new file mode 100644 index 000000000..3f14729f4 --- /dev/null +++ b/roles/network_plugin/calico/templates/kdd-crds.yml.j2 @@ -0,0 +1,133 @@ +# Create all the CustomResourceDefinitions needed for + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: felixconfigurations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: FelixConfiguration + plural: felixconfigurations + singular: felixconfiguration +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: bgppeers.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BGPPeer + plural: bgppeers + singular: bgppeer + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: bgpconfigurations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BGPConfiguration + plural: bgpconfigurations + singular: bgpconfiguration + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPPool + plural: ippools + singular: ippool + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: HostEndpoint + plural: hostendpoints + singular: hostendpoint + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: ClusterInformation + plural: clusterinformations + singular: clusterinformation + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkPolicy + plural: globalnetworkpolicies + singular: globalnetworkpolicy + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkSet + plural: globalnetworksets + singular: globalnetworkset + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org +spec: + scope: Namespaced + group: crd.projectcalico.org + version: v1 + names: + kind: NetworkPolicy + plural: networkpolicies + singular: networkpolicy