diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml index 6ff54714e..4117d1ca3 100644 --- a/roles/network_plugin/canal/tasks/main.yml +++ b/roles/network_plugin/canal/tasks/main.yml @@ -28,10 +28,28 @@ - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} +- name: Slurp etcd cacert file + slurp: + src: "{{ canal_cert_dir }}/ca_cert.crt" + register: etcd_ca_cert_file + failed_when: false + +- name: Slurp etcd cert file + slurp: + src: "{{ canal_cert_dir }}/cert.crt" + register: etcd_cert_file + failed_when: false + +- name: Slurp etcd key file + slurp: + src: "{{ canal_cert_dir }}/key.pem" + register: etcd_key_file + failed_when: false + # Flannel need etcd v2 API - name: Canal | Set Flannel etcd configuration command: |- - {{ bin_dir }}/etcdctl set /{{ cluster_name }}/network/config \ + {{ bin_dir }}/etcdctl set /coreos.com/network/config \ '{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }' register: output retries: 4 @@ -53,14 +71,17 @@ dest: "{{ kube_config_dir }}/{{ item.file }}" mode: 0644 with_items: + - {name: canal-calico-etcd-secret, file: canal-secret-calico-etcd.yml, type: secret} - {name: canal-config, file: canal-config.yaml, type: cm} - {name: canal-node, file: canal-node.yaml, type: ds} - {name: canal-kube-controllers, file: canal-calico-kube-controllers.yml, type: deployment} + - {name: canal-cr, file: canal-cr.yml, type: clusterrole} - {name: canal, file: canal-node-sa.yml, type: sa} - {name: calico-cr, file: canal-cr-calico-node.yml, type: clusterrole} - {name: calico-kube-cr, file: canal-cr-calico-kube-controllers.yml, type: clusterrole} - {name: calico-crd, file: canal-crd-calico.yml, type: crd} - {name: flannel, file: canal-cr-flannel.yml, type: clusterrole} + - {name: canal, file: canal-crb-canal.yml, type: clusterrolebinding} - {name: canal-calico, file: canal-crb-calico.yml, type: clusterrolebinding} - {name: canal-flannel, file: canal-crb-flannel.yml, type: clusterrolebinding} register: canal_manifests diff --git a/roles/network_plugin/canal/templates/canal-calico-kube-controllers.yml.j2 b/roles/network_plugin/canal/templates/canal-calico-kube-controllers.yml.j2 index 6ce75068d..1417022a8 100644 --- a/roles/network_plugin/canal/templates/canal-calico-kube-controllers.yml.j2 +++ b/roles/network_plugin/canal/templates/canal-calico-kube-controllers.yml.j2 @@ -33,15 +33,45 @@ spec: effect: NoSchedule serviceAccountName: calico-kube-controllers priorityClassName: system-cluster-critical + # The controllers must run in the host network namespace so that + # it isn't governed by policy that would prevent it from working. + hostNetwork: true containers: - name: calico-kube-controllers image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} env: + # The location of the etcd cluster. + - name: ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_endpoints + # Location of the CA certificate for etcd. + - name: ETCD_CA_CERT_FILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_ca + # Location of the client key for etcd. + - name: ETCD_KEY_FILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_key + # Location of the client certificate for etcd. + - name: ETCD_CERT_FILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_cert # Choose which controllers to run. - name: ENABLED_CONTROLLERS - value: node - - name: DATASTORE_TYPE - value: kubernetes + value: policy,namespace,serviceaccount,workloadendpoint,node + volumeMounts: + # Mount in the etcd TLS secrets. + - mountPath: /calico-secrets + name: etcd-certs livenessProbe: exec: command: @@ -57,3 +87,10 @@ spec: - /usr/bin/check-status - -r periodSeconds: 10 + volumes: + # Mount in the etcd TLS secrets with mode 400. + # See https://kubernetes.io/docs/concepts/configuration/secret/ + - name: etcd-certs + secret: + secretName: calico-etcd-secrets + defaultMode: 0440 diff --git a/roles/network_plugin/canal/templates/canal-config.yaml.j2 b/roles/network_plugin/canal/templates/canal-config.yaml.j2 index 9bac59dca..8aab6fb75 100644 --- a/roles/network_plugin/canal/templates/canal-config.yaml.j2 +++ b/roles/network_plugin/canal/templates/canal-config.yaml.j2 @@ -7,6 +7,14 @@ metadata: name: canal-config namespace: kube-system data: + # Configure this with the location of your etcd cluster. + etcd_endpoints: "{{ etcd_access_addresses }}" + # If you're using TLS enabled etcd uncomment the following. + # You must also populate the Secret below with these files. + etcd_ca: "/calico-secrets/etcd-ca" + etcd_cert: "/calico-secrets/etcd-cert" + etcd_key: "/calico-secrets/etcd-key" + # Typha is disabled. typha_service_name: "none" @@ -28,41 +36,39 @@ data: # values in this config will be automatically populated. cni_network_config: |- { - "name": "k8s-pod-network", - "cniVersion": "0.3.1", - "plugins": [ - { - "type": "calico", - "log_level": "info", + "name": "canal", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "flannel", + "delegate": { + "type": "calico", + "include_default_routes": true, + "etcd_endpoints": "__ETCD_ENDPOINTS__", + "etcd_key_file": "__ETCD_KEY_FILE__", + "etcd_cert_file": "__ETCD_CERT_FILE__", + "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__", + "log_level": "info", {% if calico_cni_log_file_path %} - "log_file_path": "{{ calico_cni_log_file_path }}", + "log_file_path": "{{ calico_cni_log_file_path }}", {% endif %} - "datastore_type": "kubernetes", - "nodename": "__KUBERNETES_NODE_NAME__", - "mtu": __CNI_MTU__, - "ipam": { - "type": "host-local", - "subnet": "usePodCidr" - }, - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - }, - { - "type": "portmap", - "snat": true, - "capabilities": {"portMappings": true} - }, - { - "type": "bandwidth", - "capabilities": {"bandwidth": true} - } - ] + "policy": { + "type": "k8s", + "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__", + "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + } + }, + { + "type": "portmap", + "capabilities": {"portMappings": true}, + "snat": true + } + ] } - # Flannel network configuration. Mounted into the flannel container. net-conf.json: | { @@ -71,3 +77,4 @@ data: "Type": "vxlan" } } + diff --git a/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2 b/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2 index 9cb8e7ff8..b2236d161 100644 --- a/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2 +++ b/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2 @@ -1,4 +1,4 @@ ---- +# Flannel ClusterRole # Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 diff --git a/roles/network_plugin/canal/templates/canal-cr.yml.j2 b/roles/network_plugin/canal/templates/canal-cr.yml.j2 new file mode 100644 index 000000000..1209c7b1a --- /dev/null +++ b/roles/network_plugin/canal/templates/canal-cr.yml.j2 @@ -0,0 +1,30 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: canal +rules: + # Used for creating service account tokens to be used by the CNI plugin + - apiGroups: [""] + resources: + - serviceaccounts/token + verbs: + - create + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + # Pod CIDR auto-detection on kubeadm needs access to config maps. + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list diff --git a/roles/network_plugin/canal/templates/canal-crb-canal.yml.j2 b/roles/network_plugin/canal/templates/canal-crb-canal.yml.j2 new file mode 100644 index 000000000..9fcb0fcf6 --- /dev/null +++ b/roles/network_plugin/canal/templates/canal-crb-canal.yml.j2 @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: canal +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: canal +subjects: +- kind: ServiceAccount + name: canal + namespace: kube-system diff --git a/roles/network_plugin/canal/templates/canal-node-sa.yml.j2 b/roles/network_plugin/canal/templates/canal-node-sa.yml.j2 index 582d55bbd..954f6d78e 100644 --- a/roles/network_plugin/canal/templates/canal-node-sa.yml.j2 +++ b/roles/network_plugin/canal/templates/canal-node-sa.yml.j2 @@ -4,3 +4,9 @@ kind: ServiceAccount metadata: name: canal namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + namespace: kube-system diff --git a/roles/network_plugin/canal/templates/canal-node.yaml.j2 b/roles/network_plugin/canal/templates/canal-node.yaml.j2 index 737be705e..529d4b967 100644 --- a/roles/network_plugin/canal/templates/canal-node.yaml.j2 +++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2 @@ -44,6 +44,7 @@ spec: # and CNI network config file on each node. - name: install-cni image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} command: ["/opt/cni/bin/install"] envFrom: - configMapRef: @@ -71,6 +72,30 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + # The location of the etcd cluster. + - name: ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_endpoints + # Location of the CA certificate for etcd. + - name: ETCD_CA_CERT_FILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_ca + # Location of the client key for etcd. + - name: ETCD_KEY_FILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_key + # Location of the client certificate for etcd. + - name: ETCD_CERT_FILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_cert # CNI MTU Config variable - name: CNI_MTU valueFrom: @@ -85,6 +110,8 @@ spec: name: cni-bin-dir - mountPath: /host/etc/cni/net.d name: cni-net-dir + - mountPath: /calico-secrets + name: etcd-certs securityContext: privileged: true # This init container mounts the necessary filesystems needed by the BPF data plane @@ -125,17 +152,32 @@ spec: name: kubernetes-services-endpoint optional: true env: - # Use Kubernetes API as the backing datastore. - - name: DATASTORE_TYPE - value: "kubernetes" - # Configure route aggregation based on pod CIDR. - - name: USE_POD_CIDR - value: "true" - # Wait for the datastore. - - name: WAIT_FOR_DATASTORE - value: "true" - # Set based on the k8s node name. - - name: NODENAME + # The location of the etcd cluster. + - name: ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_endpoints + # Location of the CA certificate for etcd. + - name: ETCD_CA_CERT_FILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_ca + # Location of the client key for etcd. + - name: ETCD_KEY_FILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_key + # Location of the client certificate for etcd. + - name: ETCD_CERT_FILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_cert + # Set noderef for node controller. + - name: CALICO_K8S_NODE_REF valueFrom: fieldRef: fieldPath: spec.nodeName @@ -221,6 +263,8 @@ spec: - mountPath: /var/lib/calico name: var-lib-calico readOnly: false + - mountPath: /calico-secrets + name: etcd-certs - name: policysync mountPath: /var/run/nodeagent # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the @@ -230,22 +274,70 @@ spec: - name: cni-log-dir mountPath: /var/log/calico/cni readOnly: true - # This container runs flannel using the kube-subnet-mgr backend - # for allocating subnets. - - name: kube-flannel + # Runs the flannel daemon to enable vxlan networking between + # container hosts. + - name: flannel image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}" + command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"] imagePullPolicy: {{ k8s_image_pull_policy }} - command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] - securityContext: - privileged: true - resources: - limits: - cpu: {{ flannel_cpu_limit }} - memory: {{ flannel_memory_limit }} - requests: - cpu: {{ flannel_cpu_requests }} - memory: {{ flannel_memory_requests }} env: + # The location of the etcd cluster. + - name: FLANNELD_ETCD_ENDPOINTS + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_endpoints + # Location of the CA certificate for etcd. + - name: ETCD_CA_CERT_FILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_ca + # Location of the client key for etcd. + - name: ETCD_KEY_FILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_key + # Location of the client certificate for etcd. + - name: ETCD_CERT_FILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_cert + # Location of the CA certificate for etcd. + - name: FLANNELD_ETCD_CAFILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_ca + # Location of the client key for etcd. + - name: FLANNELD_ETCD_KEYFILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_key + # Location of the client certificate for etcd. + - name: FLANNELD_ETCD_CERTFILE + valueFrom: + configMapKeyRef: + name: canal-config + key: etcd_cert + # The interface flannel should run on. + - name: FLANNELD_IFACE + valueFrom: + configMapKeyRef: + name: canal-config + key: canal_iface + # Perform masquerade on traffic leaving the pod cidr. + - name: FLANNELD_IP_MASQ + valueFrom: + configMapKeyRef: + name: canal-config + key: masquerade + # Write the subnet.env file to the mounted directory. + - name: FLANNELD_SUBNET_FILE + value: "/run/flannel/subnet.env" - name: POD_NAME valueFrom: fieldRef: @@ -254,24 +346,22 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: FLANNELD_IFACE - valueFrom: - configMapKeyRef: - name: canal-config - key: canal_iface - - name: FLANNELD_IP_MASQ - valueFrom: - configMapKeyRef: - name: canal-config - key: masquerade + securityContext: + privileged: true volumeMounts: - - mountPath: /run/xtables.lock - name: xtables-lock - readOnly: false - - name: flannel-cfg - mountPath: /etc/kube-flannel/ + - mountPath: /etc/resolv.conf + name: resolv + - mountPath: /run/flannel + name: run-flannel + - mountPath: /calico-secrets + name: etcd-certs + - name: flannel-cfg + mountPath: /etc/kube-flannel/ volumes: - # Used by canal. + - name: flannel-cfg + configMap: + name: canal-config + # Used by canal-node. - name: lib-modules hostPath: path: /lib/modules @@ -298,9 +388,12 @@ spec: hostPath: path: /proc # Used by flannel. - - name: flannel-cfg - configMap: - name: canal-config + - name: run-flannel + hostPath: + path: /run/flannel + - name: resolv + hostPath: + path: /etc/resolv.conf # Used to install CNI. - name: cni-bin-dir hostPath: @@ -312,6 +405,12 @@ spec: - name: cni-log-dir hostPath: path: /var/log/calico/cni + # Mount in the etcd TLS secrets with mode 400. + # See https://kubernetes.io/docs/concepts/configuration/secret/ + - name: etcd-certs + secret: + secretName: calico-etcd-secrets + defaultMode: 0400 # Used to create per-pod Unix Domain Sockets - name: policysync hostPath: diff --git a/roles/network_plugin/canal/templates/canal-secret-calico-etcd.yml.j2 b/roles/network_plugin/canal/templates/canal-secret-calico-etcd.yml.j2 new file mode 100644 index 000000000..bed51c713 --- /dev/null +++ b/roles/network_plugin/canal/templates/canal-secret-calico-etcd.yml.j2 @@ -0,0 +1,18 @@ +# Source: calico/templates/calico-etcd-secrets.yaml +# The following contains k8s Secrets for use with a TLS enabled etcd cluster. +# For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/ +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: calico-etcd-secrets + namespace: kube-system +data: + # Populate the following with etcd TLS configuration if desired, but leave blank if + # not using TLS for etcd. + # The keys below should be uncommented and the values populated with the base64 + # encoded contents of each file that would be associated with the TLS data. + # Example command for encoding a file contents: cat | base64 -w 0 + etcd-key: {{ etcd_key_file.content }} + etcd-cert: {{ etcd_cert_file.content }} + etcd-ca: {{ etcd_ca_cert_file.content }} diff --git a/roles/network_plugin/canal/templates/cni-canal.conflist.j2 b/roles/network_plugin/canal/templates/cni-canal.conflist.j2 index 03619b246..3902a81c0 100644 --- a/roles/network_plugin/canal/templates/cni-canal.conflist.j2 +++ b/roles/network_plugin/canal/templates/cni-canal.conflist.j2 @@ -1,33 +1,34 @@ -{ - "name": "cni0", - "cniVersion":"0.3.1", - "plugins":[ { - "type": "flannel", - "delegate": { - "type": "calico", - "include_default_routes": true, - "etcd_endpoints": "{{ etcd_access_addresses }}", - "etcd_key_file": "{{ canal_cert_dir }}/key.pem", - "etcd_cert_file": "{{ canal_cert_dir }}/cert.crt", - "etcd_ca_cert_file": "{{ canal_cert_dir }}/ca_cert.crt", - "log_level": "info", + "name": "canal", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "flannel", + "delegate": { + "type": "calico", + "include_default_routes": true, + "etcd_endpoints": "__ETCD_ENDPOINTS__", + "etcd_key_file": "__ETCD_KEY_FILE__", + "etcd_cert_file": "__ETCD_CERT_FILE__", + "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__", + "log_level": "info", {% if calico_cni_log_file_path %} - "log_file_path": "{{ calico_cni_log_file_path }}", + "log_file_path": "{{ calico_cni_log_file_path }}", {% endif %} - "policy": { - "type": "k8s" - }, - "kubernetes": { - "kubeconfig": "__KUBECONFIG_FILEPATH__" - } - } - }, - { - "type":"portmap", - "capabilities":{ - "portMappings":true - } + "policy": { + "type": "k8s", + "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__", + "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + } + }, + { + "type": "portmap", + "capabilities": {"portMappings": true}, + "snat": true + } + ] } - ] -}