Merge pull request #588 from adidenko/canal-support

Adding support for canal network plugin

cluster.yml

@@ -27,6 +27,8 @@
 - hosts: kube-master
   roles:
     - { role: kubernetes/master, tags: master }
+    - { role: kubernetes-apps/lib, tags: apps }
+    - { role: kubernetes-apps/network_plugin, tags: network }
 
 - hosts: k8s-cluster
   roles:
@@ -34,4 +36,5 @@
 
 - hosts: kube-master[0]
   roles:
+    - { role: kubernetes-apps/lib, tags: apps }
     - { role: kubernetes-apps, tags: apps }

roles/download/defaults/main.yml

@@ -42,6 +42,8 @@ calicoctl_image_repo: "calico/ctl"
 calicoctl_image_tag: "{{ calico_version }}"
 calico_node_image_repo: "calico/node"
 calico_node_image_tag: "{{ calico_version }}"
+calico_cni_image_repo: "calico/cni"
+calico_cni_image_tag: "{{ calico_cni_version }}"
 hyperkube_image_repo: "quay.io/coreos/hyperkube"
 hyperkube_image_tag: "{{ kube_version }}_coreos.0"
 pod_infra_image_repo: "gcr.io/google_containers/pause-amd64"
@@ -56,7 +58,7 @@ downloads:
     url: "{{ calico_cni_download_url }}"
     owner: "root"
     mode: "0755"
-    enabled: "{{ kube_network_plugin == 'calico' }}"
+    enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}"
   calico_cni_plugin_ipam:
     dest: calico/bin/calico-ipam
     version: "{{calico_cni_version}}"
@@ -95,22 +97,27 @@ downloads:
     container: true
     repo: "{{ flannel_image_repo }}"
     tag: "{{ flannel_image_tag }}"
-    enabled: "{{ kube_network_plugin == 'flannel' }}"
+    enabled: "{{ kube_network_plugin == 'flannel' or kube_network_plugin == 'canal' }}"
   flannel_server_helper:
     container: true
     repo: "{{ flannel_server_helper_image_repo }}"
     tag: "{{ flannel_server_helper_image_tag }}"
-    enabled: "{{ kube_network_plugin == 'flannel' }}"
+    enabled: "{{ kube_network_plugin == 'flannel' or kube_network_plugin == 'canal' }}"
   calicoctl:
     container: true
     repo: "{{ calicoctl_image_repo }}"
     tag: "{{ calicoctl_image_tag }}"
-    enabled: "{{ kube_network_plugin == 'calico' }}"
+    enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}"
   calico_node:
     container: true
     repo: "{{ calico_node_image_repo }}"
     tag: "{{ calico_node_image_tag }}"
-    enabled: "{{ kube_network_plugin == 'calico' }}"
+    enabled: "{{ kube_network_plugin == 'calico' or kube_network_plugin == 'canal' }}"
+  calico_cni:
+    container: true
+    repo: "{{ calico_cni_image_repo }}"
+    tag: "{{ calico_cni_image_tag }}"
+    enabled: "{{ kube_network_plugin == 'canal' }}"
   pod_infra:
     container: true
     repo: "{{ pod_infra_image_repo }}"

roles/kubernetes-apps/ansible/tasks/main.yaml

@@ -18,6 +18,6 @@
   with_items: "{{ manifests.results }}"
   when: inventory_hostname == groups['kube-master'][0]
 
-
 - include: tasks/calico-policy-controller.yml
-  when: enable_network_policy is defined and enable_network_policy == True
+  when: ( enable_network_policy is defined and enable_network_policy == True ) or
+    ( kube_network_plugin == 'canal' )

roles/kubernetes-apps/network_plugin/canal/tasks/main.yaml

@@ -0,0 +1,17 @@
+- name: Create canal ConfigMap
+  run_once: true
+  kube:
+    name: "canal-config"
+    kubectl: "{{bin_dir}}/kubectl"
+    filename: "/etc/kubernetes/canal-config.yaml"
+    resource: "configmap"
+    namespace: "kube-system"
+
+- name: Start flannel and calico-node
+  run_once: true
+  kube:
+    name: "canal-node"
+    kubectl: "{{bin_dir}}/kubectl"
+    filename: "/etc/kubernetes/canal-node.yaml"
+    resource: "ds"
+    namespace: "kube-system"

roles/kubernetes-apps/network_plugin/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+ - role: kubernetes-apps/network_plugin/canal
+   when: kube_network_plugin == 'canal'

roles/kubernetes/node/tasks/main.yml

@@ -11,6 +11,13 @@
     owner: kube
   when: kube_network_plugin == "calico"
 
+- name: Write Canal cni config
+  template:
+    src: "cni-canal.conf.j2"
+    dest: "/etc/cni/net.d/10-canal.conf"
+    owner: kube
+  when: kube_network_plugin == "canal"
+
 - name: Write kubelet config file
   template: src=kubelet.j2 dest={{ kube_config_dir }}/kubelet.env backup=yes
   notify:

roles/kubernetes/node/templates/cni-canal.conf.j2

@@ -0,0 +1,15 @@
+{
+  "name": "canal-k8s-network",
+  "type": "flannel",
+  "delegate": {
+    "type": "calico",
+    "etcd_endpoints": "{{ etcd_access_endpoint }}",
+    "log_level": "info",
+    "policy": {
+      "type": "k8s"
+    },
+    "kubernetes": {
+      "kubeconfig": "{{ kube_config_dir }}/node-kubeconfig.yaml"
+    }
+  }
+}

roles/kubernetes/node/templates/kubelet.j2

@@ -26,7 +26,7 @@ KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} -
 {% else %}
 KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}"
 {% endif %}
-{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave"] %}
+{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave", "canal"] %}
 KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d"
 {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %}
 DOCKER_SOCKET="--docker-endpoint=unix:/var/run/weave/weave.sock"

roles/kubernetes/preinstall/tasks/main.yml

@@ -74,7 +74,7 @@
   with_items:
     - "/etc/cni/net.d"
     - "/opt/cni/bin"
-  when: kube_network_plugin in ["calico", "weave"] and "{{ inventory_hostname in groups['k8s-cluster'] }}"
+  when: kube_network_plugin in ["calico", "weave", "canal"] and "{{ inventory_hostname in groups['k8s-cluster'] }}"
 
 - name: Update package management cache (YUM)
   yum: update_cache=yes name='*'

roles/network_plugin/canal/defaults/main.yml

@@ -0,0 +1,11 @@
+# The interface used by canal for host <-> host communication.
+# If left blank, then the interface is chosing using the node's
+# default route.
+canal_iface: ""
+
+# Whether or not to masquerade traffic to destinations not within
+# the pod network.
+canal_masquerade: "true"
+
+# Log-level
+canal_log_level: "info"

roles/network_plugin/canal/meta/main.yml

@@ -0,0 +1,12 @@
+---
+dependencies:
+  - role: download
+    file: "{{ downloads.flannel_server_helper }}"
+  - role: download
+    file: "{{ downloads.flannel }}"
+  - role: download
+    file: "{{ downloads.calico_node }}"
+  - role: download
+    file: "{{ downloads.calicoctl }}"
+  - role: download
+    file: "{{ downloads.calico_cni }}"

roles/network_plugin/canal/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+- name: Canal | Write flannel configuration
+  template:
+    src: network.json.j2
+    dest: /etc/flannel-network.json
+    backup: yes
+
+- name: Canal | Write canal configmap
+  template:
+    src: canal-config.yml.j2
+    dest: /etc/kubernetes/canal-config.yaml
+
+- name: Canal | Write canal node configuration
+  template:
+    src: canal-node.yml.j2
+    dest: /etc/kubernetes/canal-node.yaml
+
+- name: Canal | Copy cni plugins from hyperkube
+  command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/"
+  register: cni_task_result
+  until: cni_task_result.rc == 0
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false
+
+- name: Canal | Copy cni plugins from calico/cni
+  command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp -a /opt/cni/bin/* /cnibindir/'"
+  register: cni_task_result
+  until: cni_task_result.rc == 0
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  changed_when: false

roles/network_plugin/canal/templates/canal-config.yml.j2

@@ -0,0 +1,22 @@
+# This ConfigMap can be used to configure a self-hosted Canal installation.
+# See `canal.yaml` for an example of a Canal deployment which uses
+# the config in this ConfigMap.
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: canal-config
+data:
+  # Configure this with the location of your etcd cluster.
+  etcd_endpoints: "{{ etcd_access_endpoint }}"
+
+  # The interface used by canal for host <-> host communication.
+  # If left blank, then the interface is chosing using the node's
+  # default route.
+  flanneld_iface: "{{ canal_iface }}"
+
+  # Whether or not to masquerade traffic to destinations not within
+  # the pod network.
+  masquerade: "{{ canal_masquerade }}"
+
+  # Cluster name for Flannel etcd path
+  cluster_name: "{{ cluster_name }}"

roles/network_plugin/canal/templates/canal-node.yml.j2

@@ -0,0 +1,146 @@
+---
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: canal-node
+  labels:
+    k8s-app: canal-node
+spec:
+  selector:
+    matchLabels:
+      k8s-app: canal-node
+  template:
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
+      labels:
+        k8s-app: canal-node
+    spec:
+      hostNetwork: true
+      volumes:
+        # Used by flannel-server-helper
+        - name: "networkconfig"
+          hostPath:
+            path: "/etc/flannel-network.json"
+        # Used by calico/node.
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: var-run-calico
+          hostPath:
+            path: /var/run/calico
+        # Used to install CNI.
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/cni/bin
+        - name: cni-net-dir
+          hostPath:
+            path: /etc/cni/net.d
+        # Used by flannel daemon.
+        - name: run-flannel
+          hostPath:
+            path: /run/flannel
+        - name: resolv
+          hostPath:
+            path: /etc/resolv.conf
+      containers:
+        - name: "flannel-server-helper"
+          image: "{{ flannel_server_helper_image_repo }}:{{ flannel_server_helper_image_tag }}"
+          env:
+            # Cluster name
+            - name: CLUSTER_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: cluster_name
+            # The location of the etcd cluster.
+            - name: FLANNELD_ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_endpoints
+          args:
+            - "--network-config=/etc/flannel-network.json"
+            - "--etcd-prefix=/$(CLUSTER_NAME)/network"
+            - "--etcd-server=$(FLANNELD_ETCD_ENDPOINTS)"
+          volumeMounts:
+            - name: "networkconfig"
+              mountPath: "/etc/flannel-network.json"
+          imagePullPolicy: "Always"
+        # Runs the flannel daemon to enable vxlan networking between
+        # container hosts.
+        - name: flannel
+          image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}"
+          env:
+            # Cluster name
+            - name: CLUSTER_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: cluster_name
+            # The location of the etcd cluster.
+            - name: FLANNELD_ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_endpoints
+            # The interface flannel should run on.
+            - name: FLANNELD_IFACE
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: flanneld_iface
+            # Perform masquerade on traffic leaving the pod cidr.
+            - name: FLANNELD_IP_MASQ
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: masquerade
+            # Set etcd-prefix
+            - name: DOCKER_OPT_ETCD_PREFIX
+              value: "-etcd-prefix=/$(CLUSTER_NAME)/network"
+            # Write the subnet.env file to the mounted directory.
+            - name: FLANNELD_SUBNET_FILE
+              value: "/run/flannel/subnet.env"
+          command:
+            - "/bin/sh"
+            - "-c"
+            - "/opt/bin/flanneld -etcd-prefix /$(CLUSTER_NAME)/network"
+          ports:
+            - hostPort: 10253
+              containerPort: 10253
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: "resolv"
+              mountPath: "/etc/resolv.conf"
+            - name: "run-flannel"
+              mountPath: "/run/flannel"
+        # Runs calico/node container on each Kubernetes node.  This
+        # container programs network policy and local routes on each
+        # host.
+        - name: calico-node
+          image: "{{ calico_node_image_repo }}:{{ calico_node_image_tag }}"
+          env:
+            # The location of the etcd cluster.
+            - name: ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: canal-config
+                  key: etcd_endpoints
+            # Disable Calico BGP.  Calico is simply enforcing policy.
+            - name: CALICO_NETWORKING
+              value: "false"
+            # Disable file logging so `kubectl logs` works.
+            - name: CALICO_DISABLE_FILE_LOGGING
+              value: "true"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - mountPath: /var/run/calico
+              name: var-run-calico
+              readOnly: false

roles/network_plugin/canal/templates/network.json.j2

@@ -0,0 +1 @@
+{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }

roles/network_plugin/meta/main.yml

@@ -6,3 +6,5 @@ dependencies:
    when: kube_network_plugin == 'flannel'
  - role: network_plugin/weave
    when: kube_network_plugin == 'weave'
+ - role: network_plugin/canal
+   when: kube_network_plugin == 'canal'