Merge pull request #156 from Smana/coreos_support

CoreOS support (Calico, Flannel, Weave)

README.md

@@ -13,6 +13,7 @@ Linux distributions tested:
 * **Ubuntu** 14.10, 15.04, 15.10
 * **Fedora** 23
 * **CentOS/RHEL** 7
+* **CoreOS**
 
 ### Requirements
 * The target servers must have **access to the Internet** in order to pull docker imaqes.
@@ -68,6 +69,18 @@ You can jump directly to "*Available apps, installation procedure*"
 
 Ansible
 -------------------------
+### Coreos bootstrap
+Before running the cluster playbook you must satisfy the following requirements:
+* On each CoreOS nodes a writable directory **/opt/bin** (~400M disk space)
+
+* Uncomment the variable **ansible_python_interpreter** in the file `inventory/group_vars/all.yml`
+
+* run the Python bootstrap playbook
+```
+ansible-playbook -u smana -e ansible_ssh_user=smana  -b --become-user=root -i inventory/inventory.cfg coreos-bootstrap.yml
+```
+Then you can proceed to cluster deployment
+
 ### Variables
 The main variables to change are located in the directory ```inventory/group_vars/all.yml```.
 
@@ -179,7 +192,6 @@ For the master nodes you'll have to see the docker logs for the apiserver
 docker logs [apiserver docker id]
 ```
 
-
 ### Available apps, installation procedure
 
 There are two ways of installing new apps

cluster.yml

@@ -5,7 +5,7 @@
     - { role: download, tags: download }
     - { role: kubernetes/preinstall, tags: preinstall }
     - { role: etcd, tags: etcd }
-    - { role: docker, tags: docker }
+    - { role: docker, tags: docker, when: ansible_os_family != "CoreOS" }
     - { role: kubernetes/node, tags: node }
     - { role: network_plugin, tags: network }
     - { role: dnsmasq, tags: dnsmasq }

coreos-bootstrap.yml

@@ -0,0 +1,5 @@
+---
+- hosts: k8s-cluster
+  gather_facts: False
+  roles:
+    - coreos-bootstrap

inventory/group_vars/all.yml

@@ -1,10 +1,14 @@
- # Directory where the binaries will be installed
+# Directory where the binaries will be installed
 bin_dir: /usr/local/bin
 
 # Where the binaries will be downloaded.
 # Note: ensure that you've enough disk space (about 1G)
 local_release_dir: "/tmp/releases"
 
+# Uncomment this line for CoreOS only.
+# Directory where python binary is installed
+# ansible_python_interpreter: "/opt/bin/python"
+
 # This is the group that the cert creation scripts chgrp the
 # cert files to. Not really changable...
 kube_cert_group: kube-cert

roles/adduser/tasks/main.yml

@@ -1,3 +1,18 @@
+---
+- name: gather os specific variables
+  include_vars: "{{ item }}"
+  with_first_found:
+    - files:
+      - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
+      - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml"
+      - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
+      - "{{ ansible_distribution|lower }}.yml"
+      - "{{ ansible_os_family|lower }}.yml"
+      - defaults.yml
+      paths:
+      - ../vars
+      skip: true
+
 - name: User | Create User Group
   group: name={{item.group|default(item.name)}} system={{item.system|default(omit)}}
   with_items: addusers

roles/adduser/vars/coreos.yml

@@ -0,0 +1,8 @@
+---
+addusers:
+  - name: kube
+    comment: "Kubernetes user"
+    shell: /sbin/nologin
+    system: yes
+    group: "{{ kube_cert_group }}"
+    createhome: no

roles/adduser/vars/debian.yml

@@ -12,4 +12,4 @@ addusers:
     shell: /sbin/nologin
     system: yes
     group: "{{ kube_cert_group }}"
-    createhome: no
+    createhome: no

roles/adduser/vars/redhat.yml

@@ -0,0 +1,15 @@
+---
+addusers:
+  - name: etcd
+    comment: "Etcd user"
+    createhome: yes
+    home: "/var/lib/etcd"
+    system: yes
+    shell: /bin/nologin
+
+  - name: kube
+    comment: "Kubernetes user"
+    shell: /sbin/nologin
+    system: yes
+    group: "{{ kube_cert_group }}"
+    createhome: no

roles/coreos-bootstrap/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+pypy_version: 2.4.0
+pip_python_modules:
+  - httplib2

roles/coreos-bootstrap/files/bootstrap.sh

@@ -1,7 +1,7 @@
 #/bin/bash
 set -e
 
-BINDIR="/usr/local/bin"
+BINDIR="/opt/bin"
 
 cd $BINDIR
 

roles/coreos-bootstrap/files/runner

@@ -1,3 +1,3 @@
 #!/bin/bash
-BINDIR="/usr/local/bin"
+BINDIR="/opt/bin"
 LD_LIBRARY_PATH=$BINDIR/pypy/lib:$LD_LIBRARY_PATH $BINDIR/pypy/bin/$(basename $0) $@

roles/coreos-bootstrap/tasks/main.yml

@@ -1,41 +1,40 @@
 ---
-- name: Python | Check if bootstrap is needed
+- name: Bootstrap | Check if bootstrap is needed
-  raw: stat {{ bin_dir}}/.bootstrapped
+  raw: stat /opt/bin/.bootstrapped
   register: need_bootstrap
   ignore_errors: True
 
-- name: Python | Run bootstrap.sh
+- name: Bootstrap | Run bootstrap.sh
   script: bootstrap.sh
   when: need_bootstrap | failed
 
 - set_fact:
-    ansible_python_interpreter: "{{ bin_dir }}/python"
+    ansible_python_interpreter: "/opt/bin/python"
 
-- name: Python | Check if we need to install pip
+- name: Bootstrap | Check if we need to install pip
   shell: "{{ansible_python_interpreter}} -m pip --version"
   register: need_pip
   ignore_errors: True
   changed_when: false
   when: need_bootstrap | failed
 
-- name: Python | Copy get-pip.py
+- name: Bootstrap | Copy get-pip.py
   copy: src=get-pip.py dest=~/get-pip.py
   when: need_pip | failed
 
-- name: Python | Install pip
+- name: Bootstrap | Install pip
   shell: "{{ansible_python_interpreter}} ~/get-pip.py"
   when: need_pip | failed
 
-- name: Python | Remove get-pip.py
+- name: Bootstrap | Remove get-pip.py
   file: path=~/get-pip.py state=absent
   when: need_pip | failed
 
-- name: Python | Install pip launcher
+- name: Bootstrap | Install pip launcher
-  copy: src=runner dest={{ bin_dir }}/pip mode=0755
+  copy: src=runner dest=/opt/bin/pip mode=0755
   when: need_pip | failed
 
 - name: Install required python modules
   pip:
     name: "{{ item }}"
   with_items: pip_python_modules
-

roles/coreos-bootstrap/templates/python_shim.j2

@@ -0,0 +1,2 @@
+#!/bin/bash
+LD_LIBRARY_PATH={{ pypy_install_path }}/lib:$LD_LIBRARY_PATH exec {{ pypy_install_path }}/bin/{{ item.src }} "$@"

roles/docker/tasks/main.yml

@@ -11,6 +11,7 @@
       - defaults.yml
       paths:
       - ../vars
+      skip: true
 
 - name: check for minimum kernel version
   fail:

roles/etcd/tasks/configure.yml

@@ -2,7 +2,7 @@
 - name: Configure |  Copy etcd.service systemd file
   template:
     src: etcd.service.j2
-    dest: /lib/systemd/system/etcd.service
+    dest: /etc/systemd/system/etcd.service
     backup: yes
   when: ansible_service_mgr == "systemd"
   notify: restart etcd

roles/kubernetes/master/tasks/main.yml

@@ -3,6 +3,7 @@
   copy:
     src: kubectl_bash_completion.sh
     dest: /etc/bash_completion.d/kubectl.sh
+  when: ansible_os_family in ["Debian","RedHat"]
 
 - name: Copy kube-apiserver binary
   command: rsync -piu "{{ local_release_dir }}/kubernetes/bin/kube-apiserver" "{{ bin_dir }}/kube-apiserver"

roles/kubernetes/node/tasks/install.yml

@@ -18,12 +18,3 @@
   command: rsync -piu "{{ local_release_dir }}/kubernetes/bin/kubelet" "{{ bin_dir }}/kubelet"
   register: kubelet_copy
   changed_when: false
-
-- name: install | Calico-plugin | Directory
-  file: path=/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/ state=directory
-  when: kube_network_plugin == "calico"
-
-- name: install | Calico-plugin | Binary
-  command: rsync -piu "{{ local_release_dir }}/calico/bin/calico" "/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/calico"
-  when: kube_network_plugin == "calico"
-  changed_when: false

roles/kubernetes/preinstall/defaults/main.yml

@@ -8,5 +8,3 @@ common_required_pkgs:
   - rsync
   - bash-completion
 
-pypy_version: 2.4.0
-python_pypy_url: "https://bitbucket.org/pypy/pypy/downloads/pypy-{{ pypy_version }}.tar.bz2"

roles/kubernetes/preinstall/tasks/main.yml

@@ -14,6 +14,12 @@
       - defaults.yml
       paths:
       - ../vars
+      skip: true
+
+- name: Force binaries directory for CoreOS
+  set_fact:
+    bin_dir: "/opt/bin"
+  when: ansible_os_family == "CoreOS"
 
 - name: Create kubernetes config directory
   file:
@@ -85,8 +91,8 @@
     module: "{{ ansible_pkg_mgr }}"
     name: "{{ item }}"
     state: latest
-  with_items: "{{required_pkgs | union(common_required_pkgs)}}"
+  with_items: "{{required_pkgs | default([]) | union(common_required_pkgs|default([]))}}"
-  when: ansible_os_family in [ "Debian", "RedHat" ]
+  when: ansible_os_family != "CoreOS"
 
 # Todo : selinux configuration
 - name: Set selinux policy to permissive
@@ -95,6 +101,3 @@
   changed_when: False
 
 - include: etchosts.yml
-
-- include: python-bootstrap.yml
-  when: ansible_os_family not in [ "Debian", "RedHat" ]

roles/network_plugin/calico/tasks/main.yml

@@ -8,13 +8,14 @@
     mode: 0644
   notify:
     - restart docker
+  when: ansible_os_family != "CoreOS"
 
 - name: Calico | Write docker.service systemd file
   template:
     src: systemd-docker.service
     dest: /lib/systemd/system/docker.service
   notify: restart docker
-  when: ansible_service_mgr == "systemd"
+  when: ansible_service_mgr == "systemd" and ansible_os_family != "CoreOS"
 
 - meta: flush_handlers
 
@@ -34,12 +35,6 @@
 - name: Calico | install calicoctl
   file: path={{ bin_dir }}/calicoctl mode=0755 state=file
 
-- name: Calico | Create calicoctl symlink (needed by kubelet)
-  file:
-    src: /usr/local/bin/calicoctl
-    dest: /usr/bin/calicoctl
-    state: link
-
 - name: Calico | wait for etcd
   wait_for:
     port: 2379
@@ -54,12 +49,12 @@
   run_once: true
 
 - name: Calico | Configure calico network pool for cloud
-  command: "calicoctl pool add {{ kube_pods_subnet }} --ipip --nat-outgoing"
+  command: "{{ bin_dir }}/calicoctl pool add {{ kube_pods_subnet }} --ipip --nat-outgoing"
   run_once: true
   when: calico_conf.status == 404 and cloud_provider is defined and cloud_provider == True
 
 - name: Calico | Configure calico network pool
-  command: "calicoctl pool add {{ kube_pods_subnet }}"
+  command: "{{ bin_dir }}/calicoctl pool add {{ kube_pods_subnet }}"
   run_once: true
   when: calico_conf.status == 404 and (cloud_provider is not defined or cloud_provider != True)
 
@@ -112,13 +107,13 @@
   when: calico_copy.stdout_lines
 
 - name: Calico | Disable node mesh
-  shell: calicoctl bgp node-mesh off
+  shell: "{{ bin_dir }}/calicoctl bgp node-mesh off"
   environment:
      ETCD_AUTHORITY: "127.0.0.1:2379"
   when: peer_with_router|default(false) and inventory_hostname in groups['kube-node']
 
 - name: Calico | Configure peering with router(s)
-  shell: calicoctl node bgp peer add {{ item.router_id }} as {{ item.as }}
+  shell: "{{ bin_dir }}/calicoctl node bgp peer add {{ item.router_id }} as {{ item.as }}"
   environment:
      ETCD_AUTHORITY: "127.0.0.1:2379"
   with_items: peers

roles/network_plugin/flannel/tasks/main.yml

@@ -4,7 +4,6 @@
     src: network.json
     dest: /etc/flannel-network.json
     backup: yes
-
 - name: Flannel | Create flannel pod manifest
   template:
     src: flannel-pod.yml
@@ -15,6 +14,7 @@
   wait_for:
     path: /run/flannel/subnet.env
     delay: 5
+    timeout: 600
 
 - name: Flannel | Get flannel_subnet from subnet.env
   shell: cat /run/flannel/subnet.env | awk -F'=' '$1 == "FLANNEL_SUBNET" {print $2}'
@@ -42,11 +42,18 @@
   notify:
     - restart docker
 
+- name: Flannel | Create docker config symlink for CoreOS
+  file:
+    src: "/etc/default/docker"
+    dest: "/run/flannel_docker_opts.env"
+    state: link
+  when: ansible_os_family == "CoreOS"
+
 - name: Flannel | Write docker.service systemd file
   template:
     src: systemd-docker.service
     dest: /lib/systemd/system/docker.service
   notify: restart docker
-  when: ansible_service_mgr == "systemd"
+  when: ansible_service_mgr == "systemd" and ansible_os_family != "CoreOS"
 
 - meta: flush_handlers

roles/network_plugin/flannel/templates/docker

@@ -1,5 +1,6 @@
 # Deployed by Ansible
-{% if ansible_service_mgr in ["sysvinit","upstart"] and kube_network_plugin == "flannel" and ansible_os_family == "Debian" %}
+{% if (ansible_service_mgr in ["sysvinit","upstart"] and kube_network_plugin == "flannel" and ansible_os_family == "Debian") or
+   (kube_network_plugin == "flannel" and ansible_os_family == "CoreOS") %}
 DOCKER_OPTS="--bip={{ flannel_subnet }} --mtu={{ flannel_mtu }}"
 {% elif kube_network_plugin == "flannel" %}
 OPTIONS="--bip={{ flannel_subnet }} --mtu={{ flannel_mtu }}"

roles/network_plugin/weave/tasks/main.yml

@@ -14,7 +14,7 @@
     src: systemd-docker.service
     dest: /lib/systemd/system/docker.service
   notify: restart docker
-  when: ansible_service_mgr == "systemd"
+  when: ansible_service_mgr == "systemd" and ansible_os_family != "CoreOS"
 
 - meta: flush_handlers