diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index f72276ae6..cbb1a2760 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -36,6 +36,9 @@ cluster_name: cluster.local # Subdomains of DNS domain to be resolved via /etc/resolv.conf ndots: 5 +# Choose the container engine (docker, rkt) +kube_container_engine: docker + # For some environments, each node has a pubilcally accessible # address and an address it should bind services to. These are # really inventory level variables, but described here for consistency. diff --git a/roles/adduser/defaults/main.yml b/roles/adduser/defaults/main.yml index b3a69229c..281f4db89 100644 --- a/roles/adduser/defaults/main.yml +++ b/roles/adduser/defaults/main.yml @@ -14,6 +14,13 @@ addusers: system: yes group: "{{ kube_cert_group }}" createhome: no + rkt: + name: rkt + comment: "rkt user" + shell: /sbin/nologin + system: yes + group: rkt + createhome: no adduser: name: "{{ user.name }}" diff --git a/roles/docker/.gitignore b/roles/container_engine/docker/.gitignore similarity index 100% rename from roles/docker/.gitignore rename to roles/container_engine/docker/.gitignore diff --git a/roles/docker/defaults/main.yml b/roles/container_engine/docker/defaults/main.yml similarity index 100% rename from roles/docker/defaults/main.yml rename to roles/container_engine/docker/defaults/main.yml diff --git a/roles/docker/files/rh_docker.repo b/roles/container_engine/docker/files/rh_docker.repo similarity index 100% rename from roles/docker/files/rh_docker.repo rename to roles/container_engine/docker/files/rh_docker.repo diff --git a/roles/docker/handlers/main.yml b/roles/container_engine/docker/handlers/main.yml similarity index 100% rename from roles/docker/handlers/main.yml rename to roles/container_engine/docker/handlers/main.yml diff --git a/roles/docker/tasks/main.yml b/roles/container_engine/docker/tasks/main.yml similarity index 100% rename from roles/docker/tasks/main.yml rename to roles/container_engine/docker/tasks/main.yml diff --git a/roles/docker/tasks/systemd-proxies.yml b/roles/container_engine/docker/tasks/systemd-proxies.yml similarity index 100% rename from roles/docker/tasks/systemd-proxies.yml rename to roles/container_engine/docker/tasks/systemd-proxies.yml diff --git a/roles/docker/templates/http-proxy.conf.j2 b/roles/container_engine/docker/templates/http-proxy.conf.j2 similarity index 100% rename from roles/docker/templates/http-proxy.conf.j2 rename to roles/container_engine/docker/templates/http-proxy.conf.j2 diff --git a/roles/container_engine/docker/templates/systemd-docker.service.j2 b/roles/container_engine/docker/templates/systemd-docker.service.j2 new file mode 100644 index 000000000..b19b1caaf --- /dev/null +++ b/roles/container_engine/docker/templates/systemd-docker.service.j2 @@ -0,0 +1,40 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=http://docs.docker.com +{% if ansible_os_family == "RedHat" %} +After=network.target docker-storage-setup.service +Wants=docker-storage-setup.service +{% elif ansible_os_family == "Debian" %} +After=network.target docker.socket +Wants=docker.socket +{% endif %} + +[Service] +Type=notify +{% if ansible_os_family == "RedHat" %} +EnvironmentFile=-/etc/default/docker +EnvironmentFile=-/etc/sysconfig/docker +EnvironmentFile=-/etc/sysconfig/docker-network +EnvironmentFile=-/etc/sysconfig/docker-storage +{% elif ansible_os_family == "Debian" %} +EnvironmentFile=-/etc/default/docker +{% endif %} +Environment=GOTRACEBACK=crash +ExecReload=/bin/kill -s HUP $MAINPID +Delegate=yes +KillMode=process +ExecStart=/usr/bin/docker daemon \ + $OPTIONS \ + $DOCKER_STORAGE_OPTIONS \ + $DOCKER_NETWORK_OPTIONS \ + $INSECURE_REGISTRY \ + $DOCKER_OPTS +TasksMax=infinity +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +TimeoutStartSec=1min +Restart=on-abnormal + +[Install] +WantedBy=multi-user.target diff --git a/roles/docker/vars/centos-6.yml b/roles/container_engine/docker/vars/centos-6.yml similarity index 100% rename from roles/docker/vars/centos-6.yml rename to roles/container_engine/docker/vars/centos-6.yml diff --git a/roles/docker/vars/debian.yml b/roles/container_engine/docker/vars/debian.yml similarity index 100% rename from roles/docker/vars/debian.yml rename to roles/container_engine/docker/vars/debian.yml diff --git a/roles/docker/vars/fedora-20.yml b/roles/container_engine/docker/vars/fedora-20.yml similarity index 100% rename from roles/docker/vars/fedora-20.yml rename to roles/container_engine/docker/vars/fedora-20.yml diff --git a/roles/docker/vars/fedora.yml b/roles/container_engine/docker/vars/fedora.yml similarity index 100% rename from roles/docker/vars/fedora.yml rename to roles/container_engine/docker/vars/fedora.yml diff --git a/roles/docker/vars/redhat.yml b/roles/container_engine/docker/vars/redhat.yml similarity index 100% rename from roles/docker/vars/redhat.yml rename to roles/container_engine/docker/vars/redhat.yml diff --git a/roles/docker/vars/ubuntu-16.04.yml b/roles/container_engine/docker/vars/ubuntu-16.04.yml similarity index 100% rename from roles/docker/vars/ubuntu-16.04.yml rename to roles/container_engine/docker/vars/ubuntu-16.04.yml diff --git a/roles/docker/vars/ubuntu.yml b/roles/container_engine/docker/vars/ubuntu.yml similarity index 100% rename from roles/docker/vars/ubuntu.yml rename to roles/container_engine/docker/vars/ubuntu.yml diff --git a/roles/container_engine/meta/main.yml b/roles/container_engine/meta/main.yml new file mode 100644 index 000000000..f497d8622 --- /dev/null +++ b/roles/container_engine/meta/main.yml @@ -0,0 +1,6 @@ +--- +dependencies: + - role: container_engine/rkt + when: kube_container_engine == 'rkt' + - role: container_engine/docker + when: kube_container_engine == 'docker' diff --git a/roles/container_engine/rkt/defaults/main.yml b/roles/container_engine/rkt/defaults/main.yml new file mode 100644 index 000000000..1b0cd7f73 --- /dev/null +++ b/roles/container_engine/rkt/defaults/main.yml @@ -0,0 +1,5 @@ +--- +rkt_version: v1.17.0 +rkt_bin_dir: "{{ local_release_dir }}/rkt/rkt-{{ rkt_version }}/" +rkt_stage1_dir: "/usr/lib/rkt/stage1-images" +rkt_netconfig_dir: "/etc/rkt/net.d" diff --git a/roles/container_engine/rkt/meta/main.yml b/roles/container_engine/rkt/meta/main.yml new file mode 100644 index 000000000..e386a7772 --- /dev/null +++ b/roles/container_engine/rkt/meta/main.yml @@ -0,0 +1,8 @@ +--- +dependencies: + - role: adduser + user: "{{ addusers.rkt }}" + when: ansible_os_family != 'CoreOS' + + - role: download + file: "{{ downloads.rkt }}" diff --git a/roles/container_engine/rkt/tasks/main.yml b/roles/container_engine/rkt/tasks/main.yml new file mode 100644 index 000000000..ab4324b00 --- /dev/null +++ b/roles/container_engine/rkt/tasks/main.yml @@ -0,0 +1,48 @@ +--- +- name: Copy rkt binary from downloaddir + command: rsync -piu "{{ rkt_bin_dir }}/rkt" "{{ bin_dir }}/rkt" + changed_when: false + +- name: Create rkt stage1 image directory + file: + path: "{{ rkt_stage1_dir }}" + state: directory + owner: rkt + recurse: yes + +- name: Copy rkt stage1 images from downloaddir + command: rsync -piu "{{ rkt_bin_dir }}/stage1-{{ item }}.aci" "{{ rkt_stage1_dir }}" + changed_when: false + with_items: + - coreos + - fly + - kvm + +- name: Copy rkt manpages from downloaddir + command: rsync -piu "{{ rkt_bin_dir }}/manpages" "/usr/share/man/man1" + changed_when: false + +- name: Copy systemd units from downloaddir + command: rsync -piu "{{ rkt_bin_dir }}/init/systemd/{{ item }}" "/usr/lib/systemd/system" + changed_when: false + with_items: + - rkt-gc.service + - rkt-gc.timer + - rkt-metadata.socket + - rkt-metadata.service + +- name: Create rkt network config directory + file: + path: "{{ rkt_netconfig_dir }}" + state: directory + owner: root + recurse: yes + +- name: Trust CoreOS images repository + command: rkt trust --prefix quay.io/coreos --skip-fingerprint-review + changed_when: false + +- name: Configure flannel network plugin for rkt + template: + src: flannel_cni.conf + dest: "{{ rkt_netconfig_dir }}" diff --git a/roles/container_engine/rkt/templates/flannel_cni.conf b/roles/container_engine/rkt/templates/flannel_cni.conf new file mode 100644 index 000000000..585248237 --- /dev/null +++ b/roles/container_engine/rkt/templates/flannel_cni.conf @@ -0,0 +1,7 @@ +{ + "name": "rkt.kubernetes.io", + "type": "flannel", + "delegate": { + "isDefaultGateway": true + } +} diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 1ea220fd1..e2c72bd98 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -16,8 +16,10 @@ weave_version: v1.6.1 flannel_version: v0.6.2 flannel_server_helper_version: 0.1 pod_infra_version: 3.0 +rkt_version: v1.17.0 # Download URL's +rkt_download_url: "https://github.com/coreos/rkt/releases/download/{{rkt_version}}/rkt-{{rkt_version}}.tar.gz" etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd" calico_cni_download_url: "https://storage.googleapis.com/kargo/{{calico_cni_version}}_calico-cni-plugin" calico_cni_ipam_download_url: "https://storage.googleapis.com/kargo/{{calico_cni_version}}_calico-cni-plugin-ipam" @@ -31,7 +33,7 @@ etcd_checksum: "385afd518f93e3005510b7aaa04d38ee4a39f06f5152cd33bb86d4f0c94c7485 # Containers # Possible values: host, docker -etcd_deployment_type: "docker" +etcd_deployment_type: "container" etcd_image_repo: "quay.io/coreos/etcd" etcd_image_tag: "{{ etcd_version }}" flannel_server_helper_image_repo: "gcr.io/google_containers/flannel-server-helper" @@ -50,6 +52,7 @@ hyperkube_image_repo: "quay.io/coreos/hyperkube" hyperkube_image_tag: "{{ kube_version }}_coreos.0" pod_infra_image_repo: "gcr.io/google_containers/pause-amd64" pod_infra_image_tag: "{{ pod_infra_version }}" +rkt_checksum: "285b4f18bf7ec3f80b42dd506a86fe367b6e7068d014d5187621c5c4ab168b89" downloads: calico_cni_plugin: @@ -79,6 +82,15 @@ downloads: owner: "root" mode: "0755" enabled: "{{ kube_network_plugin == 'weave' }}" + rkt: + version: "{{rkt_version}}" + dest: "rkt/rkt-{{ rkt_version }}.tar.gz" + sha256: "{{ rkt_checksum }}" + source_url: "{{ rkt_download_url }}" + url: "{{ rkt_download_url }}" + unarchive: true + owner: "root" + mode: "0750" etcd: version: "{{etcd_version}}" dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz" @@ -88,7 +100,7 @@ downloads: unarchive: true owner: "etcd" mode: "0755" - container: "{{ etcd_deployment_type == 'docker' }}" + container: "{{ etcd_deployment_type == 'container' }}" repo: "{{ etcd_image_repo }}" tag: "{{ etcd_image_tag }}" hyperkube: diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index e715f380d..43694ab7d 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -54,7 +54,7 @@ until: pull_task_result.rc == 0 retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: "{{ download.enabled|bool and download.container|bool }}" + when: "{{ download.enabled|bool and download.container|bool and kube_container_engine == 'docker' }}" delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else inventory_hostname }}" run_once: "{{ download_run_once|bool }}" @@ -68,7 +68,7 @@ - name: "Update the 'container_changed' fact" set_fact: container_changed: "{{ not 'up to date' in pull_task_result.stdout }}" - when: "{{ download.enabled|bool and download.container|bool }}" + when: "{{ download.enabled|bool and download.container|bool and kube_container_engine == 'docker' }}" delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else inventory_hostname }}" run_once: "{{ download_run_once|bool }}" @@ -76,7 +76,7 @@ shell: docker save "{{ download.repo }}:{{ download.tag }}" > "{{ fname }}" delegate_to: "{{groups['kube-master'][0]}}" run_once: true - when: ansible_os_family != "CoreOS" and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool + when: ansible_os_family != "CoreOS" and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool and kube_container_engine == 'docker' - name: Download | get container images synchronize: @@ -87,8 +87,12 @@ until: get_task|success retries: 4 delay: "{{ retry_stagger | random + 3 }}" - when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool + when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] + and download_run_once|bool and download.enabled|bool and download.container|bool + and container_changed|bool and kube_container_engine == 'docker' - name: Download | load container images shell: docker load < "{{ fname }}" - when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool + when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] + and download_run_once|bool and download.enabled|bool and download.container|bool + and container_changed|bool and kube_container_engine == 'docker' diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml index b55966a99..f561ffd37 100644 --- a/roles/etcd/meta/main.yml +++ b/roles/etcd/meta/main.yml @@ -3,7 +3,12 @@ dependencies: - role: adduser user: "{{ addusers.etcd }}" when: ansible_os_family != 'CoreOS' - - role: docker - when: (ansible_os_family != "CoreOS" and etcd_deployment_type == "docker" or inventory_hostname in groups['k8s-cluster']) + + - role: container_engine + when: (ansible_os_family != "CoreOS" and etcd_deployment_type == "container" or inventory_hostname in groups['k8s-cluster']) + - role: download file: "{{ downloads.etcd }}" + + - role: container_engine + when: (ansible_os_family != "CoreOS" and etcd_deployment_type == "container") diff --git a/roles/etcd/tasks/configure.yml b/roles/etcd/tasks/configure.yml index a2ef38f2c..484a27237 100644 --- a/roles/etcd/tasks/configure.yml +++ b/roles/etcd/tasks/configure.yml @@ -6,6 +6,11 @@ changed_when: false when: is_etcd_master +- name: Configure | Set container engine deployment type + set_fact: + etcd_deployment_type: "{{ kube_container_engine }}" + when: etcd_deployment_type == "container" + - name: Configure | Add member to the cluster if it is not there when: is_etcd_master and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0 shell: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}" diff --git a/roles/etcd/tasks/install.yml b/roles/etcd/tasks/install.yml index aa7f32ca3..3553933b4 100644 --- a/roles/etcd/tasks/install.yml +++ b/roles/etcd/tasks/install.yml @@ -11,12 +11,24 @@ changed_when: false #Plan A: no docker-py deps -- name: Install | Copy etcdctl binary from container +- name: Install | Copy etcdctl binary from docker container command: sh -c "/usr/bin/docker rm -f etcdctl-binarycopy; /usr/bin/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} && - /usr/bin/docker cp etcdctl-binarycopy:{{ etcd_container_bin_dir }}etcdctl {{ bin_dir }}/etcdctl && + /usr/bin/docker cp etcdctl-binarycopy:{{ etcd_container_bin_dir }}/etcdctl {{ bin_dir }}/etcdctl && /usr/bin/docker rm -f etcdctl-binarycopy" - when: etcd_deployment_type == "docker" + when: etcd_deployment_type == "container" and kube_container_engine == "docker" + register: etcd_task_result + until: etcd_task_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + +- name: Install | Copy etcdctl binary from rkt container + command: sh -c "{{ bin_dir }}/rkt run {{ etcd_image_repo }}:{{ etcd_image_tag }} + --insecure-options=image --volume bindir,kind=host,source={{ bin_dir }} --mount volume=bindir,target=/etcd + --exec cp -- {{ etcd_container_bin_dir }}/etcdctl /etcd && + {{ bin_dir }}/rkt gc --grace-period=0" + when: etcd_deployment_type == "container" and kube_container_engine == "rkt" register: etcd_task_result until: etcd_task_result.rc == 0 retries: 4 @@ -29,15 +41,15 @@ # name: etcd-binarycopy # state: present # image: "{{ etcd_image_repo }}:{{ etcd_image_tag }}" -# when: etcd_deployment_type == "docker" +# when: etcd_deployment_type == "container" # #- name: Install | Copy etcdctl from etcd-binarycopy container # command: /usr/bin/docker cp "etcd-binarycopy:{{ etcd_container_bin_dir }}etcdctl" "{{ bin_dir }}/etcdctl" -# when: etcd_deployment_type == "docker" +# when: etcd_deployment_type == "container" # #- name: Install | Clean up etcd-binarycopy container # docker: # name: etcd-binarycopy # state: absent # image: "{{ etcd_image_repo }}:{{ etcd_image_tag }}" -# when: etcd_deployment_type == "docker" +# when: etcd_deployment_type == "container" diff --git a/roles/etcd/tasks/pre_upgrade.yml b/roles/etcd/tasks/pre_upgrade.yml index d1962ea92..290a6bce9 100644 --- a/roles/etcd/tasks/pre_upgrade.yml +++ b/roles/etcd/tasks/pre_upgrade.yml @@ -1,12 +1,12 @@ - name: "Pre-upgrade | check for etcd-proxy unit file" stat: path: /etc/systemd/system/etcd-proxy.service - register: kube_apiserver_service_file + register: etcd_proxy_service_file - name: "Pre-upgrade | check for etcd-proxy init script" stat: path: /etc/init.d/etcd-proxy - register: kube_apiserver_init_script + register: etcd_proxy_init_script - name: "Pre-upgrade | stop etcd-proxy if service defined" service: @@ -23,12 +23,16 @@ - /etc/systemd/system/etcd-proxy.service - /etc/init.d/etcd-proxy -- name: "Pre-upgrade | find etcd-proxy container" - command: docker ps -aq --filter "name=etcd-proxy*" - register: etcd_proxy_container +# TODO: Smana +#- name: "Pre-upgrade | stop etcd-proxy service" +# systemd: +# name: etcd-proxy +# state: stopped +# register: etcd_proxy_status + +#- debug: msg={{etcd_proxy_status}} + +- name: "Pre-upgrade | remove etcd-proxy container if it exists" + command: docker rm -f etcd-proxy ignore_errors: true - -- name: "Pre-upgrade | remove etcd-proxy if it exists" - command: "docker rm -f {{item}}" - with_items: "{{etcd_proxy_container.stdout_lines}}" - + when: "{{ kube_container_engine == 'docker' }}" diff --git a/roles/etcd/templates/etcd-proxy-rkt.service.j2 b/roles/etcd/templates/etcd-proxy-rkt.service.j2 new file mode 100644 index 000000000..728c7f4ad --- /dev/null +++ b/roles/etcd/templates/etcd-proxy-rkt.service.j2 @@ -0,0 +1,20 @@ +[Unit] +Description=etcd-proxy rkt wrapper +Documentation=http://kargo.kubespray.io +Requires=network-online.target +After=network-online.target + +[Service] +ExecStart={{ bin_dir }}/rkt run {{ etcd_image_repo }}:{{ etcd_image_tag }} \ +--set-env-file=/etc/etcd-proxy.env \ +--net=host \ +--volume=certs,kind=host,source=/usr/share/ca-certificates/,readOnly=true \ +--mount=volume=certs,target=/etc/ssl/certs \ +--exec {{ etcd_container_bin_dir }}/etcd +ExecStopPost={{ bin_dir }}/rkt gc --mark-only +KillMode=mixed +Restart=always +RestartSec=15s + +[Install] +WantedBy=multi-user.target diff --git a/roles/etcd/templates/etcd-rkt.service.j2 b/roles/etcd/templates/etcd-rkt.service.j2 new file mode 100644 index 000000000..eef27d426 --- /dev/null +++ b/roles/etcd/templates/etcd-rkt.service.j2 @@ -0,0 +1,20 @@ +[Unit] +Description=etcd rkt wrapper +Documentation=http://kargo.kubespray.io +Requires=network-online.target +After=network-online.target + +[Service] +ExecStart={{ bin_dir }}/rkt run {{ etcd_image_repo }}:{{ etcd_image_tag }} \ +--set-env-file=/etc/etcd.env \ +--net=host \ +--volume=certs,kind=host,source=/usr/share/ca-certificates/,readOnly=true \ +--mount=volume=certs,target=/etc/ssl/certs \ +--exec {{ etcd_container_bin_dir }}/etcd +ExecStopPost={{ bin_dir }}/rkt gc --mark-only +KillMode=mixed +Restart=always +RestartSec=15s + +[Install] +WantedBy=multi-user.target diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index 419be1f5a..fc32ede45 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -7,8 +7,21 @@ dest: /etc/bash_completion.d/kubectl.sh when: ansible_os_family in ["Debian","RedHat"] -- name: Copy kubectl from hyperkube container +- name: Copy kubectl from hyperkube container (docker) command: "/usr/bin/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp /hyperkube /systembindir/kubectl" + when: kube_container_engine == "docker" + register: kube_task_result + until: kube_task_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + +- name: Copy kubectl from hyperkube container (rkt) + command: sh -c "{{ bin_dir }}/rkt run --insecure-options=image {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} + --volume bindir,kind=host,source={{ bin_dir }} --mount volume=bindir,target=/systembindir + --exec cp -- /hyperkube /systembindir/kubectl && + {{ bin_dir }}/rkt gc --grace-period=0" + when: kube_container_engine == "rkt" register: kube_task_result until: kube_task_result.rc == 0 retries: 4 diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index 46678691a..756f3d2a1 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -31,6 +31,9 @@ KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %} DOCKER_SOCKET="--docker-endpoint=unix:/var/run/weave/weave.sock" {% endif %} +{% if kube_container_engine == "rkt" %} +RKT_OPTS="--container-runtime=rkt --rkt-path={{ bin_dir }}/rkt" +{% endif %} # Should this cluster be allowed to run privileged docker containers KUBE_ALLOW_PRIV="--allow-privileged=true" {% if cloud_provider is defined and cloud_provider == "openstack" %} @@ -42,6 +45,6 @@ KUBELET_CLOUDPROVIDER="" {% endif %} {% if ansible_service_mgr in ["sysvinit","upstart"] %} DAEMON_ARGS="$KUBE_LOGGING $KUBE_LOG_LEVEL $KUBE_ALLOW_PRIV $KUBELET_API_SERVER $KUBELET_ADDRESS \ -$KUBELET_HOSTNAME $KUBELET_REGISTER_NODE $KUBELET_ARGS $DOCKER_SOCKET $KUBELET_ARGS $KUBELET_NETWORK_PLUGIN \ +$KUBELET_HOSTNAME $KUBELET_REGISTER_NODE $KUBELET_ARGS $DOCKER_SOCKET $RKT_OPTS $KUBELET_ARGS $KUBELET_NETWORK_PLUGIN \ $KUBELET_CLOUDPROVIDER" {% endif %} diff --git a/roles/kubernetes/node/templates/kubelet.service.j2 b/roles/kubernetes/node/templates/kubelet.service.j2 index ad62d8562..e0451f6c1 100644 --- a/roles/kubernetes/node/templates/kubelet.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.service.j2 @@ -20,6 +20,7 @@ ExecStart={{ bin_dir }}/kubelet \ $KUBELET_HOSTNAME \ $KUBE_ALLOW_PRIV \ $KUBELET_ARGS \ + $RKT_OPTS \ $DOCKER_SOCKET \ $KUBELET_REGISTER_NODE \ $KUBELET_NETWORK_PLUGIN \ diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index 6563a1f65..d4608a567 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -40,8 +40,21 @@ changed_when: false notify: restart calico-node -- name: Calico | Copy cni plugins from hyperkube +- name: Calico | Copy cni plugins from hyperkube (docker) command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/" + when: kube_container_engine == "docker" + register: cni_task_result + until: cni_task_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false + +- name: Calico | Copy cni plugins from hyperkube (rkt) + command: sh -c "{{ bin_dir }}/rkt run {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} + --volume cnibindir,kind=host,source={{ bin_dir }} --mount volume=cnibindir,target=/cnibindir + --exec /usr/bin/rsync -- -a /opt/cni/bin /cnibindir && + {{ bin_dir }}/rkt gc --grace-period=0" + when: kube_container_engine == "rkt" register: cni_task_result until: cni_task_result.rc == 0 retries: 4 diff --git a/roles/network_plugin/calico/templates/calico-node.service.j2 b/roles/network_plugin/calico/templates/calico-node.service.j2 index 87a51fac8..0510fcdc6 100644 --- a/roles/network_plugin/calico/templates/calico-node.service.j2 +++ b/roles/network_plugin/calico/templates/calico-node.service.j2 @@ -8,17 +8,33 @@ Wants=docker.socket User=root PermissionsStartOnly=true {% if legacy_calicoctl %} -{% if inventory_hostname in groups['kube-node'] and peer_with_router|default(false)%} -ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --as={{ local_as }} --detach=false --node-image={{ calico_node_image_repo }}:{{ calico_node_image_tag }} + {%- if inventory_hostname in groups['kube-node'] and peer_with_router|default(false)%} +ExecStart={{ bin_dir }}/calicoctl node \ + {%- if kube_container_engine == 'rkt' %} +--runtime=rkt \ + {%- endif %} +--ip={{ip | default(ansible_default_ipv4.address) }} \ +--as={{ local_as }} \ +--detach=false \ +--node-image={{ calico_node_image_repo }}:{{ calico_node_image_tag }} + {% else %} +ExecStart={{ bin_dir }}/calicoctl node \ + {%- if kube_container_engine == 'rkt' %} +--runtime=rkt \ + {%- endif %} +--ip={{ip | default(ansible_default_ipv4.address) }} \ +--detach=false \ +--node-image={{ calico_node_image_repo }}:{{ calico_node_image_tag }} + {%- endif %} {% else %} -ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --detach=false --node-image={{ calico_node_image_repo }}:{{ calico_node_image_tag }} -{% endif %} -{% else %} -{% if inventory_hostname in groups['kube-node'] and peer_with_router|default(false)%} -ExecStart={{ bin_dir }}/calicoctl node run --ip={{ip | default(ansible_default_ipv4.address) }} --as={{ local_as }} --node-image={{ calico_node_image_repo }}:{{ calico_node_image_tag }} -{% else %} + {%- if inventory_hostname in groups['kube-node'] and peer_with_router|default(false)%} +ExecStart={{ bin_dir }}/calicoctl node run \ +--ip={{ip | default(ansible_default_ipv4.address) }} \ +--as={{ local_as }} \ +--node-image={{ calico_node_image_repo }}:{{ calico_node_image_tag }} + {%- else %} ExecStart={{ bin_dir }}/calicoctl node run --ip={{ip | default(ansible_default_ipv4.address) }} --node-image={{ calico_node_image_repo }}:{{ calico_node_image_tag }} -{% endif %} + {%- endif %} {% endif %} Restart=always RestartSec=10s diff --git a/roles/network_plugin/calico/templates/calicoctl-container.j2 b/roles/network_plugin/calico/templates/calicoctl-container.j2 index 7be30928a..0589af365 100644 --- a/roles/network_plugin/calico/templates/calicoctl-container.j2 +++ b/roles/network_plugin/calico/templates/calicoctl-container.j2 @@ -1,4 +1,5 @@ #!/bin/bash +{% if kube_container_engine == "docker" %} /usr/bin/docker run -i --privileged --rm \ --net=host --pid=host \ -e ETCD_ENDPOINTS={{ etcd_access_endpoint }} \ @@ -11,3 +12,6 @@ -v /etc/calico/certs:/etc/calico/certs:ro \ {{ calicoctl_image_repo }}:{{ calicoctl_image_tag}} \ $@ +{% elif kube_container_engine == "rkt" %} +@TODO-RKT +{% endif %} diff --git a/roles/uploads/defaults/main.yml b/roles/uploads/defaults/main.yml index 7b5797881..97aee60da 100644 --- a/roles/uploads/defaults/main.yml +++ b/roles/uploads/defaults/main.yml @@ -8,18 +8,21 @@ etcd_version: v3.0.6 calico_version: v0.23.0 calico_cni_version: v1.4.2 weave_version: v1.6.1 +rkt_version: v1.17.0 # Download URL's etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz" calico_cni_download_url: "https://github.com/projectcalico/calico-cni/releases/download/{{calico_cni_version}}/calico" calico_cni_ipam_download_url: "https://github.com/projectcalico/calico-cni/releases/download/{{calico_cni_version}}/calico-ipam" weave_download_url: "https://github.com/weaveworks/weave/releases/download/{{weave_version}}/weave" +rkt_download_url: "https://github.com/coreos/rkt/releases/download/{{ rkt_version }}/rkt-{{ rkt_version }}.tar.gz" # Checksums calico_cni_checksum: "9cab29764681e9d80da826e4b2cd10841cc01a749e0018867d96dd76a4691548" calico_cni_ipam_checksum: "09d076b15b791956efee91646e47fdfdcf382db16082cef4f542a9fff7bae172" weave_checksum: "9bf9d6e5a839e7bcbb28cc00c7acae9d09284faa3e7a3720ca9c2b9e93c68580" etcd_checksum: "385afd518f93e3005510b7aaa04d38ee4a39f06f5152cd33bb86d4f0c94c7485" +rkt_checksum: "285b4f18bf7ec3f80b42dd506a86fe367b6e7068d014d5187621c5c4ab168b89" downloads: - name: calico-cni-plugin @@ -49,6 +52,16 @@ downloads: owner: "root" mode: "0755" + - name: rkt + version: "{{rkt_version}}" + dest: "rkt/rkt-{{ rkt_version }}.tar.gz" + sha256: "{{ rkt_checksum }}" + source_url: "{{ rkt_download_url }}" + url: "{{ rkt_download_url }}" + unarchive: true + owner: "root" + mode: "0750" + - name: etcd version: "{{etcd_version}}" dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz"