Merge pull request #784 from bradbeam/rkt

rkt support for control plane ( etcd + kubelet )
This commit is contained in:
Bogdan Dobrelya 2017-01-05 10:34:49 +01:00 committed by GitHub
commit 6d54d9f49a
18 changed files with 231 additions and 17 deletions

View file

@ -49,8 +49,10 @@ before_script:
ANSIBLE_KEEP_REMOTE_FILES: "1" ANSIBLE_KEEP_REMOTE_FILES: "1"
BOOTSTRAP_OS: none BOOTSTRAP_OS: none
LOG_LEVEL: "-vv" LOG_LEVEL: "-vv"
ETCD_DEPLOYMENT: "docker"
KUBELET_DEPLOYMENT: "docker"
MAGIC: "ci check this" MAGIC: "ci check this"
.gce: &gce .gce: &gce
<<: *job <<: *job
<<: *docker_service <<: *docker_service
@ -103,6 +105,8 @@ before_script:
-e download_localhost=true -e download_localhost=true
-e deploy_netchecker=true -e deploy_netchecker=true
-e local_release_dir=${PWD}/downloads -e local_release_dir=${PWD}/downloads
-e etcd_deployment_type=${ETCD_DEPLOYMENT}
-e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
cluster.yml cluster.yml
@ -203,6 +207,15 @@ before_script:
CLUSTER_MODE: ha CLUSTER_MODE: ha
BOOTSTRAP_OS: coreos BOOTSTRAP_OS: coreos
.ubuntu_rkt_sep_variables: &ubuntu_rkt_sep_variables
# stage: deploy-gce-part1
KUBE_NETWORK_PLUGIN: flannel
CLOUD_IMAGE: ubuntu-1604-xenial
CLOUD_REGION: us-central1-b
CLUSTER_MODE: separated
ETCD_DEPLOYMENT: rkt
KUBELET_DEPLOYMENT: rkt
# Builds for PRs only (premoderated by unit-tests step) and triggers (auto) # Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
coreos-calico-sep: coreos-calico-sep:
stage: deploy-gce-part1 stage: deploy-gce-part1
@ -406,6 +419,17 @@ coreos-alpha-weave-ha:
except: ['triggers'] except: ['triggers']
only: ['master', /^pr-.*$/] only: ['master', /^pr-.*$/]
ubuntu-rkt-sep:
stage: deploy-gce-part1
<<: *job
<<: *gce
variables:
<<: *gce_variables
<<: *ubuntu_rkt_sep_variables
when: manual
except: ['triggers']
only: ['master', /^pr-.*$/]
# Premoderated with manual actions # Premoderated with manual actions
syntax-check: syntax-check:
<<: *job <<: *job

View file

@ -28,6 +28,7 @@
roles: roles:
- { role: kubernetes/preinstall, tags: preinstall } - { role: kubernetes/preinstall, tags: preinstall }
- { role: docker, tags: docker } - { role: docker, tags: docker }
- { role: rkt, tags: rkt, when: "'rkt' in [ etcd_deployment_type, kubelet_deployment_type ]" }
- hosts: etcd:!k8s-cluster - hosts: etcd:!k8s-cluster
any_errors_fatal: true any_errors_fatal: true

View file

@ -197,3 +197,7 @@ k8s_image_pull_policy: IfNotPresent
# default packages to install within the cluster # default packages to install within the cluster
kpm_packages: [] kpm_packages: []
# - name: kube-system/grafana # - name: kube-system/grafana
rkt_version: 1.21.0
etcd_deployment_type: docker
kubelet_deployment_type: docker

View file

@ -115,13 +115,13 @@ downloads:
version: "{{etcd_version}}" version: "{{etcd_version}}"
dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz" dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
sha256: >- sha256: >-
{%- if etcd_deployment_type == 'docker' -%}{{etcd_digest_checksum|default(None)}}{%- else -%}{{etcd_checksum}}{%- endif -%} {%- if etcd_deployment_type in [ 'docker', 'rkt' ] -%}{{etcd_digest_checksum|default(None)}}{%- else -%}{{etcd_checksum}}{%- endif -%}
source_url: "{{ etcd_download_url }}" source_url: "{{ etcd_download_url }}"
url: "{{ etcd_download_url }}" url: "{{ etcd_download_url }}"
unarchive: true unarchive: true
owner: "etcd" owner: "etcd"
mode: "0755" mode: "0755"
container: "{{ etcd_deployment_type == 'docker' }}" container: "{{ etcd_deployment_type in [ 'docker', 'rkt' ] }}"
repo: "{{ etcd_image_repo }}" repo: "{{ etcd_image_repo }}"
tag: "{{ etcd_image_tag }}" tag: "{{ etcd_image_tag }}"
hyperkube: hyperkube:

View file

@ -1,17 +1,6 @@
--- ---
- name: Install | Copy etcd binary from downloaddir
command: rsync -piu "{{ etcd_bin_dir }}/etcd" "{{ bin_dir }}/etcd"
when: etcd_deployment_type == "host"
register: etcd_copy
changed_when: false
- name: Install | Copy etcdctl binary from downloaddir
command: rsync -piu "{{ etcd_bin_dir }}/etcdctl" "{{ bin_dir }}/etcdctl"
when: etcd_deployment_type == "host"
changed_when: false
#Plan A: no docker-py deps #Plan A: no docker-py deps
- name: Install | Copy etcdctl binary from container - name: Install | Copy etcdctl binary from docker container
command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy; command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy;
{{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} && {{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} &&
{{ docker_bin_dir }}/docker cp etcdctl-binarycopy:{{ etcd_container_bin_dir }}etcdctl {{ bin_dir }}/etcdctl && {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:{{ etcd_container_bin_dir }}etcdctl {{ bin_dir }}/etcdctl &&

View file

@ -0,0 +1,9 @@
---
- name: Install | Copy etcd binary from downloaddir
command: rsync -piu "{{ etcd_bin_dir }}/etcd" "{{ bin_dir }}/etcd"
register: etcd_copy
changed_when: false
- name: Install | Copy etcdctl binary from downloaddir
command: rsync -piu "{{ etcd_bin_dir }}/etcdctl" "{{ bin_dir }}/etcdctl"
changed_when: false

View file

@ -0,0 +1,26 @@
---
- name: Trust etcd container
command: >-
/usr/bin/rkt trust
--skip-fingerprint-review
--root
https://quay.io/aci-signing-key
register: etcd_rkt_trust_result
until: etcd_rkt_trust_result.rc == 0
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
changed_when: false
- name: Install | Copy etcdctl binary from rkt container
command: >-
/usr/bin/rkt run
--volume=bin-dir,kind=host,source={{ bin_dir}},readOnly=false
--mount=volume=bin-dir,target=/host/bin
{{ etcd_image_repo }}:{{ etcd_image_tag }}
--name=etcdctl-binarycopy
--exec=/bin/cp -- {{ etcd_container_bin_dir }}/etcdctl /host/bin/etcdctl
register: etcd_task_result
until: etcd_task_result.rc == 0
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
changed_when: false

View file

@ -5,7 +5,7 @@
tags: [etcd-secrets, facts] tags: [etcd-secrets, facts]
- include: gen_certs.yml - include: gen_certs.yml
tags: etcd-secrets tags: etcd-secrets
- include: install.yml - include: "install_{{ etcd_deployment_type }}.yml"
when: is_etcd_master when: is_etcd_master
tags: upgrade tags: upgrade
- include: set_cluster_health.yml - include: set_cluster_health.yml

View file

@ -0,0 +1,29 @@
[Unit]
Description=etcd rkt wrapper
Documentation=https://github.com/coreos/etcd
Wants=network.target
[Service]
Restart=on-failure
RestartSec=10s
TimeoutStartSec=0
LimitNOFILE=40000
ExecStart=/usr/bin/rkt run \
--uuid-file-save=/var/run/etcd.uuid \
--volume=etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
--mount=volume=etc-ssl-certs,target=/etc/ssl/certs \
--volume=etcd-cert-dir,kind=host,source={{ etcd_cert_dir }},readOnly=true \
--mount=volume=etcd-cert-dir,target={{ etcd_cert_dir }} \
--volume=var-lib-etcd,kind=host,source=/var/lib/etcd,readOnly=false \
--mount=volume=var-lib-etcd,target=/var/lib/etcd \
--set-env-file=/etc/etcd.env \
--stage1-from-dir=stage1-fly.aci \
{{ etcd_image_repo }}:{{ etcd_image_tag }} \
--name={{ etcd_member_name | default("etcd") }}
ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/etcd.uuid
ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/etcd.uuid
[Install]
WantedBy=multi-user.target

View file

@ -1,8 +1,31 @@
--- ---
- name: Trust kubelet container
command: >-
/usr/bin/rkt trust
--skip-fingerprint-review
--root
{{ item }}
register: kubelet_rkt_trust_result
until: kubelet_rkt_trust_result.rc == 0
with_items:
- "https://quay.io/aci-signing-key"
- "https://coreos.com/dist/pubkeys/aci-pubkeys.gpg"
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
changed_when: false
when: kubelet_deployment_type == "rkt"
- name: create kubelet working directory
file:
state: directory
path: /var/lib/kubelet
when: kubelet_deployment_type == "rkt"
- name: install | Write kubelet systemd init file - name: install | Write kubelet systemd init file
template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes template: "src=kubelet.{{ kubelet_deployment_type }}.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes"
notify: restart kubelet notify: restart kubelet
- name: install | Install kubelet launch script - name: install | Install kubelet launch script
template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes
notify: restart kubelet notify: restart kubelet
when: kubelet_deployment_type == "docker"

View file

@ -0,0 +1,58 @@
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
After=calico-node.service
Wants=network.target calico-node.service
{% else %}
Wants=network.target
{% endif %}
[Service]
Restart=on-failure
RestartSec=10s
TimeoutStartSec=0
LimitNOFILE=40000
ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet.uuid
ExecStartPre=-/bin/mkdir -p /var/lib/kubelet
EnvironmentFile={{kube_config_dir}}/kubelet.env
# stage1-fly mounts /proc /sys /dev so no need to duplicate the mounts
ExecStart=/usr/bin/rkt run \
--volume var-log,kind=host,source=/var/log \
--volume dns,kind=host,source=/etc/resolv.conf \
--volume etc-kubernetes,kind=host,source={{ kube_config_dir }},readOnly=false \
--volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
--volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
--volume var-lib-docker,kind=host,source={{ docker_daemon_graph }},readOnly=false \
--volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false \
--volume run,kind=host,source=/run,readOnly=false \
--mount volume=var-log,target=/var/log \
--mount volume=dns,target=/etc/resolv.conf \
--mount volume=etc-kubernetes,target={{ kube_config_dir }} \
--mount volume=etc-ssl-certs,target=/etc/ssl/certs \
--mount volume=usr-share-certs,target=/usr/share/ca-certificates \
--mount volume=var-lib-docker,target=/var/lib/docker \
--mount volume=var-lib-kubelet,target=/var/lib/kubelet \
--mount volume=run,target=/run \
--stage1-from-dir=stage1-fly.aci \
{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} \
--uuid-file-save=/var/run/kubelet.uuid \
--debug --exec=/kubelet -- \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBELET_API_SERVER \
$KUBELET_ADDRESS \
$KUBELET_PORT \
$KUBELET_HOSTNAME \
$KUBE_ALLOW_PRIV \
$KUBELET_ARGS \
$DOCKER_SOCKET \
$KUBELET_REGISTER_NODE \
$KUBELET_NETWORK_PLUGIN
ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet.uuid
[Install]
WantedBy=multi-user.target

View file

@ -0,0 +1,6 @@
---
rkt_version: 1.12.0
rkt_pkg_version: "{{ rkt_version }}-1"
rkt_download_src: https://github.com/coreos/rkt
rkt_download_url: "{{ rkt_download_src }}/releases/download/v{{ rkt_version }}"

View file

@ -0,0 +1,35 @@
---
- name: gather os specific variables for rkt
include_vars: "{{ item }}"
with_first_found:
- files:
- "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
- "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml"
- "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
- "{{ ansible_distribution|lower }}.yml"
- "{{ ansible_os_family|lower }}.yml"
- defaults.yml
paths:
- ../vars
skip: true
tags: facts
- name: install rkt pkg on ubuntu
apt:
deb: "{{ rkt_download_url }}/{{ rkt_pkg_name }}"
state: present
register: rkt_task_result
until: rkt_task_result|success
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
when: ansible_os_family == "Debian"
- name: install rkt pkg on centos
yum:
pkg: "{{ rkt_download_url }}/{{ rkt_pkg_name }}"
state: present
register: rkt_task_result
until: rkt_task_result|success
retries: 4
delay: "{{ retry_stagger | random + 3 }}"
when: ansible_os_family == "RedHat"

4
roles/rkt/tasks/main.yml Normal file
View file

@ -0,0 +1,4 @@
---
- name: Install rkt
include: install.yml

View file

@ -0,0 +1,2 @@
---
rkt_pkg_name: "rkt_{{ rkt_pkg_version }}_amd64.deb"

View file

@ -0,0 +1,2 @@
---
rkt_pkg_name: "rkt-{{ rkt_pkg_version }}.x86_64.rpm"

View file

@ -0,0 +1,2 @@
---
rkt_pkg_name: "rkt-{{ rkt_pkg_version }}.x86_64.rpm"