Merge pull request #784 from bradbeam/rkt
rkt support for control plane ( etcd + kubelet )
This commit is contained in:
commit
55620f1708
18 changed files with 231 additions and 17 deletions
|
@ -49,8 +49,10 @@ before_script:
|
|||
ANSIBLE_KEEP_REMOTE_FILES: "1"
|
||||
BOOTSTRAP_OS: none
|
||||
LOG_LEVEL: "-vv"
|
||||
ETCD_DEPLOYMENT: "docker"
|
||||
KUBELET_DEPLOYMENT: "docker"
|
||||
MAGIC: "ci check this"
|
||||
|
||||
|
||||
.gce: &gce
|
||||
<<: *job
|
||||
<<: *docker_service
|
||||
|
@ -103,6 +105,8 @@ before_script:
|
|||
-e download_localhost=true
|
||||
-e deploy_netchecker=true
|
||||
-e local_release_dir=${PWD}/downloads
|
||||
-e etcd_deployment_type=${ETCD_DEPLOYMENT}
|
||||
-e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
|
||||
cluster.yml
|
||||
|
||||
|
||||
|
@ -203,6 +207,15 @@ before_script:
|
|||
CLUSTER_MODE: ha
|
||||
BOOTSTRAP_OS: coreos
|
||||
|
||||
.ubuntu_rkt_sep_variables: &ubuntu_rkt_sep_variables
|
||||
# stage: deploy-gce-part1
|
||||
KUBE_NETWORK_PLUGIN: flannel
|
||||
CLOUD_IMAGE: ubuntu-1604-xenial
|
||||
CLOUD_REGION: us-central1-b
|
||||
CLUSTER_MODE: separated
|
||||
ETCD_DEPLOYMENT: rkt
|
||||
KUBELET_DEPLOYMENT: rkt
|
||||
|
||||
# Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
|
||||
coreos-calico-sep:
|
||||
stage: deploy-gce-part1
|
||||
|
@ -406,6 +419,17 @@ coreos-alpha-weave-ha:
|
|||
except: ['triggers']
|
||||
only: ['master', /^pr-.*$/]
|
||||
|
||||
ubuntu-rkt-sep:
|
||||
stage: deploy-gce-part1
|
||||
<<: *job
|
||||
<<: *gce
|
||||
variables:
|
||||
<<: *gce_variables
|
||||
<<: *ubuntu_rkt_sep_variables
|
||||
when: manual
|
||||
except: ['triggers']
|
||||
only: ['master', /^pr-.*$/]
|
||||
|
||||
# Premoderated with manual actions
|
||||
syntax-check:
|
||||
<<: *job
|
||||
|
|
|
@ -28,6 +28,7 @@
|
|||
roles:
|
||||
- { role: kubernetes/preinstall, tags: preinstall }
|
||||
- { role: docker, tags: docker }
|
||||
- { role: rkt, tags: rkt, when: "'rkt' in [ etcd_deployment_type, kubelet_deployment_type ]" }
|
||||
|
||||
- hosts: etcd:!k8s-cluster
|
||||
any_errors_fatal: true
|
||||
|
|
|
@ -197,3 +197,7 @@ k8s_image_pull_policy: IfNotPresent
|
|||
# default packages to install within the cluster
|
||||
kpm_packages: []
|
||||
# - name: kube-system/grafana
|
||||
|
||||
rkt_version: 1.21.0
|
||||
etcd_deployment_type: docker
|
||||
kubelet_deployment_type: docker
|
||||
|
|
|
@ -115,13 +115,13 @@ downloads:
|
|||
version: "{{etcd_version}}"
|
||||
dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
|
||||
sha256: >-
|
||||
{%- if etcd_deployment_type == 'docker' -%}{{etcd_digest_checksum|default(None)}}{%- else -%}{{etcd_checksum}}{%- endif -%}
|
||||
{%- if etcd_deployment_type in [ 'docker', 'rkt' ] -%}{{etcd_digest_checksum|default(None)}}{%- else -%}{{etcd_checksum}}{%- endif -%}
|
||||
source_url: "{{ etcd_download_url }}"
|
||||
url: "{{ etcd_download_url }}"
|
||||
unarchive: true
|
||||
owner: "etcd"
|
||||
mode: "0755"
|
||||
container: "{{ etcd_deployment_type == 'docker' }}"
|
||||
container: "{{ etcd_deployment_type in [ 'docker', 'rkt' ] }}"
|
||||
repo: "{{ etcd_image_repo }}"
|
||||
tag: "{{ etcd_image_tag }}"
|
||||
hyperkube:
|
||||
|
|
|
@ -1,17 +1,6 @@
|
|||
---
|
||||
- name: Install | Copy etcd binary from downloaddir
|
||||
command: rsync -piu "{{ etcd_bin_dir }}/etcd" "{{ bin_dir }}/etcd"
|
||||
when: etcd_deployment_type == "host"
|
||||
register: etcd_copy
|
||||
changed_when: false
|
||||
|
||||
- name: Install | Copy etcdctl binary from downloaddir
|
||||
command: rsync -piu "{{ etcd_bin_dir }}/etcdctl" "{{ bin_dir }}/etcdctl"
|
||||
when: etcd_deployment_type == "host"
|
||||
changed_when: false
|
||||
|
||||
#Plan A: no docker-py deps
|
||||
- name: Install | Copy etcdctl binary from container
|
||||
- name: Install | Copy etcdctl binary from docker container
|
||||
command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy;
|
||||
{{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} &&
|
||||
{{ docker_bin_dir }}/docker cp etcdctl-binarycopy:{{ etcd_container_bin_dir }}etcdctl {{ bin_dir }}/etcdctl &&
|
9
roles/etcd/tasks/install_host.yml
Normal file
9
roles/etcd/tasks/install_host.yml
Normal file
|
@ -0,0 +1,9 @@
|
|||
---
|
||||
- name: Install | Copy etcd binary from downloaddir
|
||||
command: rsync -piu "{{ etcd_bin_dir }}/etcd" "{{ bin_dir }}/etcd"
|
||||
register: etcd_copy
|
||||
changed_when: false
|
||||
|
||||
- name: Install | Copy etcdctl binary from downloaddir
|
||||
command: rsync -piu "{{ etcd_bin_dir }}/etcdctl" "{{ bin_dir }}/etcdctl"
|
||||
changed_when: false
|
26
roles/etcd/tasks/install_rkt.yml
Normal file
26
roles/etcd/tasks/install_rkt.yml
Normal file
|
@ -0,0 +1,26 @@
|
|||
---
|
||||
- name: Trust etcd container
|
||||
command: >-
|
||||
/usr/bin/rkt trust
|
||||
--skip-fingerprint-review
|
||||
--root
|
||||
https://quay.io/aci-signing-key
|
||||
register: etcd_rkt_trust_result
|
||||
until: etcd_rkt_trust_result.rc == 0
|
||||
retries: 4
|
||||
delay: "{{ retry_stagger | random + 3 }}"
|
||||
changed_when: false
|
||||
|
||||
- name: Install | Copy etcdctl binary from rkt container
|
||||
command: >-
|
||||
/usr/bin/rkt run
|
||||
--volume=bin-dir,kind=host,source={{ bin_dir}},readOnly=false
|
||||
--mount=volume=bin-dir,target=/host/bin
|
||||
{{ etcd_image_repo }}:{{ etcd_image_tag }}
|
||||
--name=etcdctl-binarycopy
|
||||
--exec=/bin/cp -- {{ etcd_container_bin_dir }}/etcdctl /host/bin/etcdctl
|
||||
register: etcd_task_result
|
||||
until: etcd_task_result.rc == 0
|
||||
retries: 4
|
||||
delay: "{{ retry_stagger | random + 3 }}"
|
||||
changed_when: false
|
|
@ -5,7 +5,7 @@
|
|||
tags: [etcd-secrets, facts]
|
||||
- include: gen_certs.yml
|
||||
tags: etcd-secrets
|
||||
- include: install.yml
|
||||
- include: "install_{{ etcd_deployment_type }}.yml"
|
||||
when: is_etcd_master
|
||||
tags: upgrade
|
||||
- include: set_cluster_health.yml
|
||||
|
|
29
roles/etcd/templates/etcd-rkt.service.j2
Normal file
29
roles/etcd/templates/etcd-rkt.service.j2
Normal file
|
@ -0,0 +1,29 @@
|
|||
[Unit]
|
||||
Description=etcd rkt wrapper
|
||||
Documentation=https://github.com/coreos/etcd
|
||||
Wants=network.target
|
||||
|
||||
[Service]
|
||||
Restart=on-failure
|
||||
RestartSec=10s
|
||||
TimeoutStartSec=0
|
||||
LimitNOFILE=40000
|
||||
|
||||
ExecStart=/usr/bin/rkt run \
|
||||
--uuid-file-save=/var/run/etcd.uuid \
|
||||
--volume=etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
|
||||
--mount=volume=etc-ssl-certs,target=/etc/ssl/certs \
|
||||
--volume=etcd-cert-dir,kind=host,source={{ etcd_cert_dir }},readOnly=true \
|
||||
--mount=volume=etcd-cert-dir,target={{ etcd_cert_dir }} \
|
||||
--volume=var-lib-etcd,kind=host,source=/var/lib/etcd,readOnly=false \
|
||||
--mount=volume=var-lib-etcd,target=/var/lib/etcd \
|
||||
--set-env-file=/etc/etcd.env \
|
||||
--stage1-from-dir=stage1-fly.aci \
|
||||
{{ etcd_image_repo }}:{{ etcd_image_tag }} \
|
||||
--name={{ etcd_member_name | default("etcd") }}
|
||||
|
||||
ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/etcd.uuid
|
||||
ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/etcd.uuid
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
|
@ -1,8 +1,31 @@
|
|||
---
|
||||
- name: Trust kubelet container
|
||||
command: >-
|
||||
/usr/bin/rkt trust
|
||||
--skip-fingerprint-review
|
||||
--root
|
||||
{{ item }}
|
||||
register: kubelet_rkt_trust_result
|
||||
until: kubelet_rkt_trust_result.rc == 0
|
||||
with_items:
|
||||
- "https://quay.io/aci-signing-key"
|
||||
- "https://coreos.com/dist/pubkeys/aci-pubkeys.gpg"
|
||||
retries: 4
|
||||
delay: "{{ retry_stagger | random + 3 }}"
|
||||
changed_when: false
|
||||
when: kubelet_deployment_type == "rkt"
|
||||
|
||||
- name: create kubelet working directory
|
||||
file:
|
||||
state: directory
|
||||
path: /var/lib/kubelet
|
||||
when: kubelet_deployment_type == "rkt"
|
||||
|
||||
- name: install | Write kubelet systemd init file
|
||||
template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes
|
||||
template: "src=kubelet.{{ kubelet_deployment_type }}.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes"
|
||||
notify: restart kubelet
|
||||
|
||||
- name: install | Install kubelet launch script
|
||||
template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes
|
||||
notify: restart kubelet
|
||||
when: kubelet_deployment_type == "docker"
|
||||
|
|
58
roles/kubernetes/node/templates/kubelet.rkt.service.j2
Normal file
58
roles/kubernetes/node/templates/kubelet.rkt.service.j2
Normal file
|
@ -0,0 +1,58 @@
|
|||
[Unit]
|
||||
Description=Kubernetes Kubelet Server
|
||||
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
|
||||
{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
|
||||
After=calico-node.service
|
||||
Wants=network.target calico-node.service
|
||||
{% else %}
|
||||
Wants=network.target
|
||||
{% endif %}
|
||||
|
||||
[Service]
|
||||
Restart=on-failure
|
||||
RestartSec=10s
|
||||
TimeoutStartSec=0
|
||||
LimitNOFILE=40000
|
||||
|
||||
ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet.uuid
|
||||
ExecStartPre=-/bin/mkdir -p /var/lib/kubelet
|
||||
|
||||
EnvironmentFile={{kube_config_dir}}/kubelet.env
|
||||
# stage1-fly mounts /proc /sys /dev so no need to duplicate the mounts
|
||||
ExecStart=/usr/bin/rkt run \
|
||||
--volume var-log,kind=host,source=/var/log \
|
||||
--volume dns,kind=host,source=/etc/resolv.conf \
|
||||
--volume etc-kubernetes,kind=host,source={{ kube_config_dir }},readOnly=false \
|
||||
--volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
|
||||
--volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
|
||||
--volume var-lib-docker,kind=host,source={{ docker_daemon_graph }},readOnly=false \
|
||||
--volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false \
|
||||
--volume run,kind=host,source=/run,readOnly=false \
|
||||
--mount volume=var-log,target=/var/log \
|
||||
--mount volume=dns,target=/etc/resolv.conf \
|
||||
--mount volume=etc-kubernetes,target={{ kube_config_dir }} \
|
||||
--mount volume=etc-ssl-certs,target=/etc/ssl/certs \
|
||||
--mount volume=usr-share-certs,target=/usr/share/ca-certificates \
|
||||
--mount volume=var-lib-docker,target=/var/lib/docker \
|
||||
--mount volume=var-lib-kubelet,target=/var/lib/kubelet \
|
||||
--mount volume=run,target=/run \
|
||||
--stage1-from-dir=stage1-fly.aci \
|
||||
{{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} \
|
||||
--uuid-file-save=/var/run/kubelet.uuid \
|
||||
--debug --exec=/kubelet -- \
|
||||
$KUBE_LOGTOSTDERR \
|
||||
$KUBE_LOG_LEVEL \
|
||||
$KUBELET_API_SERVER \
|
||||
$KUBELET_ADDRESS \
|
||||
$KUBELET_PORT \
|
||||
$KUBELET_HOSTNAME \
|
||||
$KUBE_ALLOW_PRIV \
|
||||
$KUBELET_ARGS \
|
||||
$DOCKER_SOCKET \
|
||||
$KUBELET_REGISTER_NODE \
|
||||
$KUBELET_NETWORK_PLUGIN
|
||||
|
||||
ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet.uuid
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
6
roles/rkt/defaults/main.yml
Normal file
6
roles/rkt/defaults/main.yml
Normal file
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
|
||||
rkt_version: 1.12.0
|
||||
rkt_pkg_version: "{{ rkt_version }}-1"
|
||||
rkt_download_src: https://github.com/coreos/rkt
|
||||
rkt_download_url: "{{ rkt_download_src }}/releases/download/v{{ rkt_version }}"
|
35
roles/rkt/tasks/install.yml
Normal file
35
roles/rkt/tasks/install.yml
Normal file
|
@ -0,0 +1,35 @@
|
|||
---
|
||||
- name: gather os specific variables for rkt
|
||||
include_vars: "{{ item }}"
|
||||
with_first_found:
|
||||
- files:
|
||||
- "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml"
|
||||
- "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml"
|
||||
- "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml"
|
||||
- "{{ ansible_distribution|lower }}.yml"
|
||||
- "{{ ansible_os_family|lower }}.yml"
|
||||
- defaults.yml
|
||||
paths:
|
||||
- ../vars
|
||||
skip: true
|
||||
tags: facts
|
||||
|
||||
- name: install rkt pkg on ubuntu
|
||||
apt:
|
||||
deb: "{{ rkt_download_url }}/{{ rkt_pkg_name }}"
|
||||
state: present
|
||||
register: rkt_task_result
|
||||
until: rkt_task_result|success
|
||||
retries: 4
|
||||
delay: "{{ retry_stagger | random + 3 }}"
|
||||
when: ansible_os_family == "Debian"
|
||||
|
||||
- name: install rkt pkg on centos
|
||||
yum:
|
||||
pkg: "{{ rkt_download_url }}/{{ rkt_pkg_name }}"
|
||||
state: present
|
||||
register: rkt_task_result
|
||||
until: rkt_task_result|success
|
||||
retries: 4
|
||||
delay: "{{ retry_stagger | random + 3 }}"
|
||||
when: ansible_os_family == "RedHat"
|
4
roles/rkt/tasks/main.yml
Normal file
4
roles/rkt/tasks/main.yml
Normal file
|
@ -0,0 +1,4 @@
|
|||
---
|
||||
|
||||
- name: Install rkt
|
||||
include: install.yml
|
2
roles/rkt/vars/debian.yml
Normal file
2
roles/rkt/vars/debian.yml
Normal file
|
@ -0,0 +1,2 @@
|
|||
---
|
||||
rkt_pkg_name: "rkt_{{ rkt_pkg_version }}_amd64.deb"
|
2
roles/rkt/vars/fedora.yml
Normal file
2
roles/rkt/vars/fedora.yml
Normal file
|
@ -0,0 +1,2 @@
|
|||
---
|
||||
rkt_pkg_name: "rkt-{{ rkt_pkg_version }}.x86_64.rpm"
|
2
roles/rkt/vars/redhat.yml
Normal file
2
roles/rkt/vars/redhat.yml
Normal file
|
@ -0,0 +1,2 @@
|
|||
---
|
||||
rkt_pkg_name: "rkt-{{ rkt_pkg_version }}.x86_64.rpm"
|
Loading…
Reference in a new issue