From 8fc5a844b30a83d27ed5e2de843d11311e834bb4 Mon Sep 17 00:00:00 2001 From: Vijay Katam Date: Fri, 24 Feb 2017 13:41:27 -0800 Subject: [PATCH] Add support for atomic host Updates based on feedback Simplify checks for file exists remove invalid char Review feedback. Use regular systemd file. Add template for docker systemd atomic --- Vagrantfile | 3 +- docs/atomic.md | 22 +++++++++++ docs/vars.md | 1 - roles/bootstrap-os/tasks/main.yml | 10 ++++- roles/docker/tasks/main.yml | 8 ++-- roles/docker/tasks/systemd.yml | 9 ++++- roles/docker/templates/docker-dns.conf.j2 | 2 +- roles/docker/templates/docker-options.conf.j2 | 2 +- .../docker/templates/docker_atomic.service.j2 | 38 +++++++++++++++++++ roles/etcd/meta/main.yml | 2 +- roles/kernel-upgrade/tasks/main.yml | 2 +- .../manifests/kube-proxy.manifest.j2 | 4 ++ roles/kubernetes/preinstall/meta/main.yml | 1 + roles/kubernetes/preinstall/tasks/main.yml | 6 +-- .../kubernetes/preinstall/tasks/set_facts.yml | 12 ++++++ 15 files changed, 107 insertions(+), 15 deletions(-) create mode 100644 docs/atomic.md create mode 100644 roles/docker/templates/docker_atomic.service.j2 diff --git a/Vagrantfile b/Vagrantfile index 8d3f2bbdd..b769199b1 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -23,6 +23,7 @@ $etcd_instances = $num_instances $kube_master_instances = $num_instances == 1 ? $num_instances : ($num_instances - 1) # All nodes are kube nodes $kube_node_instances = $num_instances +$local_release_dir = "/vagrant/temp" host_vars = {} @@ -97,7 +98,7 @@ Vagrant.configure("2") do |config| "ip": ip, "flannel_interface": ip, "flannel_backend_type": "host-gw", - "local_release_dir": "/vagrant/temp", + "local_release_dir" => $local_release_dir, "download_run_once": "False", # Override the default 'calico' with flannel. # inventory/group_vars/k8s-cluster.yml diff --git a/docs/atomic.md b/docs/atomic.md new file mode 100644 index 000000000..cb506a9f3 --- /dev/null +++ b/docs/atomic.md @@ -0,0 +1,22 @@ +Atomic host bootstrap +===================== + +Atomic host testing has been done with the network plugin flannel. Change the inventory var `kube_network_plugin: flannel`. + +Note: Flannel is the only plugin that has currently been tested with atomic + +### Vagrant + +* For bootstrapping with Vagrant, use box centos/atomic-host +* Update VagrantFile variable `local_release_dir` to `/var/vagrant/temp`. +* Update `vm_memory = 2048` and `vm_cpus = 2` +* Networking on vagrant hosts has to be brought up manually once they are booted. + + ``` + vagrant ssh + sudo /sbin/ifup enp0s8 + ``` + +* For users of vagrant-libvirt download qcow2 format from https://wiki.centos.org/SpecialInterestGroup/Atomic/Download/ + +Then you can proceed to [cluster deployment](#run-deployment) \ No newline at end of file diff --git a/docs/vars.md b/docs/vars.md index b763f6a34..966b3ffc8 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -102,4 +102,3 @@ Stack](https://github.com/kubernetes-incubator/kargo/blob/master/docs/dns-stack. Kargo sets up two Kubernetes accounts by default: ``root`` and ``kube``. Their passwords default to changeme. You can set this by changing ``kube_api_pwd``. - diff --git a/roles/bootstrap-os/tasks/main.yml b/roles/bootstrap-os/tasks/main.yml index 7f1355577..4adefb394 100644 --- a/roles/bootstrap-os/tasks/main.yml +++ b/roles/bootstrap-os/tasks/main.yml @@ -8,4 +8,12 @@ - include: bootstrap-centos.yml when: bootstrap_os == "centos" -- include: setup-pipelining.yml \ No newline at end of file +- include: setup-pipelining.yml + +- name: check if atomic host + stat: + path: /run/ostree-booted + register: ostree + +- set_fact: + is_atomic: "{{ ostree.stat.exists }}" \ No newline at end of file diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index 3e7b342f2..cdfae8242 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -38,7 +38,7 @@ retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: "{{ docker_repo_key_info.repo_keys }}" - when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic) - name: ensure docker repository is enabled action: "{{ docker_repo_info.pkg_repo }}" @@ -46,13 +46,13 @@ repo: "{{item}}" state: present with_items: "{{ docker_repo_info.repos }}" - when: (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]) and (docker_repo_info.repos|length > 0) + when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic) and (docker_repo_info.repos|length > 0) - name: Configure docker repository on RedHat/CentOS template: src: "rh_docker.repo.j2" dest: "/etc/yum.repos.d/docker.repo" - when: ansible_distribution in ["CentOS","RedHat"] + when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic - name: ensure docker packages are installed action: "{{ docker_package_info.pkg_mgr }}" @@ -66,7 +66,7 @@ delay: "{{ retry_stagger | random + 3 }}" with_items: "{{ docker_package_info.pkgs }}" notify: restart docker - when: (not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]) and (docker_package_info.pkgs|length > 0) + when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic) and (docker_package_info.pkgs|length > 0) - name: check minimum docker version for docker_dns mode. You need at least docker version >= 1.12 for resolvconf_mode=docker_dns command: "docker version -f '{{ '{{' }}.Client.Version{{ '}}' }}'" diff --git a/roles/docker/tasks/systemd.yml b/roles/docker/tasks/systemd.yml index 18710ac49..1275de5d7 100644 --- a/roles/docker/tasks/systemd.yml +++ b/roles/docker/tasks/systemd.yml @@ -15,7 +15,14 @@ src: docker.service.j2 dest: /etc/systemd/system/docker.service register: docker_service_file - when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic) + +- name: Write docker.service systemd file for atomic + template: + src: docker_atomic.service.j2 + dest: /etc/systemd/system/docker.service + notify: restart docker + when: is_atomic - name: Write docker options systemd drop-in template: diff --git a/roles/docker/templates/docker-dns.conf.j2 b/roles/docker/templates/docker-dns.conf.j2 index 01dbd3b20..d501a19c0 100644 --- a/roles/docker/templates/docker-dns.conf.j2 +++ b/roles/docker/templates/docker-dns.conf.j2 @@ -3,4 +3,4 @@ Environment="DOCKER_DNS_OPTIONS=\ {% for d in docker_dns_servers %}--dns {{ d }} {% endfor %} \ {% for d in docker_dns_search_domains %}--dns-search {{ d }} {% endfor %} \ {% for o in docker_dns_options %}--dns-opt {{ o }} {% endfor %} \ -" +" \ No newline at end of file diff --git a/roles/docker/templates/docker-options.conf.j2 b/roles/docker/templates/docker-options.conf.j2 index 50356a9f4..012795898 100644 --- a/roles/docker/templates/docker-options.conf.j2 +++ b/roles/docker/templates/docker-options.conf.j2 @@ -1,2 +1,2 @@ [Service] -Environment="DOCKER_OPTS={% if docker_options is defined %}{{ docker_options }}{% endif %}" +Environment="DOCKER_OPTS={% if docker_options is defined %}{{ docker_options }}{% endif %}" \ No newline at end of file diff --git a/roles/docker/templates/docker_atomic.service.j2 b/roles/docker/templates/docker_atomic.service.j2 new file mode 100644 index 000000000..ba37bf4c3 --- /dev/null +++ b/roles/docker/templates/docker_atomic.service.j2 @@ -0,0 +1,38 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=http://docs.docker.com +After=network.target +Wants=docker-storage-setup.service + +[Service] +Type=notify +NotifyAccess=all +EnvironmentFile=-/etc/sysconfig/docker +EnvironmentFile=-/etc/sysconfig/docker-storage +EnvironmentFile=-/etc/sysconfig/docker-network +Environment=GOTRACEBACK=crash +Environment=DOCKER_HTTP_HOST_COMPAT=1 +Environment=PATH=/usr/libexec/docker:/usr/bin:/usr/sbin +ExecReload=/bin/kill -s HUP $MAINPID +Delegate=yes +KillMode=process +ExecStart=/usr/bin/dockerd-current \ + --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current \ + --default-runtime=docker-runc \ + --exec-opt native.cgroupdriver=systemd \ + --userland-proxy-path=/usr/libexec/docker/docker-proxy-current \ + $DOCKER_OPTS \ + $DOCKER_STORAGE_OPTIONS \ + $DOCKER_NETWORK_OPTIONS \ + $DOCKER_DNS_OPTIONS \ + $ADD_REGISTRY \ + $BLOCK_REGISTRY \ + $INSECURE_REGISTRY +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +TimeoutStartSec=1min +Restart=on-abnormal + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml index bff76a129..9bd6f02a3 100644 --- a/roles/etcd/meta/main.yml +++ b/roles/etcd/meta/main.yml @@ -2,7 +2,7 @@ dependencies: - role: adduser user: "{{ addusers.etcd }}" - when: not ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] + when: not (ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] or is_atomic) - role: download file: "{{ downloads.etcd }}" tags: download diff --git a/roles/kernel-upgrade/tasks/main.yml b/roles/kernel-upgrade/tasks/main.yml index 999eb94ae..a16f0f37b 100644 --- a/roles/kernel-upgrade/tasks/main.yml +++ b/roles/kernel-upgrade/tasks/main.yml @@ -2,4 +2,4 @@ - include: centos-7.yml when: ansible_distribution in ["CentOS","RedHat"] and - ansible_distribution_major_version >= 7 + ansible_distribution_major_version >= 7 and not is_atomic \ No newline at end of file diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index a965ef792..2dbcf74d1 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -50,7 +50,11 @@ spec: volumes: - name: ssl-certs-host hostPath: +{% if ansible_os_family == 'RedHat' %} + path: /etc/pki/tls +{% else %} path: /usr/share/ca-certificates +{% endif %} - name: "kubeconfig" hostPath: path: "{{kube_config_dir}}/node-kubeconfig.yaml" diff --git a/roles/kubernetes/preinstall/meta/main.yml b/roles/kubernetes/preinstall/meta/main.yml index cf440f5e2..203d968a7 100644 --- a/roles/kubernetes/preinstall/meta/main.yml +++ b/roles/kubernetes/preinstall/meta/main.yml @@ -3,3 +3,4 @@ dependencies: - role: adduser user: "{{ addusers.kube }}" tags: kubelet + when: not is_atomic \ No newline at end of file diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 5b79c101d..27e98949d 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -91,7 +91,7 @@ yum: update_cache: yes name: '*' - when: ansible_pkg_mgr == 'yum' + when: ansible_pkg_mgr == 'yum' and not is_atomic tags: bootstrap-os - name: Install latest version of python-apt for Debian distribs @@ -112,7 +112,7 @@ - name: Install epel-release on RedHat/CentOS shell: rpm -qa | grep epel-release || rpm -ivh {{ epel_rpm_download_url }} - when: ansible_distribution in ["CentOS","RedHat"] + when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic changed_when: False check_mode: no tags: bootstrap-os @@ -127,7 +127,7 @@ retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: "{{required_pkgs | default([]) | union(common_required_pkgs|default([]))}}" - when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] + when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic) tags: bootstrap-os # Todo : selinux configuration diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index 2481fcd7f..6a25c785e 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -83,5 +83,17 @@ - set_fact: peer_with_calico_rr: "{{ 'calico-rr' in groups and groups['calico-rr']|length > 0 }}" +- name: check if atomic host + stat: + path: /run/ostree-booted + register: ostree + +- set_fact: + is_atomic: "{{ ostree.stat.exists }}" + +- set_fact: + kube_cert_group: "kube" + when: is_atomic + - include: set_resolv_facts.yml tags: [bootstrap-os, resolvconf, facts]