Split deploy steps in separate playbooks: part1 (#3451)

* Fix bootstrap_os/ubuntu idempotency

* Update bastion role

* move container_engine in sub-roles

* requires ansible 2.5

* ubuntu18 as first CI job

.gitlab-ci.yml

@@ -42,7 +42,7 @@ before_script:
   tags:
     - kubernetes
     - docker
-  image: quay.io/kubespray/kubespray:latest
+  image: quay.io/kubespray/kubespray:v2.7
 
 .docker_service: &docker_service
   services:
@@ -323,12 +323,13 @@ before_script:
 
 # Builds for PRs only (premoderated by unit-tests step) and triggers (auto)
 ### PR JOBS PART1
-gce_coreos-calico-aio:
+
+gce_ubuntu18-flannel-aio:
   stage: deploy-part1
   <<: *job
   <<: *gce
   variables:
-    <<: *coreos_calico_aio_variables
+    <<: *ubuntu18_flannel_aio_variables
     <<: *gce_variables
   when: on_success
   except: ['triggers']
@@ -336,14 +337,14 @@ gce_coreos-calico-aio:
 
 ### PR JOBS PART2
 
-gce_ubuntu18-flannel-aio:
+gce_coreos-calico-aio:
   stage: deploy-part2
   <<: *job
   <<: *gce
   variables:
-    <<: *ubuntu18_flannel_aio_variables
+    <<: *coreos_calico_aio_variables
     <<: *gce_variables
-  when: manual
+  when: on_success
   except: ['triggers']
   only: [/^pr-.*$/]
 
@@ -358,7 +359,7 @@ gce_centos7-flannel-addons:
   except: ['triggers']
   only: [/^pr-.*$/]
 
-gce_centos-weave-kubeadm:
+gce_centos-weave-kubeadm-sep:
   stage: deploy-part2
   <<: *job
   <<: *gce
@@ -369,6 +370,8 @@ gce_centos-weave-kubeadm:
   except: ['triggers']
   only: [/^pr-.*$/]
 
+### MANUAL JOBS
+
 gce_ubuntu-weave-sep:
   stage: deploy-part2
   <<: *job
@@ -376,11 +379,10 @@ gce_ubuntu-weave-sep:
   variables:
     <<: *gce_variables
     <<: *ubuntu_weave_sep_variables
-  when: on_success
+  when: manual
   except: ['triggers']
   only: [/^pr-.*$/]
 
-### MANUAL JOBS
 gce_coreos-calico-sep-triggers:
   stage: deploy-part2
   <<: *job

cluster.yml

@@ -1,5 +1,5 @@
 ---
-- hosts: localhost
+- hosts: bastion[0]
   gather_facts: False
   roles:
     - { role: kubespray-defaults}
@@ -33,11 +33,7 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
-    - { role: docker, tags: docker, when: container_manager == 'docker' }
+    - { role: "container-engine", tags: "container-engine" }
-    - { role: cri-o, tags: crio, when: container_manager == 'crio' }
-    - role: rkt
-      tags: rkt
-      when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
     - { role: download, tags: download, when: "not skip_downloads" }
   environment: "{{proxy_env}}"
 

requirements.txt

@@ -1,4 +1,4 @@
-ansible>=2.4.0
+ansible>=2.5.0
 jinja2>=2.9.6
 netaddr
 pbr>=1.6

roles/bastion-ssh-config/tasks/main.yml

@@ -1,21 +1,16 @@
 ---
 - set_fact:
-    has_bastion: "{{ 'bastion' in groups['all'] }}"
+    bastion_ip: "{{ hostvars[groups['bastion'][0]]['ansible_host'] | d(hostvars[groups['bastion'][0]]['ansible_ssh_host']) }}"
-
+  delegate_to: localhost
-- set_fact:
-    bastion_ip: "{{ hostvars['bastion']['ansible_host'] }}"
-  when: has_bastion
 
 # As we are actually running on localhost, the ansible_ssh_user is your local user when you try to use it directly
 # To figure out the real ssh user, we delegate this task to the bastion and store the ansible_user in real_user
 - set_fact:
     real_user: "{{ ansible_user }}"
-  delegate_to: bastion
-  when: has_bastion
 
 - name: create ssh bastion conf
   become: false
+  delegate_to: localhost
   template:
     src: ssh-bastion.conf
     dest: "{{ playbook_dir }}/ssh-bastion.conf"
-  when: has_bastion

roles/bastion-ssh-config/templates/ssh-bastion.conf

@@ -1,9 +1,8 @@
-{% if has_bastion %}
 {% set vars={'hosts': ''} %}
 {% set user='' %}
 
 {% for h in groups['all'] %}
-{% if h != 'bastion' %}
+{% if h not in groups['bastion'] %}
 {% if vars.update({'hosts': vars['hosts'] + ' ' + (hostvars[h].get('ansible_ssh_host') or hostvars[h]['ansible_host'])}) %}{% endif %}
 {% endif %}
 {% endfor %}
@@ -17,4 +16,3 @@ Host {{ bastion_ip }}
 
 Host {{ vars['hosts'] }}
   ProxyCommand ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -W %h:%p {{ real_user }}@{{ bastion_ip }} {% if ansible_ssh_private_key_file is defined %}-i {{ ansible_ssh_private_key_file }}{% endif %}
-{% endif %}

roles/bootstrap-os/defaults/main.yml

@@ -1,9 +1,10 @@
 ---
+bootstrap_os:
+os_family: "{{bootstrap_os}}"
+
 pip_python_coreos_modules:
   - httplib2
   - six
 
 override_system_hostname: true
-
-
 coreos_auto_upgrade: true

roles/bootstrap-os/tasks/bootstrap-ubuntu.yml

@@ -1,25 +1,30 @@
 ---
 #  raw: cat /etc/issue.net | grep '{{ bootstrap_versions }}'
 
+- name: List ubuntu_packages
+  set_fact:
+    ubuntu_packages:
+      - python
+      - python-apt
+      - python-pip
+      - dbus
+
 - name: Bootstrap | Check if bootstrap is needed
-  raw: which "{{ item }}"
+  raw: dpkg -l | cut -d' ' -f3 |grep -e ^{{item}}$
   register: need_bootstrap
   failed_when: false
   changed_when: false
-  with_items:
+  with_items: "{{ubuntu_packages}}"
-    - python
+
-    - python-apt
-    - pip
-    - dbus-daemon
   tags:
     - facts
 
 - name: Bootstrap | Install python 2.x and pip
   raw:
     apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y python-minimal python-pip dbus
+    DEBIAN_FRONTEND=noninteractive apt-get install -y {{ubuntu_packages | join(" ")}}
   when:
-    "{{ need_bootstrap.results | map(attribute='rc') | sort | last | bool }}"
+    - need_bootstrap.results | map(attribute='rc') | sort | last | bool
 
 - set_fact:
     ansible_python_interpreter: "/usr/bin/python"

roles/bootstrap-os/tasks/main.yml

@@ -1,32 +1,25 @@
 ---
-- import_tasks: bootstrap-ubuntu.yml
-  when: bootstrap_os == "ubuntu"
 
-- import_tasks: bootstrap-debian.yml
+- include_tasks: bootstrap-ubuntu.yml
-  when: bootstrap_os == "debian"
+  when: os_family == "ubuntu"
 
-- import_tasks: bootstrap-coreos.yml
+- include_tasks: bootstrap-debian.yml
-  when: bootstrap_os == "coreos"
+  when: os_family == "debian"
 
-- import_tasks: bootstrap-fedora.yml
+- include_tasks: bootstrap-coreos.yml
-  when: bootstrap_os == "fedora"
+  when: os_family == "coreos"
 
-- import_tasks: bootstrap-centos.yml
+- include_tasks: bootstrap-fedora.yml
-  when: bootstrap_os == "centos"
+  when: os_family == "fedora"
 
-- import_tasks: bootstrap-opensuse.yml
+- include_tasks: bootstrap-centos.yml
-  when: bootstrap_os == "opensuse"
+  when: os_family == "centos"
+
+- include_tasks: bootstrap-opensuse.yml
+  when: os_family == "opensuse"
 
 - import_tasks: setup-pipelining.yml
 
-- name: check if atomic host
-  stat:
-    path: /run/ostree-booted
-  register: ostree
-
-- set_fact:
-    is_atomic: "{{ ostree.stat.exists }}"
-
 - name: Gather nodes hostnames
   setup:
     gather_subset: '!all'

roles/container-engine/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+## DNS
+dns_domain: cluster.local
+dns_servers: []
+upstream_dns_servers: []
+searchdomains: []

roles/container-engine/docker/defaults/main.yml

@@ -43,3 +43,4 @@ dockerproject_rh_repo_base_url: 'https://yum.dockerproject.org/repo/main/centos/
 dockerproject_rh_repo_gpgkey: 'https://yum.dockerproject.org/gpg'
 dockerproject_apt_repo_base_url: 'https://apt.dockerproject.org/repo'
 dockerproject_apt_repo_gpgkey: 'https://apt.dockerproject.org/gpg'
+docker_bin_dir: "/usr/bin"

roles/container-engine/docker/meta/main.yml

@@ -1,4 +1,4 @@
 ---
 dependencies:
-  - role: docker/docker-storage
+  - role: container-engine/docker/docker-storage
     when: docker_container_storage_setup and ansible_os_family == "RedHat"

roles/container-engine/docker/tasks/main.yml

@@ -1,4 +1,12 @@
 ---
+- name: check if atomic host
+  stat:
+    path: /run/ostree-booted
+  register: ostree
+
+- set_fact:
+    is_atomic: "{{ ostree.stat.exists }}"
+
 - name: gather os specific variables
   include_vars: "{{ item }}"
   with_first_found:
@@ -52,7 +60,7 @@
   register: keyserver_task_result
   until: keyserver_task_result|succeeded
   retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+  delay: "{{ retry_stagger | d(3) }}"
   with_items: "{{ docker_repo_key_info.repo_keys }}"
   when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS", "RedHat", "Suse"] or is_atomic)
 
@@ -73,7 +81,7 @@
   register: keyserver_task_result
   until: keyserver_task_result|succeeded
   retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+  delay: "{{ retry_stagger | d(3) }}"
   with_items: "{{ dockerproject_repo_key_info.repo_keys }}"
   when:
     - not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS", "RedHat", "Suse"] or is_atomic)
@@ -128,7 +136,7 @@
   register: docker_task_result
   until: docker_task_result|succeeded
   retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
+  delay: "{{ retry_stagger | d(3) }}"
   with_items: "{{ docker_package_info.pkgs }}"
   notify: restart docker
   when: not (ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] or is_atomic) and (docker_package_info.pkgs|length > 0)

roles/container-engine/docker/tasks/set_facts_dns.yml

@@ -2,16 +2,11 @@
 
 - name: set dns server for docker
   set_fact:
-    docker_dns_servers: |-
+    docker_dns_servers: "{{dns_servers}}"
-      {%- if dns_mode in ['kubedns', 'coredns'] -%}
+
-        {{ [ skydns_server ] }}
+- name: show docker_dns_servers
-      {%- elif dns_mode == 'coredns_dual' -%}
+  debug:
-        {{ [ skydns_server ] + [ skydns_server_secondary ] }}
+    msg: "{{docker_dns_servers}}"
-      {%- elif dns_mode == 'dnsmasq_kubedns' -%}
-        {{ [ dnsmasq_dns_server ] }}
-      {%- elif dns_mode == 'manual' -%}
-        {{ [ manual_dns_server ] }}
-      {%- endif -%}
 
 - name: set base docker dns facts
   set_fact:
@@ -23,6 +18,7 @@
       - timeout:2
       - attempts:2
 
+
 - name: add upstream dns servers (only when dnsmasq is not used)
   set_fact:
     docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers|default([]) }}"

roles/container-engine/docker/vars/ubuntu-bionic.yml

@@ -1,9 +1,6 @@
 ---
 docker_kernel_min_version: '3.10'
 
-# overide defaults, missing 17.03 for Ubuntu 18.04
-docker_version: '18.06'
-
 use_docker_engine: false
 
 docker_versioned_pkg:

roles/container-engine/meta/main.yml

@@ -0,0 +1,23 @@
+---
+dependencies:
+  - role: container-engine/cri-o
+    when:
+      - container_manager == 'crio'
+    tags:
+      - container-engine
+      - crio
+
+  - role: container-engine/rkt
+    when:
+      - container_manager == 'rkt'
+    tags:
+      - container-engine
+      - rkt
+
+  - role: container-engine/docker
+    when:
+      - container_manager == 'docker' or container_manager == "rkt"
+    tags:
+      - container-engine
+      - docker
+      - rkt

roles/kubespray-defaults/defaults/main.yaml

@@ -59,6 +59,14 @@ skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('addres
 skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipaddr('address') }}"
 dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
 dns_domain: "{{ cluster_name }}"
+kube_dns_servers:
+  kubedns: ["{{skydns_server}}"]
+  coredns: ["{{skydns_server}}"]
+  coredns_dual: "{{[skydns_server] + [ skydns_server_secondary ]}}"
+  manual: ["{{manual_dns_server}}"]
+  dnsmasq_kubedns: ["{{dnsmasq_dns_server}}"]
+
+dns_servers: "{{kube_dns_servers[dns_mode]}}"
 
 # Kubernetes configuration dirs and system namespace.
 # Those are where all the additional config stuff goes

scale.yml

@@ -35,12 +35,7 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
-
+    - { role: container-engine, tags: "container-engine"}
-    - { role: docker, tags: docker, when: container_manager == 'docker' }
-    - { role: cri-o, tags: crio, when: container_manager == 'crio' }
-    - role: rkt
-      tags: rkt
-      when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
     - { role: download, tags: download, when: "not skip_downloads" }
     - { role: etcd, tags: etcd, etcd_cluster_setup: false }
     - { role: vault, tags: vault, when: "cert_management == 'vault'"}

tests/files/gce_centos-weave-kubeadm-sep.yml

@@ -3,6 +3,7 @@ cloud_image_family: centos-7
 cloud_machine_type: "n1-standard-1"
 cloud_region: us-central1-b
 mode: ha
+
 startup_script: ""
 
 # Deployment settings

tests/files/gce_ubuntu18-flannel-aio.yml

@@ -5,10 +5,10 @@ cloud_machine_type: "n1-standard-1"
 mode: aio
 
 # Deployment settings
-
+kubeadm_enabled: false
 bootstrap_os: ubuntu
 kube_network_plugin: flannel
-
+dynamic_kubelet_configuration: true
 deploy_netchecker: true
 kubedns_min_replicas: 1
 cloud_provider: gce

upgrade-cluster.yml

@@ -1,5 +1,5 @@
 ---
-- hosts: localhost
+- hosts: bastion[0]
   gather_facts: False
   roles:
     - { role: kubespray-defaults}
@@ -34,11 +34,7 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
-    - { role: docker, tags: docker, when: container_manager == 'docker' }
+    - { role: container-engine, tags: "container-engine"}
-    - { role: cri-o, tags: crio, when: container_manager == 'crio' }
-    - role: rkt
-      tags: rkt
-      when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
     - { role: download, tags: download, when: "not skip_downloads" }
   environment: "{{proxy_env}}"