Fedora CoreOS support (#5657)

* fedora coreos support
- bootstrap and new fact for

* fedora coreos support
- fix bootstrap condition

* fedora coreos support
- allow customize packages for fedora coreos bootstrap

* fedora coreos support
- prevent install ptyhon3 and epel via dnf for fedora coreos

* fedora coreos support
- handle all ostree like os in same way

* fedora coreos support
- handle all ostree like os in same way for crio

* fedora coreos support
- add fcos documentations

README.md

@@ -83,6 +83,7 @@ vagrant up
 - [Network plugins](#network-plugins)
 - [Vagrant install](docs/vagrant.md)
 - [CoreOS bootstrap](docs/coreos.md)
+- [Fedora CoreOS bootstrap](docs/fcos.md)
 - [Debian Jessie setup](docs/debian.md)
 - [openSUSE setup](docs/opensuse.md)
 - [Downloaded artifacts](docs/downloads.md)
@@ -105,6 +106,7 @@ vagrant up
 - **CentOS/RHEL** 7
 - **Fedora** 28
 - **Fedora/CentOS** Atomic
+- **Fedora CoreOS** (experimental: see [fcos Note](docs/fcos.md)
 - **openSUSE** Leap 42.3/Tumbleweed
 - **Oracle Linux** 7
 

docs/fcos.md

@@ -0,0 +1,76 @@
+# Fedora CoreOS
+
+Tested with stable version 31.20200223.3.0
+Because package installation with `rpm-ostree` requires a reboot, playbook may fail while bootstrap.
+Restart playbook again.
+
+## Containers
+
+Tested with
+
+- docker
+- crio
+
+### docker
+
+OS base packages contains docker.
+
+### cri-o
+
+To use `cri-o` disable docker service with ignition:
+
+```yaml
+#workaround, see https://github.com/coreos/fedora-coreos-tracker/issues/229
+systemd:
+  units:
+    - name: docker.service
+      enabled: false
+      contents: |
+        [Unit]
+        Description=disable docker
+
+        [Service]
+
+        [Install]
+        WantedBy=multi-user.target
+```
+
+## libvirt setup
+
+### Prepare
+
+Prepare ignition and serve via http (a.e. python -m SimpleHTTPServer )
+
+```json
+{
+  "ignition": {
+     "version": "3.0.0"
+  },
+
+  "passwd": {
+    "users": [
+      {
+        "name": "adi",
+        "passwordHash": "$1$.RGu8J4x$U7uxcOg/eotTEIRxhk62I0",
+        "sshAuthorizedKeys": [
+          "ssh-rsa ..fillyouruser"
+        ],
+        "groups": [ "wheel" ]
+      }
+    ]
+  }
+}
+```
+
+### create guest
+
+```shell script
+fcos_version=31.20200223.3.0
+kernel=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-kernel-x86_64
+initrd=https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/${fcos_version}/x86_64/fedora-coreos-${fcos_version}-live-initramfs.x86_64.img
+ignition_url=http://mywebserver/fcos.ign
+kernel_args="ip=dhcp rd.neednet=1 console=tty0 coreos.liveiso=/ console=ttyS0 coreos.inst.install_dev=/dev/sda coreos.inst.stream=stable coreos.inst.ignition_url=${ignition_url}"
+sudo virt-install --name ${machine_name} --ram 4048 --graphics=none --vcpus 2 --disk size=20 \
+                --network bridge=virbr0 \
+                --install kernel=${kernel},initrd=${initrd},kernel_args_overwrite=yes,kernel_args="${kernel_args}"
+```

roles/bootstrap-os/defaults/main.yml

@@ -13,6 +13,13 @@ coreos_locksmithd_disable: false
 # Install public repo on Oracle Linux
 use_oracle_public_repo: true
 
+fedora_coreos_packages:
+  - python
+  - libselinux-python3
+  - dbus-tools              # because of networkManager reload bug (https://bugzilla.redhat.com/show_bug.cgi?id=1745659)
+  - ethtool                 # required in kubeadm preflight phase for verifying the environment
+  - ipset                   # required in kubeadm preflight phase for verifying the environment
+
 ## General
 # Set the hostname to inventory_hostname
 override_system_hostname: true

roles/bootstrap-os/tasks/bootstrap-fedora-coreos.yml

@@ -0,0 +1,35 @@
+---
+
+- name: Check if bootstrap is needed
+  raw: which python
+  register: need_bootstrap
+  failed_when: false
+  changed_when: false
+  tags:
+    - facts
+
+- name: Install required packages on fedora coreos
+  raw: "export http_proxy={{ http_proxy | default('') }};rpm-ostree install {{ fedora_coreos_packages|join(' ') }}"
+  become: true
+  when: need_bootstrap.rc != 0
+
+# playbook fails because connection lost
+- name: Reboot immediately for updated ostree, please run playbook again if failed first time.
+  raw: "nohup bash -c 'sleep 5s && shutdown -r now'"
+  become: true
+  ignore_errors: yes
+  when: need_bootstrap.rc != 0
+
+- name: Wait for the reboot to complete
+  wait_for_connection:
+    timeout: 240
+    connect_timeout: 20
+    delay: 5
+    sleep: 5
+  when: need_bootstrap.rc != 0
+
+- name: Store the fact if this is an fedora core os host
+  set_fact:
+    is_fedora_coreos: True
+  tags:
+    - facts

roles/bootstrap-os/tasks/main.yml

@@ -13,14 +13,21 @@
 - include_tasks: bootstrap-clearlinux.yml
   when: '"Clear Linux OS" in os_release.stdout'
 
+- include_tasks: bootstrap-fedora-coreos.yml
+  when: '"ID=fedora" in os_release.stdout and "VARIANT_ID=coreos" in os_release.stdout'
+
 - include_tasks: bootstrap-coreos.yml
-  when: '"CoreOS" in os_release.stdout or "Flatcar" in os_release.stdout'
+  when:
+    - '"CoreOS" in os_release.stdout or "Flatcar" in os_release.stdout'
+    - '"ID=fedora" not in os_release.stdout'
 
 - include_tasks: bootstrap-debian.yml
   when: '"Debian" in os_release.stdout or "Ubuntu" in os_release.stdout'
 
 - include_tasks: bootstrap-fedora.yml
-  when: '"Fedora" in os_release.stdout'
+  when:
+    - '"Fedora" in os_release.stdout'
+    - '"VARIANT_ID=coreos" not in os_release.stdout'
 
 - include_tasks: bootstrap-opensuse.yml
   when: '"openSUSE" in os_release.stdout'
@@ -43,7 +50,7 @@
     name: "{{ inventory_hostname }}"
   when:
     - override_system_hostname
-    - ansible_os_family not in ['Suse', 'Container Linux by CoreOS', 'Flatcar Container Linux by Kinvolk', 'ClearLinux']
+    - ansible_os_family not in ['Suse', 'Container Linux by CoreOS', 'Flatcar Container Linux by Kinvolk', 'ClearLinux'] and not is_fedora_coreos
 
 # (2/3)
 - name: Assign inventory name to unconfigured hostnames (CoreOS, non-Flatcar, Suse and ClearLinux only)
@@ -52,7 +59,7 @@
   changed_when: false
   when:
     - override_system_hostname
-    - ansible_os_family in ['Suse', 'Container Linux by CoreOS', 'Flatcar Container Linux by Kinvolk', 'ClearLinux']
+    - ansible_os_family in ['Suse', 'Container Linux by CoreOS', 'Flatcar Container Linux by Kinvolk', 'ClearLinux'] or is_fedora_coreos
 
 # (3/3)
 - name: Update hostname fact (CoreOS, Flatcar, Suse and ClearLinux only)
@@ -61,7 +68,7 @@
     filter: ansible_hostname
   when:
     - override_system_hostname
-    - ansible_os_family in ['Suse', 'Flatcar Container Linux by Kinvolk', 'Container Linux by CoreOS', 'ClearLinux']
+    - ansible_os_family in ['Suse', 'Flatcar Container Linux by Kinvolk', 'Container Linux by CoreOS', 'ClearLinux'] or is_fedora_coreos
 
 - name: "Install ceph-commmon package"
   package:

roles/container-engine/cri-o/tasks/main.yaml

@@ -1,4 +1,15 @@
 ---
+
+- name: check if atomic host or fedora coreos
+  stat:
+    path: /run/ostree-booted
+  register: ostree
+
+- name: set is_ostree
+  set_fact:
+    is_ostree: "{{ ostree.stat.exists }}"
+
+
 - name: gather os specific variables
   include_vars: "{{ item }}"
   with_first_found:
@@ -22,7 +33,7 @@
     description: OpenShift Origin Repo
     baseurl: "{{ crio_rhel_repo_base_url }}"
     gpgcheck: no
-  when: ansible_distribution in ["CentOS","RedHat","OracleLinux"] and not is_atomic
+  when: ansible_distribution in ["CentOS","RedHat","OracleLinux"] and not is_ostree
 
 - name: Add CRI-O PPA
   apt_repository:
@@ -51,8 +62,25 @@
   package:
     name: "{{ item }}"
     state: present
+  when: not is_ostree
   with_items: "{{ crio_packages }}"
 
+- name: Check if already installed
+  stat:
+    path: "/bin/crio"
+  register: need_bootstrap_crio
+  when: is_ostree
+
+- name: Install cri-o packages with osttree
+  raw: "export http_proxy={{ http_proxy | default('') }} && rpm-ostree install {{ crio_packages|join(' ') }}"
+  when: is_ostree and not need_bootstrap_crio.stat.exists
+  become: true
+
+- name: Reboot immediately for updated ostree
+  reboot:
+  become: true
+  when: is_ostree and not need_bootstrap_crio.stat.exists
+
 - name: Install cri-o config
   template:
     src: crio.conf.j2

roles/container-engine/cri-o/templates/crio.conf.j2

@@ -104,7 +104,7 @@ selinux = {{ (preinstall_selinux_state == 'enforcing')|lower }}
 # for the runtime.
 {% if ansible_os_family == "ClearLinux" %}
 seccomp_profile = "/usr/share/defaults/crio/seccomp.json"
-{% elif ansible_distribution == "Ubuntu" %}
+{% elif ansible_distribution == "Ubuntu" or is_fedora_coreos %}
 seccomp_profile = ""
 {% else %}
 seccomp_profile = "/etc/crio/seccomp.json"
@@ -242,7 +242,7 @@ ctr_stop_timeout = 0
 default_transport = "docker://"
 
 # The image used to instantiate infra containers.
-pause_image = "docker://k8s.gcr.io/pause:3.1"
+pause_image = "docker://{{kube_image_repo}}/pause:3.1"
 
 # If not empty, the path to a docker/config.json-like file containing credentials
 # necessary for pulling the image specified by pause_image above.

roles/container-engine/docker/handlers/main.yml

@@ -15,7 +15,7 @@
   service:
     name: docker.socket
     state: restarted
-  when: ansible_os_family in ['Coreos', 'CoreOS', 'Container Linux by CoreOS', 'Flatcar', 'Flatcar Container Linux by Kinvolk']
+  when: ansible_os_family in ['Coreos', 'CoreOS', 'Container Linux by CoreOS', 'Flatcar', 'Flatcar Container Linux by Kinvolk'] or is_fedora_coreos
 
 - name: Docker | reload docker
   service:

roles/container-engine/docker/tasks/main.yml

@@ -1,12 +1,12 @@
 ---
-- name: check if atomic host
+- name: check if atomic host or fedora coreos
   stat:
     path: /run/ostree-booted
   register: ostree
 
-- name: set is_atomic
+- name: set is_ostree
   set_fact:
-    is_atomic: "{{ ostree.stat.exists }}"
+    is_ostree: "{{ ostree.stat.exists }}"
 
 - name: gather os specific variables
   include_vars: "{{ item }}"
@@ -60,7 +60,7 @@
   retries: 4
   delay: "{{ retry_stagger | d(3) }}"
   with_items: "{{ docker_repo_key_info.repo_keys }}"
-  when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat", "Suse", "ClearLinux"] or is_atomic)
+  when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat", "Suse", "ClearLinux"] or is_ostree)
 
 - name: ensure docker-ce repository is enabled
   action: "{{ docker_repo_info.pkg_repo }}"
@@ -68,7 +68,7 @@
     repo: "{{ item }}"
     state: present
   with_items: "{{ docker_repo_info.repos }}"
-  when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat", "Suse", "ClearLinux"] or is_atomic) and (docker_repo_info.repos|length > 0)
+  when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat", "Suse", "ClearLinux"] or is_ostree) and (docker_repo_info.repos|length > 0)
 
 - name: ensure docker-engine repository public key is installed
   action: "{{ dockerproject_repo_key_info.pkg_key }}"
@@ -82,7 +82,7 @@
   delay: "{{ retry_stagger | d(3) }}"
   with_items: "{{ dockerproject_repo_key_info.repo_keys }}"
   when:
-    - not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat", "Suse", "ClearLinux"] or is_atomic)
+    - not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat", "Suse", "ClearLinux"] or is_ostree)
     - use_docker_engine is defined and use_docker_engine
 
 - name: ensure docker-engine repository is enabled
@@ -93,13 +93,13 @@
   with_items: "{{ dockerproject_repo_info.repos }}"
   when:
     - use_docker_engine is defined and use_docker_engine
-    - not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat", "Suse", "ClearLinux"] or is_atomic) and (dockerproject_repo_info.repos|length > 0)
+    - not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "RedHat", "Suse", "ClearLinux"] or is_ostree) and (dockerproject_repo_info.repos|length > 0)
 
 - name: Configure docker repository on Fedora
   template:
     src: "fedora_docker.repo.j2"
     dest: "{{ yum_repo_dir }}/docker.repo"
-  when: ansible_distribution == "Fedora" and not is_atomic
+  when: ansible_distribution == "Fedora" and not is_ostree
 
 - name: Configure docker repository on RedHat/CentOS/Oracle Linux
   yum_repository:
@@ -110,13 +110,13 @@
     gpgkey: "{{ docker_rh_repo_gpgkey }}"
     keepcache: "{{ docker_rpm_keepcache | default('1') }}"
     proxy: " {{ http_proxy | default('_none_') }}"
-  when: ansible_distribution in ["CentOS","RedHat","OracleLinux"] and not is_atomic
+  when: ansible_distribution in ["CentOS","RedHat","OracleLinux"] and not is_ostree
 
 - name: check if container-selinux is available
   yum:
     list: "container-selinux"
   register: yum_result
-  when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic
+  when: ansible_distribution in ["CentOS","RedHat"] and not is_ostree
 
 - name: Configure extras repository on RedHat/CentOS if container-selinux is not available in current repos
   yum_repository:
@@ -130,7 +130,7 @@
     keepcache: "{{ docker_rpm_keepcache | default('1') }}"
     proxy: " {{ http_proxy | default('_none_') }}"
   when:
-    - ansible_distribution in ["CentOS","RedHat"] and not is_atomic
+    - ansible_distribution in ["CentOS","RedHat"] and not is_ostree
     - yum_result.results | length == 0
 
 - name: Copy yum.conf for editing
@@ -138,7 +138,7 @@
     src: "{{ yum_conf }}"
     dest: "{{ docker_yum_conf }}"
     remote_src: yes
-  when: ansible_distribution in ["CentOS","RedHat","OracleLinux"] and not is_atomic
+  when: ansible_distribution in ["CentOS","RedHat","OracleLinux"] and not is_ostree
 
 - name: Edit copy of yum.conf to set obsoletes=0
   lineinfile:
@@ -146,7 +146,7 @@
     state: present
     regexp: '^obsoletes='
     line: 'obsoletes=0'
-  when: ansible_distribution in ["CentOS","RedHat","OracleLinux"] and not is_atomic
+  when: ansible_distribution in ["CentOS","RedHat","OracleLinux"] and not is_ostree
 
 - name: ensure docker packages are installed
   action: "{{ docker_package_info.pkg_mgr }}"
@@ -162,7 +162,7 @@
   delay: "{{ retry_stagger | d(3) }}"
   with_items: "{{ docker_package_info.pkgs }}"
   notify: restart docker
-  when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_atomic) and (docker_package_info.pkgs|length > 0)
+  when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_ostree) and (docker_package_info.pkgs|length > 0)
   ignore_errors: true
 
 - name: Ensure docker packages are installed

roles/container-engine/docker/tasks/pre-upgrade.yml

@@ -5,7 +5,7 @@
     state: absent
   when:
     - ansible_distribution in ["CentOS","RedHat","OracleLinux"]
-    - not is_atomic
+    - not is_ostree
 
 - name: Ensure old versions of Docker are not installed. | Debian
   apt:
@@ -22,4 +22,4 @@
   when:
     - ansible_os_family == 'RedHat'
     - (docker_versioned_pkg[docker_version | string] is search('docker-ce'))
-    - not is_atomic
+    - not is_ostree

roles/container-engine/docker/tasks/systemd.yml

@@ -15,7 +15,7 @@
   # noqa 303 - systemctl is called intentionally here
   shell: systemctl --version | head -n 1 | cut -d " " -f 2
   register: systemd_version
-  when: not is_atomic
+  when: not is_ostree
   changed_when: false
 
 - name: Write docker.service systemd file
@@ -24,7 +24,7 @@
     dest: /etc/systemd/system/docker.service
   register: docker_service_file
   notify: restart docker
-  when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk"] or is_atomic)
+  when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk"] or is_ostree)
 
 - name: Write docker options systemd drop-in
   template:

roles/container-engine/docker/templates/docker.service.j2

@@ -32,7 +32,7 @@ ExecStart={{ docker_bin_dir }}/docker{% if installed_docker_version.stdout is ve
           $DOCKER_NETWORK_OPTIONS \
           $DOCKER_DNS_OPTIONS \
           $INSECURE_REGISTRY
-{% if not is_atomic and systemd_version.stdout|int >= 226 %}
+{% if not is_ostree and systemd_version.stdout|int >= 226 %}
 TasksMax=infinity
 {% endif %}
 LimitNOFILE=1048576

roles/etcd/meta/main.yml

@@ -2,7 +2,7 @@
 dependencies:
   - role: adduser
     user: "{{ addusers.etcd }}"
-    when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_atomic)
+    when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_atomic or is_fedora_coreos)
   - role: adduser
     user: "{{ addusers.kube }}"
-    when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_atomic)
+    when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_atomic or is_fedora_coreos)

roles/kubernetes/preinstall/meta/main.yml

@@ -2,6 +2,8 @@
 dependencies:
   - role: adduser
     user: "{{ addusers.kube }}"
-    when: not is_atomic
+    when:
+      - not is_atomic
+      - not is_fedora_coreos
     tags:
       - kubelet

roles/kubernetes/preinstall/tasks/0040-set_facts.yml

@@ -27,9 +27,21 @@
     path: /run/ostree-booted
   register: ostree
 
+- name: set is_fedora_coreos
+  lineinfile:
+    path: /etc/os-release
+    line: "VARIANT_ID=coreos"
+    state: present
+  check_mode: yes
+  register: os_variant_coreos
+
+- name: set is_fedora_coreos
+  set_fact:
+    is_fedora_coreos: "{{ ostree.stat.exists and os_variant_coreos is not changed }}"
+
 - name: set is_atomic
   set_fact:
-    is_atomic: "{{ ostree.stat.exists }}"
+    is_atomic: "{{ ostree.stat.exists and not is_fedora_coreos }}"
 
 - name: set kube_cert_group on atomic hosts
   set_fact:

roles/kubernetes/preinstall/tasks/0070-system-packages.yml

@@ -24,6 +24,7 @@
   when:
     - ansible_distribution in ["CentOS","RedHat","OracleLinux"]
     - not is_atomic
+    - not is_fedora_coreos
 
 - name: Install python-dnf for latest RedHat versions
   command: dnf install -y python-dnf yum
@@ -36,6 +37,7 @@
     - ansible_distribution_major_version|int > 21
     - ansible_distribution_major_version|int <= 29
     - not is_atomic
+    - not is_fedora_coreos
   changed_when: False
   tags:
     - bootstrap-os
@@ -50,6 +52,7 @@
     - ansible_distribution == "Fedora"
     - ansible_distribution_major_version|int >= 30
     - not is_atomic
+    - not is_fedora_coreos
   changed_when: False
   tags:
     - bootstrap-os
@@ -61,6 +64,7 @@
   when:
     - ansible_distribution in ["CentOS","RedHat"]
     - not is_atomic
+    - not is_fedora_coreos
     - epel_enabled|bool
   tags:
     - bootstrap-os
@@ -79,7 +83,7 @@
   until: pkgs_task_result is succeeded
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
-  when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_atomic)
+  when: not (ansible_os_family in ["CoreOS", "Coreos", "Container Linux by CoreOS", "Flatcar", "Flatcar Container Linux by Kinvolk", "ClearLinux"] or is_atomic or is_fedora_coreos)
   tags:
     - bootstrap-os
 

roles/kubespray-defaults/defaults/main.yaml

@@ -10,6 +10,7 @@ kube_api_anonymous_auth: true
 
 # Default value, but will be set to true automatically if detected
 is_atomic: false
+is_fedora_coreos: false
 
 # optional disable the swap
 disable_swap: true