Merge branch 'master' into upgrade-to-kubeadm2

.gitlab-ci.yml

@@ -116,8 +116,8 @@ before_script:
       -e ansible_python_interpreter=${PYPATH}
       -e ansible_ssh_user=${SSH_USER}
       -e bootstrap_os=${BOOTSTRAP_OS}
-      -e cert_management=${CERT_MGMT:-script}
       -e cloud_provider=gce
+      -e cert_management=${CERT_MGMT:-script}
       -e "{deploy_netchecker: true}"
       -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
       -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
@@ -185,6 +185,7 @@ before_script:
       -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       --private-key=${HOME}/.ssh/id_rsa
       -e bootstrap_os=${BOOTSTRAP_OS}
+      -e cloud_provider=gce
       -e ansible_python_interpreter=${PYPATH}
       -e "{deploy_netchecker: true}"
       -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
@@ -219,6 +220,7 @@ before_script:
       -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       --private-key=${HOME}/.ssh/id_rsa
       -e bootstrap_os=${BOOTSTRAP_OS}
+      -e cloud_provider=gce
       -e ansible_python_interpreter=${PYPATH}
       -e reset_confirmation=yes
       --limit "all:!fake_hosts"
@@ -232,6 +234,7 @@ before_script:
       -b --become-user=root -e cloud_provider=gce $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       --private-key=${HOME}/.ssh/id_rsa
       -e bootstrap_os=${BOOTSTRAP_OS}
+      -e cloud_provider=gce
       -e ansible_python_interpreter=${PYPATH}
       -e "{deploy_netchecker: true}"
       -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"

cluster.yml

@@ -62,7 +62,6 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/node, tags: node }
-    - { role: network_plugin, tags: network }
 
 - hosts: kube-master
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
@@ -75,6 +74,7 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" }
+    - { role: network_plugin, tags: network }
 
 - hosts: kube-master
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"

docs/getting-started.md

@@ -28,10 +28,10 @@ an example inventory located
 You can use an
 [inventory generator](https://github.com/kubernetes-incubator/kubespray/blob/master/contrib/inventory_builder/inventory.py)
 to create or modify an Ansible inventory. Currently, it is limited in
-functionality and is only use for making a basic Kubespray cluster, but it does
+functionality and is only used for configuring a basic Kubespray cluster inventory, but it does
-support creating large clusters. It now supports
+support creating inventory file for large clusters as well. It now supports
 separated ETCD and Kubernetes master roles from node role if the size exceeds a
-certain threshold. Run inventory.py help for more information.
+certain threshold. Run `python3 contrib/inventory_builder/inventory.py help` help for more information.
 
 Example inventory generator usage:
 
@@ -59,7 +59,7 @@ See more details in the [ansible guide](ansible.md).
 Adding nodes
 ------------
 
-You may want to add worker nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your masters. This is especially helpful when doing something like autoscaling your clusters.
+You may want to add **worker** nodes to your existing cluster. This can be done by re-running the `cluster.yml` playbook, or you can target the bare minimum needed to get kubelet installed on the worker and talking to your masters. This is especially helpful when doing something like autoscaling your clusters.
 
 - Add the new worker node to your inventory under kube-node (or utilize a [dynamic inventory](https://docs.ansible.com/ansible/intro_dynamic_inventory.html)).
 - Run the ansible-playbook command, substituting `scale.yml` for `cluster.yml`:
@@ -75,7 +75,7 @@ kube-apiserver via port 8080. A kubeconfig file is not necessary in this case,
 because kubectl will use http://localhost:8080 to connect. The kubeconfig files
 generated will point to localhost (on kube-masters) and kube-node hosts will
 connect either to a localhost nginx proxy or to a loadbalancer if configured.
-More details on this process is in the [HA guide](ha.md).
+More details on this process are in the [HA guide](ha.md).
 
 Kubespray permits connecting to the cluster remotely on any IP of any 
 kube-master host on port 6443 by default. However, this requires 

docs/vars.md

@@ -109,6 +109,9 @@ Stack](https://github.com/kubernetes-incubator/kubespray/blob/master/docs/dns-st
   dynamic kernel services are needed for mounting persistent volumes into containers.  These may not be
   loaded by preinstall kubernetes processes.  For example, ceph and rbd backed volumes.  Set this variable to
   true to let kubelet load kernel modules.
+* *kubelet_cgroup_driver* - Allows manual override of the
+  cgroup-driver option for Kubelet. By default autodetection is used
+  to match Docker configuration.
 
 ##### Custom flags for Kube Components
 For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. This can be done by providing a list of flags. Example:
@@ -126,5 +129,8 @@ The possible vars are:
 
 #### User accounts
 
-Kubespray sets up two Kubernetes accounts by default: ``root`` and ``kube``. Their
+By default, a user with admin rights is created, named `kube`.
-passwords default to changeme. You can set this by changing ``kube_api_pwd``.
+The password can be viewed after deployment by looking at the file
+`PATH_TO_KUBESPRAY/credentials/kube_user`. This contains a randomly generated
+password. If you wish to set your own password, just precreate/modify this
+file yourself or change `kube_api_pwd` var.

roles/docker/defaults/main.yml

@@ -12,5 +12,7 @@ docker_repo_info:
 
 docker_dns_servers_strict: yes
 
+docker_container_storage_setup: false
+
 docker_rh_repo_base_url: 'https://yum.dockerproject.org/repo/main/centos/7'
 docker_rh_repo_gpgkey: 'https://yum.dockerproject.org/gpg'

roles/docker/docker-storage/defaults/main.yml

@@ -0,0 +1,15 @@
+---
+docker_container_storage_setup_version: v0.6.0
+docker_container_storage_setup_profile_name: kubespray
+docker_container_storage_setup_storage_driver: devicemapper
+docker_container_storage_setup_container_thinpool: docker-pool
+docker_container_storage_setup_data_size: 40%FREE
+docker_container_storage_setup_min_data_size: 2G
+docker_container_storage_setup_chunk_size: 512K
+docker_container_storage_setup_growpart: false
+docker_container_storage_setup_auto_extend_pool: yes
+docker_container_storage_setup_pool_autoextend_threshold: 60
+docker_container_storage_setup_pool_autoextend_percent: 20
+docker_container_storage_setup_device_wait_timeout: 60
+docker_container_storage_setup_wipe_signatures: false
+docker_container_storage_setup_container_root_lv_size: 40%FREE

roles/docker/docker-storage/files/install_container_storage_setup.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -e
+
+version=${1:-master}
+profile_name=${2:-kubespray}
+dir=`mktemp -d`
+export GIT_DIR=$dir/.git
+export GIT_WORK_TREE=$dir
+
+git init
+git fetch --depth 1 https://github.com/projectatomic/container-storage-setup.git $version
+git merge FETCH_HEAD
+make -C $dir install
+rm -rf /var/lib/container-storage-setup/$profile_name $dir
+
+set +e
+
+/usr/bin/container-storage-setup create $profile_name /etc/sysconfig/docker-storage-setup && /usr/bin/container-storage-setup activate $profile_name
+# FIXME: exit status can be 1 for both fatal and non fatal errors in current release, 
+# could be improved by matching error strings 
+exit 0

roles/docker/docker-storage/tasks/main.yml

@@ -0,0 +1,37 @@
+---
+
+- name: docker-storage-setup | install git and make
+  with_items: [git, make]
+  package:
+    pkg: "{{ item }}"
+    state: present
+
+- name: docker-storage-setup | docker-storage-setup sysconfig template
+  template:
+    src: docker-storage-setup.j2
+    dest: /etc/sysconfig/docker-storage-setup
+
+- name: docker-storage-override-directory | docker service storage-setup override dir
+  file:
+    dest: /etc/systemd/system/docker.service.d
+    mode: 0755
+    owner: root
+    group: root
+    state: directory
+
+- name: docker-storage-override | docker service storage-setup override file
+  copy:
+    dest: /etc/systemd/system/docker.service.d/override.conf
+    content: |-
+      ### Thie file is managed by Ansible
+      [Service]
+      EnvironmentFile=-/etc/sysconfig/docker-storage
+
+    owner: root
+    group: root
+    mode: 0644
+
+- name: docker-storage-setup | install and run container-storage-setup
+  become: yes
+  script: install_container_storage_setup.sh {{ docker_container_storage_setup_version }} {{ docker_container_storage_setup_profile_name }}
+  notify: Docker | reload systemd

roles/docker/docker-storage/templates/docker-storage-setup.j2

@@ -0,0 +1,35 @@
+{%if docker_container_storage_setup_storage_driver is defined%}STORAGE_DRIVER={{docker_container_storage_setup_storage_driver}}{%endif%}
+
+{%if docker_container_storage_setup_extra_storage_options is defined%}EXTRA_STORAGE_OPTIONS={{docker_container_storage_setup_extra_storage_options}}{%endif%}
+
+{%if docker_container_storage_setup_devs is defined%}DEVS={{docker_container_storage_setup_devs}}{%endif%}
+
+{%if docker_container_storage_setup_container_thinpool is defined%}CONTAINER_THINPOOL={{docker_container_storage_setup_container_thinpool}}{%endif%}
+
+{%if docker_container_storage_setup_vg is defined%}VG={{docker_container_storage_setup_vg}}{%endif%}
+
+{%if docker_container_storage_setup_root_size is defined%}ROOT_SIZE={{docker_container_storage_setup_root_size}}{%endif%}
+
+{%if docker_container_storage_setup_data_size is defined%}DATA_SIZE={{docker_container_storage_setup_data_size}}{%endif%}
+
+{%if docker_container_storage_setup_min_data_size is defined%}MIN_DATA_SIZE={{docker_container_storage_setup_min_data_size}}{%endif%}
+
+{%if docker_container_storage_setup_chunk_size is defined%}CHUNK_SIZE={{docker_container_storage_setup_chunk_size}}{%endif%}
+
+{%if docker_container_storage_setup_growpart is defined%}GROWPART={{docker_container_storage_setup_growpart}}{%endif%}
+
+{%if docker_container_storage_setup_auto_extend_pool is defined%}AUTO_EXTEND_POOL={{docker_container_storage_setup_auto_extend_pool}}{%endif%}
+
+{%if docker_container_storage_setup_pool_autoextend_threshold is defined%}POOL_AUTOEXTEND_THRESHOLD={{docker_container_storage_setup_pool_autoextend_threshold}}{%endif%}
+
+{%if docker_container_storage_setup_pool_autoextend_percent is defined%}POOL_AUTOEXTEND_PERCENT={{docker_container_storage_setup_pool_autoextend_percent}}{%endif%}
+
+{%if docker_container_storage_setup_device_wait_timeout is defined%}DEVICE_WAIT_TIMEOUT={{docker_container_storage_setup_device_wait_timeout}}{%endif%}
+
+{%if docker_container_storage_setup_wipe_signatures is defined%}WIPE_SIGNATURES={{docker_container_storage_setup_wipe_signatures}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_name is defined%}CONTAINER_ROOT_LV_NAME={{docker_container_storage_setup_container_root_lv_name}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_size is defined%}CONTAINER_ROOT_LV_SIZE={{docker_container_storage_setup_container_root_lv_size}}{%endif%}
+
+{%if docker_container_storage_setup_container_root_lv_mount_path is defined%}CONTAINER_ROOT_LV_MOUNT_PATH={{docker_container_storage_setup_container_root_lv_mount_path}}{%endif%}

roles/docker/meta/main.yml

@@ -0,0 +1,4 @@
+---
+dependencies:
+  - role: docker/docker-storage
+    when: docker_container_storage_setup and ansible_os_family == "RedHat"

roles/download/defaults/main.yml

@@ -28,22 +28,18 @@ calico_version: "v2.5.0"
 calico_ctl_version: "v1.5.0"
 calico_cni_version: "v1.10.0"
 calico_policy_version: "v0.7.0"
-weave_version: 2.0.1
+weave_version: 2.0.4
 flannel_version: "v0.8.0"
 flannel_cni_version: "v0.2.0"
 pod_infra_version: 3.0
 
 # Download URLs
-etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd"
 kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/amd64/kubeadm"
 
 # Checksums
-etcd_checksum: "274c46a7f8d26f7ae99d6880610f54933cbcf7f3beafa19236c52eb5df8c7a0b"
 kubeadm_checksum: "8f6ceb26b8503bfc36a99574cf6f853be1c55405aa31669561608ad8099bf5bf"
 
 # Containers
-# Possible values: host, docker
-etcd_deployment_type: "docker"
 etcd_image_repo: "quay.io/coreos/etcd"
 etcd_image_tag: "{{ etcd_version }}"
 flannel_image_repo: "quay.io/coreos/flannel"
@@ -124,18 +120,10 @@ downloads:
     sha256: "{{ netcheck_agent_digest_checksum|default(None) }}"
     enabled: "{{ deploy_netchecker|bool }}"
   etcd:
-    version: "{{etcd_version}}"
+    container: true
-    dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
-    sha256: >-
-      {%- if etcd_deployment_type in [ 'docker', 'rkt' ] -%}{{etcd_digest_checksum|default(None)}}{%- else -%}{{etcd_checksum}}{%- endif -%}
-    source_url: "{{ etcd_download_url }}"
-    url: "{{ etcd_download_url }}"
-    unarchive: true
-    owner: "etcd"
-    mode: "0755"
-    container: "{{ etcd_deployment_type in [ 'docker', 'rkt' ] }}"
     repo: "{{ etcd_image_repo }}"
     tag: "{{ etcd_image_tag }}"
+    sha256: "{{etcd_digest_checksum|default(None)}}"
   kubeadm:
     version: "{{ kubeadm_version }}"
     dest: "kubeadm"

roles/download/tasks/set_docker_image_facts.yml

@@ -26,3 +26,5 @@
   assert:
     that: "{{download.repo}}:{{download.tag}} in docker_images.stdout.split(',')"
   when: not download_always_pull|bool and not pull_required|bool and pull_by_digest|bool
+  tags:
+    - asserts

roles/etcd/defaults/main.yml

@@ -3,7 +3,6 @@
 etcd_cluster_setup: true
 
 etcd_backup_prefix: "/var/backups"
-etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"
 etcd_data_dir: "/var/lib/etcd"
 
 etcd_config_dir: /etc/ssl/etcd
@@ -23,6 +22,8 @@ etcd_memory_limit: 512M
 # Uncomment to set CPU share for etcd
 # etcd_cpu_limit: 300m
 
+etcd_blkio_weight: 1000
+
 etcd_node_cert_hosts: "{{ groups['k8s-cluster'] | union(groups.get('calico-rr', [])) }}"
 
 etcd_compaction_retention: "8"

roles/etcd/tasks/gen_certs_script.yml

@@ -115,7 +115,7 @@
 
 # FIXME(mattymo): Use tempfile module in ansible 2.3
 - name: Gen_certs | Prepare tempfile for unpacking certs
-  shell: mktemp /tmp/certsXXXXX.tar.gz
+  command: mktemp /tmp/certsXXXXX.tar.gz
   register: cert_tempfile
   when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
         inventory_hostname != groups['etcd'][0]

roles/etcd/tasks/install_docker.yml

@@ -1,5 +1,4 @@
 ---
-# Plan A: no docker-py deps
 - name: Install | Copy etcdctl binary from docker container
   command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy;
            {{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} &&

roles/etcd/templates/etcd.j2

@@ -12,6 +12,9 @@
   {% if etcd_cpu_limit is defined %}
   --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \
   {% endif %}
+  {% if etcd_blkio_weight is defined %}
+  --blkio-weight={{ etcd_blkio_weight }} \
+  {% endif %}
   --name={{ etcd_member_name | default("etcd") }} \
   {{ etcd_image_repo }}:{{ etcd_image_tag }} \
   {% if etcd_after_v3 %}

roles/kubernetes/master/tasks/pre-upgrade.yml

@@ -17,14 +17,14 @@
   file:
     path: "/etc/kubernetes/manifests/{{item}}.manifest"
     state: absent
-  with_nested:
+  with_items:
     - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
   register: kube_apiserver_manifest_replaced
   when: (secret_changed|default(false) or etcd_secret_changed|default(false))
 
 - name: "Pre-upgrade | Delete master containers forcefully"
   shell: "docker ps -f name=k8s-{{item}}* -q | xargs --no-run-if-empty docker rm -f"
-  with_nested:
+  with_items:
     - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
   when: kube_apiserver_manifest_replaced.changed
   run_once: true

roles/kubernetes/node/tasks/facts.yml

@@ -0,0 +1,8 @@
+- name: look up docker cgroup driver
+  shell: "docker info | grep 'Cgroup Driver' | awk -F': ' '{ print $2; }'"
+  register: docker_cgroup_driver_result
+
+- set_fact:
+    standalone_kubelet: >-
+      {%- if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] -%}true{%- else -%}false{%- endif -%}
+    kubelet_cgroup_driver_detected: "{{ docker_cgroup_driver_result.stdout }}"

roles/kubernetes/node/tasks/main.yml

@@ -1,7 +1,5 @@
 ---
-- set_fact:
+- include: facts.yml
-    standalone_kubelet: >-
-      {%- if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] -%}true{%- else -%}false{%- endif -%}
   tags: facts
 
 - include: pre_upgrade.yml

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -26,6 +26,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
 --kube-reserved cpu={{ kubelet_cpu_limit }},memory={{ kubelet_memory_limit|regex_replace('Mi', 'M') }} \
 --node-status-update-frequency={{ kubelet_status_update_frequency }} \
+--cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \
 {% endset %}
 
 {# DNS settings for kubelet #}

roles/kubernetes/node/templates/kubelet.standard.env.j2

@@ -19,6 +19,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% if kube_version | version_compare('v1.7', '<') %}
 --enable-cri={{ kubelet_enable_cri }} \
 {% endif %}
+--cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \
 --cgroups-per-qos={{ kubelet_cgroups_per_qos }} \
 --enforce-node-allocatable={{ kubelet_enforce_node_allocatable }} {% endif %}{% endset %}
 

roles/kubernetes/preinstall/tasks/pre-upgrade.yml

@@ -2,3 +2,5 @@
 - name: Stop if non systemd OS type
   assert:
     that: ansible_service_mgr == "systemd"
+  tags:
+    - asserts

roles/kubernetes/secrets/tasks/gen_certs_script.yml

@@ -117,7 +117,7 @@
 
 # FIXME(mattymo): Use tempfile module in ansible 2.3
 - name: Gen_certs | Prepare tempfile for unpacking certs
-  shell: mktemp /tmp/certsXXXXX.tar.gz
+  command: mktemp /tmp/certsXXXXX.tar.gz
   register: cert_tempfile
   when: inventory_hostname in groups['kube-master'] and sync_certs|default(false) and
         inventory_hostname != groups['kube-master'][0]

roles/network_plugin/calico/tasks/main.yml

@@ -6,6 +6,13 @@
     enabled: yes
   failed_when: false
 
+- name: Calico | Get kubelet hostname
+  shell: >-
+    kubectl get node -o custom-columns='NAME:.metadata.name,INTERNAL-IP:.status.addresses[?(@.type=="InternalIP")].address'
+    | egrep "[[:space:]]{{ ansible_all_ipv4_addresses | join('[[:space:]]|[[:space:]]') }}[[:space:]]*$" | cut -d" " -f1
+  register: calico_kubelet_name
+  when: cloud_provider is defined
+
 - name: Calico | Write Calico cni config
   template:
     src: "cni-calico.conf.j2"

roles/network_plugin/calico/templates/cni-calico.conf.j2

@@ -1,7 +1,7 @@
 {
   "name": "calico-k8s-network",
 {% if cloud_provider is defined %}
-  "nodename": "{{ inventory_hostname }}",
+  "nodename": "{{ calico_kubelet_name.stdout }}",
 {% else %}
   "nodename": "{{ ansible_hostname }}",
 {% endif %}