Merge branch 'master' into idempotency2

README.md

@@ -69,6 +69,7 @@ Requirements
 
 * **Ansible v2.2 (or newer) and python-netaddr is installed on the machine
   that will run Ansible commands**
+* **Jinja 2.8 (or newer) is required to run the Ansible Playbooks**
 * The target servers must have **access to the Internet** in order to pull docker images.
 * The target servers are configured to allow **IPv4 forwarding**.
 * **Your ssh key must be copied** to all the servers part of your inventory.

contrib/terraform/openstack/group_vars

@@ -0,0 +1 @@
+../../../inventory/group_vars

contrib/terraform/openstack/kubespray.tf

@@ -68,7 +68,7 @@ resource "openstack_compute_instance_v2" "k8s_master" {
     floating_ip = "${element(openstack_networking_floatingip_v2.k8s_master.*.address, count.index)}"
     metadata = {
         ssh_user = "${var.ssh_user}"
-        kubespray_groups = "etcd,kube-master,kube-node,k8s-cluster"
+        kubespray_groups = "etcd,kube-master,kube-node,k8s-cluster,vault"
     }
     
 }
@@ -87,10 +87,10 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" {
                         "${openstack_compute_secgroup_v2.k8s.name}" ]
     metadata = {
         ssh_user = "${var.ssh_user}"
-        kubespray_groups = "etcd,kube-master,kube-node,k8s-cluster"
+        kubespray_groups = "etcd,kube-master,kube-node,k8s-cluster,vault,no-floating"
     }
     provisioner "local-exec" {
-        command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/k8s-cluster.yml"
+        command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/no-floating.yml"
     }
 }
 
@@ -107,7 +107,7 @@ resource "openstack_compute_instance_v2" "k8s_node" {
     floating_ip = "${element(openstack_networking_floatingip_v2.k8s_node.*.address, count.index)}"
     metadata = {
         ssh_user = "${var.ssh_user}"
-        kubespray_groups = "kube-node,k8s-cluster"
+        kubespray_groups = "kube-node,k8s-cluster,vault"
     }
 }
 
@@ -123,10 +123,10 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" {
     security_groups = ["${openstack_compute_secgroup_v2.k8s.name}" ]
     metadata = {
         ssh_user = "${var.ssh_user}"
-        kubespray_groups = "kube-node,k8s-cluster"
+        kubespray_groups = "kube-node,k8s-cluster,vault,no-floating"
     }
     provisioner "local-exec" {
-	command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/k8s-cluster.yml"        
+	command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/no-floating.yml"        
     }
 }
 

docs/coreos.md

@@ -13,12 +13,4 @@ Before running the cluster playbook you must satisfy the following requirements:
 
 * On each CoreOS nodes a writable directory **/opt/bin** (~400M disk space)
 
-* Uncomment the variable **ansible\_python\_interpreter** in the file `inventory/group_vars/all.yml`
-
-* run the Python bootstrap playbook
-
-```
-ansible-playbook -u smana -e ansible_ssh_user=smana  -b --become-user=root -i inventory/inventory.cfg coreos-bootstrap.yml
-```
-
 Then you can proceed to [cluster deployment](#run-deployment)

inventory/group_vars/k8s-cluster.yml

@@ -58,9 +58,16 @@ kube_users:
     role: admin
 
 
+
+## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth)
+#kube_oidc_auth: false
+#kube_basic_auth: false
+#kube_token_auth: false
+
+
 ## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/
 ## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...)
-# kube_oidc_auth: false
+
 # kube_oidc_url: https:// ...
 # kube_oidc_client_id: kubernetes
 ## Optional settings for OIDC
@@ -69,7 +76,6 @@ kube_users:
 # kube_oidc_groups_claim: groups
 
 
-
 # Choose network plugin (calico, weave or flannel)
 # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
 kube_network_plugin: calico
@@ -115,7 +121,7 @@ docker_daemon_graph: "/var/lib/docker"
 ## This string should be exactly as you wish it to appear.
 ## An obvious use case is allowing insecure-registry access
 ## to self hosted registries like so:
-docker_options: "--insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }} --iptables=false"
+docker_options: "--insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }}"
 docker_bin_dir: "/usr/bin"
 
 # Settings for containerized control plane (etcd/kubelet/secrets)

requirements.txt

@@ -1,2 +1,3 @@
 ansible>=2.2.1
 netaddr
+jinja>=2.8

roles/docker/templates/docker-options.conf.j2

@@ -1,2 +1,2 @@
 [Service]
-Environment="DOCKER_OPTS={% if docker_options is defined %}{{ docker_options }}{% endif %}"
+Environment="DOCKER_OPTS={% if docker_options is defined %}{{ docker_options }}{% endif %} --iptables={% if kube_network_plugin == 'flannel' %}true{% else %}false{% endif %}"

roles/etcd/tasks/check_certs.yml

@@ -62,7 +62,7 @@
       {% if gen_node_certs[inventory_hostname] or
         (not etcdcert_node.results[0].stat.exists|default(False)) or
           (not etcdcert_node.results[1].stat.exists|default(False)) or
-            (etcdcert_node.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcdcert_node.results[1].stat.path)|first|map(attribute="checksum")|default('')) -%}
+            (etcdcert_node.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcdcert_node.results[1].stat.path)|map(attribute="checksum")|first|default('')) -%}
               {%- set _ = certs.update({'sync': True}) -%}
       {% endif %}
       {{ certs.sync }}

roles/etcd/tasks/gen_certs_script.yml

@@ -4,7 +4,8 @@
     path: "{{ etcd_cert_dir }}"
     group: "{{ etcd_cert_group }}"
     state: directory
-    owner: root
+    owner: kube
+    mode: 0700
     recurse: yes
 
 - name: "Gen_certs | create etcd script dir (on {{groups['etcd'][0]}})"
@@ -12,6 +13,7 @@
     path: "{{ etcd_script_dir }}"
     state: directory
     owner: root
+    mode: 0700
   run_once: yes
   delegate_to: "{{groups['etcd'][0]}}"
 
@@ -20,8 +22,9 @@
     path: "{{ etcd_cert_dir }}"
     group: "{{ etcd_cert_group }}"
     state: directory
-    owner: root
+    owner: kube
     recurse: yes
+    mode: 0700
   run_once: yes
   delegate_to: "{{groups['etcd'][0]}}"
 
@@ -42,6 +45,7 @@
   delegate_to: "{{groups['etcd'][0]}}"
   when: gen_certs|default(false)
 
+
 - name: Gen_certs | run cert generation script
   command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
   environment:
@@ -107,20 +111,22 @@
         sync_certs|default(false) and inventory_hostname not in groups['etcd']
   notify: set etcd_secret_changed
 
-#NOTE(mattymo): Use temporary file to copy master certs because we have a ~200k
+#NOTE(mattymo): Use temporary file to copy master certs because we have a ~200k 
-#char limit when using shell command
+#char limit when using shell command                                            
+                                                                                
+#FIXME(mattymo): Use tempfile module in ansible 2.3                             
+- name: Gen_certs | Prepare tempfile for unpacking certs                        
+  shell: mktemp /tmp/certsXXXXX.tar.gz                                          
+  register: cert_tempfile                                                       
+  when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
+        inventory_hostname != groups['etcd'][0]                          
 
-#FIXME(mattymo): Use tempfile module in ansible 2.3
+- name: Gen_certs | Write master certs to tempfile                              
-- name: Gen_certs | Prepare tempfile for unpacking certs
+  copy:                                                                         
-  shell: mktemp /tmp/certsXXXXX.tar.gz
+    content: "{{etcd_master_cert_data.stdout}}"                                      
-  register: cert_tempfile
+    dest: "{{cert_tempfile.stdout}}"                                            
-
+    owner: root                                                                 
-- name: Gen_certs | Write master certs to tempfile
+    mode: "0600"                                                                
-  copy:
-    content: "{{etcd_master_cert_data.stdout}}"
-    dest: "{{cert_tempfile.stdout}}"
-    owner: root
-    mode: "0600"
   when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
         inventory_hostname != groups['etcd'][0]
 
@@ -154,13 +160,9 @@
     group: "{{ etcd_cert_group }}"
     state: directory
     owner: kube
+    mode: "u=rwX,g-rwx,o-rwx"
     recurse: yes
 
-- name: Gen_certs | set permissions on keys
-  shell: chmod 0600 {{ etcd_cert_dir}}/*key.pem
-  when: inventory_hostname in groups['etcd']
-  changed_when: false
-
 - name: Gen_certs | target ca-certificate store file
   set_fact:
     ca_cert_path: |-

roles/kargo-defaults/defaults/main.yaml

@@ -101,7 +101,7 @@ docker_daemon_graph: "/var/lib/docker"
 ## This string should be exactly as you wish it to appear.
 ## An obvious use case is allowing insecure-registry access
 ## to self hosted registries like so:
-docker_options: "--insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }} --iptables=false"
+docker_options: "--insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }}"
 
 # Settings for containerized control plane (etcd/kubelet/secrets)
 etcd_deployment_type: docker

roles/kubernetes/master/defaults/main.yml

@@ -13,6 +13,9 @@ kube_apiserver_node_port_range: "30000-32767"
 etcd_config_dir: /etc/ssl/etcd
 etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
 
+# ETCD backend for k8s data
+kube_apiserver_storage_backend: etcd3
+
 # Limits for kube components
 kube_controller_memory_limit: 512M
 kube_controller_cpu_limit: 250m
@@ -29,11 +32,16 @@ kube_apiserver_memory_limit: 2000M
 kube_apiserver_cpu_limit: 800m
 kube_apiserver_memory_requests: 256M
 kube_apiserver_cpu_requests: 300m
-kube_apiserver_storage_backend: etcd2
+
+
+## Enable/Disable Kube API Server Authentication Methods
+kube_basic_auth: true
+kube_token_auth: true
+kube_oidc_auth: false
 
 ## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/
 ## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...)
-kube_oidc_auth: false
+
 #kube_oidc_url: https:// ...
 # kube_oidc_client_id: kubernetes
 ## Optional settings for OIDC

roles/kubernetes/master/tasks/main.yml

@@ -69,3 +69,7 @@
     dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
   notify: Master | wait for kube-scheduler
   tags: kube-scheduler
+
+- include: post-upgrade.yml
+  tags: k8s-post-upgrade
+

roles/kubernetes/master/tasks/post-upgrade.yml

@@ -0,0 +1,6 @@
+---
+- name: "Post-upgrade | etcd3 upgrade | purge etcd2 k8s data"
+  command: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} rm -r /registry"
+  environment:
+    ETCDCTL_API: 2
+  when: kube_apiserver_storage_backend == "etcd3" and  needs_etcd_migration|bool|default(false)

roles/kubernetes/master/tasks/pre-upgrade.yml

@@ -32,19 +32,64 @@
   stat:
     path: /etc/kubernetes/manifests/kube-apiserver.manifest
   register: kube_apiserver_manifest
-  when: secret_changed|default(false) or etcd_secret_changed|default(false)
 
-- name: "Pre-upgrade | Write invalid image to kube-apiserver manifest if secrets were changed"
+- name: "Pre-upgrade | etcd3 upgrade | see if old config exists"
+  command: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} ls /registry/minions"
+  environment:
+    ETCDCTL_API: 2
+  register: old_data_exists
+  delegate_to: "{{groups['kube-master'][0]}}"
+  when: kube_apiserver_storage_backend == "etcd3"
+  failed_when: false
+
+- name: "Pre-upgrade | etcd3 upgrade | see if data was already migrated"
+  command: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} get --limit=1 --prefix=true /registry/minions"
+  environment:
+    ETCDCTL_API: 3
+  register: data_migrated
+  delegate_to: "{{groups['etcd'][0]}}"
+  when: kube_apiserver_storage_backend == "etcd3"
+  failed_when: false
+
+- name: "Pre-upgrade | etcd3 upgrade | set needs_etcd_migration"
+  set_fact:
+    needs_etcd_migration: "{{ kube_apiserver_storage_backend == 'etcd3' and data_migrated.stdout_lines|length == 0 and old_data_exists.rc == 0 }}"
+
+- name: "Pre-upgrade | Write invalid image to kube-apiserver manifest if necessary"
   replace:
     dest: /etc/kubernetes/manifests/kube-apiserver.manifest
     regexp: '(\s+)image:\s+.*?$'
     replace: '\1image: kill.apiserver.using.fake.image.in:manifest'
   register: kube_apiserver_manifest_replaced
-  when: (secret_changed|default(false) or etcd_secret_changed|default(false)) and kube_apiserver_manifest.stat.exists
+  when: (secret_changed|default(false) or etcd_secret_changed|default(false) or needs_etcd_migration|bool) and kube_apiserver_manifest.stat.exists
 
 - name: "Pre-upgrade | Pause while waiting for kubelet to delete kube-apiserver pod"
   pause:
     seconds: 20
-  when: (secret_changed|default(false) or etcd_secret_changed|default(false)) and kube_apiserver_manifest.stat.exists
+  when: kube_apiserver_manifest_replaced.changed
   tags: kube-apiserver
 
+- name: "Pre-upgrade | etcd3 upgrade | stop etcd"
+  service:
+    name: etcd
+    state: stopped
+  delegate_to: "{{item}}"
+  with_items: "{{groups['etcd']}}"
+  when: needs_etcd_migration|bool
+
+- name: "Pre-upgrade | etcd3 upgrade | migrate data"
+  command: "{{ bin_dir }}/etcdctl migrate --data-dir=\"{{ etcd_data_dir }}\" --wal-dir=\"{{ etcd_data_dir }}/member/wal\""
+  environment:
+    ETCDCTL_API: 3
+  delegate_to: "{{item}}"
+  with_items: "{{groups['etcd']}}"
+  register: etcd_migrated
+  when: needs_etcd_migration|bool
+
+- name: "Pre-upgrade | etcd3 upgrade | start etcd"
+  service:
+    name: etcd
+    state: started
+  delegate_to: "{{item}}"
+  with_items: "{{groups['etcd']}}"
+  when: needs_etcd_migration|bool

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -34,10 +34,14 @@ spec:
     - --service-cluster-ip-range={{ kube_service_addresses }}
     - --service-node-port-range={{ kube_apiserver_node_port_range }}
     - --client-ca-file={{ kube_cert_dir }}/ca.pem
+{% if kube_basic_auth|default(true) %}
     - --basic-auth-file={{ kube_users_dir }}/known_users.csv
+{% endif %}
     - --tls-cert-file={{ kube_cert_dir }}/apiserver.pem
     - --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
+{% if kube_token_auth|default(true) %}
     - --token-auth-file={{ kube_token_dir }}/known_tokens.csv
+{% endif %}
     - --service-account-key-file={{ kube_cert_dir }}/apiserver-key.pem
 {% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
     - --oidc-issuer-url={{ kube_oidc_url }}

roles/kubernetes/preinstall/tasks/resolvconf.yml

@@ -3,25 +3,16 @@
   command: cp -f /etc/resolv.conf "{{ resolvconffile }}"
   when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"]
 
-- name: Remove search/domain/nameserver options
+- name: Add domain/search/nameservers/options to resolv.conf
-  lineinfile:
-    dest: "{{item[0]}}"
-    state: absent
-    regexp: "^{{ item[1] }}.*$"
-    backup: yes
-    follow: yes
-  with_nested:
-    - "{{ [resolvconffile] + [base|default('')] + [head|default('')] }}"
-    - [ 'search ', 'nameserver ', 'domain ', 'options ' ]
-  notify: Preinstall | restart network
-
-- name: Add domain/search/nameservers to resolv.conf
   blockinfile:
     dest: "{{resolvconffile}}"
     block: |-
       {% for item in [domainentry] + [searchentries] + nameserverentries.split(',') -%}
       {{ item }}
       {% endfor %}
+      options ndots:{{ ndots }}
+      options timeout:2
+      options attempts:2
     state: present
     insertbefore: BOF
     create: yes
@@ -30,21 +21,32 @@
     marker: "# Ansible entries {mark}"
   notify: Preinstall | restart network
 
-- name: Add options to resolv.conf
+- name: Remove search/domain/nameserver options before block
-  lineinfile:
+  replace:
-    line: options {{ item }}
+    dest: "{{item[0]}}"
-    dest: "{{resolvconffile}}"
+    regexp: '^{{ item[1] }}[^#]*(?=# Ansible entries BEGIN)'
-    state: present
-    regexp: "^options.*{{ item }}$"
-    insertafter: EOF
     backup: yes
     follow: yes
-  with_items:
+  with_nested:
-    - ndots:{{ ndots }}
+    - "{{ [resolvconffile] + [base|default('')] + [head|default('')] }}"
-    - timeout:2
+    - [ 'search ', 'nameserver ', 'domain ', 'options ' ]
-    - attempts:2
+  when: item[0] != ""
   notify: Preinstall | restart network
 
+- name: Remove search/domain/nameserver options after block
+  replace:
+    dest: "{{item[0]}}"
+    regexp: '(# Ansible entries END\n(?:(?!^{{ item[1] }}).*\n)*)(?:^{{ item[1] }}.*\n?)+'
+    replace: '\1'
+    backup: yes
+    follow: yes
+  with_nested:
+    - "{{ [resolvconffile] + [base|default('')] + [head|default('')] }}"
+    - [ 'search ', 'nameserver ', 'domain ', 'options ' ]
+  when: item[0] != ""
+  notify: Preinstall | restart network
+
+
 - name: get temporary resolveconf cloud init file content
   command: cat {{ resolvconffile }}
   register: cloud_config

roles/kubernetes/secrets/tasks/check-tokens.yml

@@ -14,7 +14,7 @@
 - name: "Check_tokens | Set 'sync_tokens' and 'gen_tokens' to true"
   set_fact:
     gen_tokens: true
-  when: not known_tokens_master.stat.exists
+  when: not known_tokens_master.stat.exists and kube_token_auth|default(true)
   run_once: true
 
 - name: "Check tokens | check if a cert already exists"

roles/kubernetes/secrets/tasks/main.yml

@@ -33,7 +33,7 @@
     line: '{{ item.value.pass }},{{ item.key }},{{ item.value.role }}'
     backup: yes
   with_dict: "{{ kube_users }}"
-  when: inventory_hostname in "{{ groups['kube-master'] }}"
+  when: inventory_hostname in "{{ groups['kube-master'] }}" and kube_basic_auth|default(true)
   notify: set secret_changed
 
 #