diff --git a/README.md b/README.md index 963291bdf..9fee4ff32 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ Requirements * **Ansible v2.2 (or newer) and python-netaddr is installed on the machine that will run Ansible commands** +* **Jinja 2.8 (or newer) is required to run the Ansible Playbooks** * The target servers must have **access to the Internet** in order to pull docker images. * The target servers are configured to allow **IPv4 forwarding**. * **Your ssh key must be copied** to all the servers part of your inventory. diff --git a/contrib/terraform/openstack/group_vars b/contrib/terraform/openstack/group_vars new file mode 120000 index 000000000..d64da8dc6 --- /dev/null +++ b/contrib/terraform/openstack/group_vars @@ -0,0 +1 @@ +../../../inventory/group_vars \ No newline at end of file diff --git a/contrib/terraform/openstack/kubespray.tf b/contrib/terraform/openstack/kubespray.tf index 42d529d64..0f31b3d16 100644 --- a/contrib/terraform/openstack/kubespray.tf +++ b/contrib/terraform/openstack/kubespray.tf @@ -68,7 +68,7 @@ resource "openstack_compute_instance_v2" "k8s_master" { floating_ip = "${element(openstack_networking_floatingip_v2.k8s_master.*.address, count.index)}" metadata = { ssh_user = "${var.ssh_user}" - kubespray_groups = "etcd,kube-master,kube-node,k8s-cluster" + kubespray_groups = "etcd,kube-master,kube-node,k8s-cluster,vault" } } @@ -87,10 +87,10 @@ resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" { "${openstack_compute_secgroup_v2.k8s.name}" ] metadata = { ssh_user = "${var.ssh_user}" - kubespray_groups = "etcd,kube-master,kube-node,k8s-cluster" + kubespray_groups = "etcd,kube-master,kube-node,k8s-cluster,vault,no-floating" } provisioner "local-exec" { - command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/k8s-cluster.yml" + command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/no-floating.yml" } } @@ -107,7 +107,7 @@ resource "openstack_compute_instance_v2" "k8s_node" { floating_ip = "${element(openstack_networking_floatingip_v2.k8s_node.*.address, count.index)}" metadata = { ssh_user = "${var.ssh_user}" - kubespray_groups = "kube-node,k8s-cluster" + kubespray_groups = "kube-node,k8s-cluster,vault" } } @@ -123,10 +123,10 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" { security_groups = ["${openstack_compute_secgroup_v2.k8s.name}" ] metadata = { ssh_user = "${var.ssh_user}" - kubespray_groups = "kube-node,k8s-cluster" + kubespray_groups = "kube-node,k8s-cluster,vault,no-floating" } provisioner "local-exec" { - command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/k8s-cluster.yml" + command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(openstack_networking_floatingip_v2.k8s_master.*.address, 0)}/ > contrib/terraform/openstack/group_vars/no-floating.yml" } } diff --git a/docs/coreos.md b/docs/coreos.md index e38369aef..7c9b2c8a6 100644 --- a/docs/coreos.md +++ b/docs/coreos.md @@ -13,12 +13,4 @@ Before running the cluster playbook you must satisfy the following requirements: * On each CoreOS nodes a writable directory **/opt/bin** (~400M disk space) -* Uncomment the variable **ansible\_python\_interpreter** in the file `inventory/group_vars/all.yml` - -* run the Python bootstrap playbook - -``` -ansible-playbook -u smana -e ansible_ssh_user=smana -b --become-user=root -i inventory/inventory.cfg coreos-bootstrap.yml -``` - Then you can proceed to [cluster deployment](#run-deployment) diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index 73721d03b..cbd922c63 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -58,9 +58,16 @@ kube_users: role: admin + +## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth) +#kube_oidc_auth: false +#kube_basic_auth: false +#kube_token_auth: false + + ## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/ ## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...) -# kube_oidc_auth: false + # kube_oidc_url: https:// ... # kube_oidc_client_id: kubernetes ## Optional settings for OIDC @@ -69,7 +76,6 @@ kube_users: # kube_oidc_groups_claim: groups - # Choose network plugin (calico, weave or flannel) # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico @@ -115,7 +121,7 @@ docker_daemon_graph: "/var/lib/docker" ## This string should be exactly as you wish it to appear. ## An obvious use case is allowing insecure-registry access ## to self hosted registries like so: -docker_options: "--insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }} --iptables=false" +docker_options: "--insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }}" docker_bin_dir: "/usr/bin" # Settings for containerized control plane (etcd/kubelet/secrets) diff --git a/requirements.txt b/requirements.txt index 6fd09e6c7..bf8e65e4a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ ansible>=2.2.1 netaddr +jinja>=2.8 diff --git a/roles/docker/templates/docker-options.conf.j2 b/roles/docker/templates/docker-options.conf.j2 index 012795898..0113bfc61 100644 --- a/roles/docker/templates/docker-options.conf.j2 +++ b/roles/docker/templates/docker-options.conf.j2 @@ -1,2 +1,2 @@ [Service] -Environment="DOCKER_OPTS={% if docker_options is defined %}{{ docker_options }}{% endif %}" \ No newline at end of file +Environment="DOCKER_OPTS={% if docker_options is defined %}{{ docker_options }}{% endif %} --iptables={% if kube_network_plugin == 'flannel' %}true{% else %}false{% endif %}" diff --git a/roles/etcd/tasks/check_certs.yml b/roles/etcd/tasks/check_certs.yml index dda255b68..16b7d7c28 100644 --- a/roles/etcd/tasks/check_certs.yml +++ b/roles/etcd/tasks/check_certs.yml @@ -62,7 +62,7 @@ {% if gen_node_certs[inventory_hostname] or (not etcdcert_node.results[0].stat.exists|default(False)) or (not etcdcert_node.results[1].stat.exists|default(False)) or - (etcdcert_node.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcdcert_node.results[1].stat.path)|first|map(attribute="checksum")|default('')) -%} + (etcdcert_node.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcdcert_node.results[1].stat.path)|map(attribute="checksum")|first|default('')) -%} {%- set _ = certs.update({'sync': True}) -%} {% endif %} {{ certs.sync }} diff --git a/roles/etcd/tasks/gen_certs_script.yml b/roles/etcd/tasks/gen_certs_script.yml index f54791ed9..f70c6ee21 100644 --- a/roles/etcd/tasks/gen_certs_script.yml +++ b/roles/etcd/tasks/gen_certs_script.yml @@ -4,7 +4,8 @@ path: "{{ etcd_cert_dir }}" group: "{{ etcd_cert_group }}" state: directory - owner: root + owner: kube + mode: 0700 recurse: yes - name: "Gen_certs | create etcd script dir (on {{groups['etcd'][0]}})" @@ -12,6 +13,7 @@ path: "{{ etcd_script_dir }}" state: directory owner: root + mode: 0700 run_once: yes delegate_to: "{{groups['etcd'][0]}}" @@ -20,8 +22,9 @@ path: "{{ etcd_cert_dir }}" group: "{{ etcd_cert_group }}" state: directory - owner: root + owner: kube recurse: yes + mode: 0700 run_once: yes delegate_to: "{{groups['etcd'][0]}}" @@ -42,6 +45,7 @@ delegate_to: "{{groups['etcd'][0]}}" when: gen_certs|default(false) + - name: Gen_certs | run cert generation script command: "bash -x {{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}" environment: @@ -107,20 +111,22 @@ sync_certs|default(false) and inventory_hostname not in groups['etcd'] notify: set etcd_secret_changed -#NOTE(mattymo): Use temporary file to copy master certs because we have a ~200k -#char limit when using shell command +#NOTE(mattymo): Use temporary file to copy master certs because we have a ~200k +#char limit when using shell command + +#FIXME(mattymo): Use tempfile module in ansible 2.3 +- name: Gen_certs | Prepare tempfile for unpacking certs + shell: mktemp /tmp/certsXXXXX.tar.gz + register: cert_tempfile + when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and + inventory_hostname != groups['etcd'][0] -#FIXME(mattymo): Use tempfile module in ansible 2.3 -- name: Gen_certs | Prepare tempfile for unpacking certs - shell: mktemp /tmp/certsXXXXX.tar.gz - register: cert_tempfile - -- name: Gen_certs | Write master certs to tempfile - copy: - content: "{{etcd_master_cert_data.stdout}}" - dest: "{{cert_tempfile.stdout}}" - owner: root - mode: "0600" +- name: Gen_certs | Write master certs to tempfile + copy: + content: "{{etcd_master_cert_data.stdout}}" + dest: "{{cert_tempfile.stdout}}" + owner: root + mode: "0600" when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and inventory_hostname != groups['etcd'][0] @@ -154,13 +160,9 @@ group: "{{ etcd_cert_group }}" state: directory owner: kube + mode: "u=rwX,g-rwx,o-rwx" recurse: yes -- name: Gen_certs | set permissions on keys - shell: chmod 0600 {{ etcd_cert_dir}}/*key.pem - when: inventory_hostname in groups['etcd'] - changed_when: false - - name: Gen_certs | target ca-certificate store file set_fact: ca_cert_path: |- diff --git a/roles/kargo-defaults/defaults/main.yaml b/roles/kargo-defaults/defaults/main.yaml index a2ec34cb7..ecafb1682 100644 --- a/roles/kargo-defaults/defaults/main.yaml +++ b/roles/kargo-defaults/defaults/main.yaml @@ -101,7 +101,7 @@ docker_daemon_graph: "/var/lib/docker" ## This string should be exactly as you wish it to appear. ## An obvious use case is allowing insecure-registry access ## to self hosted registries like so: -docker_options: "--insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }} --iptables=false" +docker_options: "--insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }}" # Settings for containerized control plane (etcd/kubelet/secrets) etcd_deployment_type: docker diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 527b168b9..2fd307801 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -13,6 +13,9 @@ kube_apiserver_node_port_range: "30000-32767" etcd_config_dir: /etc/ssl/etcd etcd_cert_dir: "{{ etcd_config_dir }}/ssl" +# ETCD backend for k8s data +kube_apiserver_storage_backend: etcd3 + # Limits for kube components kube_controller_memory_limit: 512M kube_controller_cpu_limit: 250m @@ -29,11 +32,16 @@ kube_apiserver_memory_limit: 2000M kube_apiserver_cpu_limit: 800m kube_apiserver_memory_requests: 256M kube_apiserver_cpu_requests: 300m -kube_apiserver_storage_backend: etcd2 + + +## Enable/Disable Kube API Server Authentication Methods +kube_basic_auth: true +kube_token_auth: true +kube_oidc_auth: false ## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/ ## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...) -kube_oidc_auth: false + #kube_oidc_url: https:// ... # kube_oidc_client_id: kubernetes ## Optional settings for OIDC diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index fd894124a..2bd9758bf 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -69,3 +69,7 @@ dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest" notify: Master | wait for kube-scheduler tags: kube-scheduler + +- include: post-upgrade.yml + tags: k8s-post-upgrade + diff --git a/roles/kubernetes/master/tasks/post-upgrade.yml b/roles/kubernetes/master/tasks/post-upgrade.yml new file mode 100644 index 000000000..07fc57b96 --- /dev/null +++ b/roles/kubernetes/master/tasks/post-upgrade.yml @@ -0,0 +1,6 @@ +--- +- name: "Post-upgrade | etcd3 upgrade | purge etcd2 k8s data" + command: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} rm -r /registry" + environment: + ETCDCTL_API: 2 + when: kube_apiserver_storage_backend == "etcd3" and needs_etcd_migration|bool|default(false) diff --git a/roles/kubernetes/master/tasks/pre-upgrade.yml b/roles/kubernetes/master/tasks/pre-upgrade.yml index 1bb0c0344..244c8b13e 100644 --- a/roles/kubernetes/master/tasks/pre-upgrade.yml +++ b/roles/kubernetes/master/tasks/pre-upgrade.yml @@ -32,19 +32,64 @@ stat: path: /etc/kubernetes/manifests/kube-apiserver.manifest register: kube_apiserver_manifest - when: secret_changed|default(false) or etcd_secret_changed|default(false) -- name: "Pre-upgrade | Write invalid image to kube-apiserver manifest if secrets were changed" +- name: "Pre-upgrade | etcd3 upgrade | see if old config exists" + command: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} ls /registry/minions" + environment: + ETCDCTL_API: 2 + register: old_data_exists + delegate_to: "{{groups['kube-master'][0]}}" + when: kube_apiserver_storage_backend == "etcd3" + failed_when: false + +- name: "Pre-upgrade | etcd3 upgrade | see if data was already migrated" + command: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} get --limit=1 --prefix=true /registry/minions" + environment: + ETCDCTL_API: 3 + register: data_migrated + delegate_to: "{{groups['etcd'][0]}}" + when: kube_apiserver_storage_backend == "etcd3" + failed_when: false + +- name: "Pre-upgrade | etcd3 upgrade | set needs_etcd_migration" + set_fact: + needs_etcd_migration: "{{ kube_apiserver_storage_backend == 'etcd3' and data_migrated.stdout_lines|length == 0 and old_data_exists.rc == 0 }}" + +- name: "Pre-upgrade | Write invalid image to kube-apiserver manifest if necessary" replace: dest: /etc/kubernetes/manifests/kube-apiserver.manifest regexp: '(\s+)image:\s+.*?$' replace: '\1image: kill.apiserver.using.fake.image.in:manifest' register: kube_apiserver_manifest_replaced - when: (secret_changed|default(false) or etcd_secret_changed|default(false)) and kube_apiserver_manifest.stat.exists + when: (secret_changed|default(false) or etcd_secret_changed|default(false) or needs_etcd_migration|bool) and kube_apiserver_manifest.stat.exists - name: "Pre-upgrade | Pause while waiting for kubelet to delete kube-apiserver pod" pause: seconds: 20 - when: (secret_changed|default(false) or etcd_secret_changed|default(false)) and kube_apiserver_manifest.stat.exists + when: kube_apiserver_manifest_replaced.changed tags: kube-apiserver +- name: "Pre-upgrade | etcd3 upgrade | stop etcd" + service: + name: etcd + state: stopped + delegate_to: "{{item}}" + with_items: "{{groups['etcd']}}" + when: needs_etcd_migration|bool + +- name: "Pre-upgrade | etcd3 upgrade | migrate data" + command: "{{ bin_dir }}/etcdctl migrate --data-dir=\"{{ etcd_data_dir }}\" --wal-dir=\"{{ etcd_data_dir }}/member/wal\"" + environment: + ETCDCTL_API: 3 + delegate_to: "{{item}}" + with_items: "{{groups['etcd']}}" + register: etcd_migrated + when: needs_etcd_migration|bool + +- name: "Pre-upgrade | etcd3 upgrade | start etcd" + service: + name: etcd + state: started + delegate_to: "{{item}}" + with_items: "{{groups['etcd']}}" + when: needs_etcd_migration|bool diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 96a0c738a..65a30929b 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -34,10 +34,14 @@ spec: - --service-cluster-ip-range={{ kube_service_addresses }} - --service-node-port-range={{ kube_apiserver_node_port_range }} - --client-ca-file={{ kube_cert_dir }}/ca.pem +{% if kube_basic_auth|default(true) %} - --basic-auth-file={{ kube_users_dir }}/known_users.csv +{% endif %} - --tls-cert-file={{ kube_cert_dir }}/apiserver.pem - --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem +{% if kube_token_auth|default(true) %} - --token-auth-file={{ kube_token_dir }}/known_tokens.csv +{% endif %} - --service-account-key-file={{ kube_cert_dir }}/apiserver-key.pem {% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %} - --oidc-issuer-url={{ kube_oidc_url }} diff --git a/roles/kubernetes/preinstall/tasks/resolvconf.yml b/roles/kubernetes/preinstall/tasks/resolvconf.yml index 55edd0ca7..6369dfd9c 100644 --- a/roles/kubernetes/preinstall/tasks/resolvconf.yml +++ b/roles/kubernetes/preinstall/tasks/resolvconf.yml @@ -3,25 +3,16 @@ command: cp -f /etc/resolv.conf "{{ resolvconffile }}" when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] -- name: Remove search/domain/nameserver options - lineinfile: - dest: "{{item[0]}}" - state: absent - regexp: "^{{ item[1] }}.*$" - backup: yes - follow: yes - with_nested: - - "{{ [resolvconffile] + [base|default('')] + [head|default('')] }}" - - [ 'search ', 'nameserver ', 'domain ', 'options ' ] - notify: Preinstall | restart network - -- name: Add domain/search/nameservers to resolv.conf +- name: Add domain/search/nameservers/options to resolv.conf blockinfile: dest: "{{resolvconffile}}" block: |- {% for item in [domainentry] + [searchentries] + nameserverentries.split(',') -%} {{ item }} {% endfor %} + options ndots:{{ ndots }} + options timeout:2 + options attempts:2 state: present insertbefore: BOF create: yes @@ -30,21 +21,32 @@ marker: "# Ansible entries {mark}" notify: Preinstall | restart network -- name: Add options to resolv.conf - lineinfile: - line: options {{ item }} - dest: "{{resolvconffile}}" - state: present - regexp: "^options.*{{ item }}$" - insertafter: EOF +- name: Remove search/domain/nameserver options before block + replace: + dest: "{{item[0]}}" + regexp: '^{{ item[1] }}[^#]*(?=# Ansible entries BEGIN)' backup: yes follow: yes - with_items: - - ndots:{{ ndots }} - - timeout:2 - - attempts:2 + with_nested: + - "{{ [resolvconffile] + [base|default('')] + [head|default('')] }}" + - [ 'search ', 'nameserver ', 'domain ', 'options ' ] + when: item[0] != "" notify: Preinstall | restart network +- name: Remove search/domain/nameserver options after block + replace: + dest: "{{item[0]}}" + regexp: '(# Ansible entries END\n(?:(?!^{{ item[1] }}).*\n)*)(?:^{{ item[1] }}.*\n?)+' + replace: '\1' + backup: yes + follow: yes + with_nested: + - "{{ [resolvconffile] + [base|default('')] + [head|default('')] }}" + - [ 'search ', 'nameserver ', 'domain ', 'options ' ] + when: item[0] != "" + notify: Preinstall | restart network + + - name: get temporary resolveconf cloud init file content command: cat {{ resolvconffile }} register: cloud_config diff --git a/roles/kubernetes/secrets/tasks/check-tokens.yml b/roles/kubernetes/secrets/tasks/check-tokens.yml index eff617408..94ee92fb5 100644 --- a/roles/kubernetes/secrets/tasks/check-tokens.yml +++ b/roles/kubernetes/secrets/tasks/check-tokens.yml @@ -14,7 +14,7 @@ - name: "Check_tokens | Set 'sync_tokens' and 'gen_tokens' to true" set_fact: gen_tokens: true - when: not known_tokens_master.stat.exists + when: not known_tokens_master.stat.exists and kube_token_auth|default(true) run_once: true - name: "Check tokens | check if a cert already exists" diff --git a/roles/kubernetes/secrets/tasks/main.yml b/roles/kubernetes/secrets/tasks/main.yml index 6da147170..919ed0df7 100644 --- a/roles/kubernetes/secrets/tasks/main.yml +++ b/roles/kubernetes/secrets/tasks/main.yml @@ -33,7 +33,7 @@ line: '{{ item.value.pass }},{{ item.key }},{{ item.value.role }}' backup: yes with_dict: "{{ kube_users }}" - when: inventory_hostname in "{{ groups['kube-master'] }}" + when: inventory_hostname in "{{ groups['kube-master'] }}" and kube_basic_auth|default(true) notify: set secret_changed #