From fad22bae9707335385665c346e9b29d75a3893fb Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Wed, 15 Mar 2017 14:00:42 +0300 Subject: [PATCH] More idempotency fixes Fixed sync_tokens fact Fixed sync_certs for k8s tokens fact Disabled register docker images changability Fixed CNI dir permission Fix idempotency for etcd pre upgrade checks --- .../download/tasks/set_docker_image_facts.yml | 1 + roles/etcd/tasks/check_certs.yml | 3 +- roles/etcd/tasks/gen_certs_script.yml | 56 +++++++++---------- roles/etcd/tasks/main.yml | 2 +- roles/etcd/tasks/pre_upgrade.yml | 2 + roles/kubernetes/master/tasks/main.yml | 1 - .../kubernetes/secrets/tasks/check-certs.yml | 3 +- .../kubernetes/secrets/tasks/check-tokens.yml | 2 +- .../secrets/tasks/gen_certs_script.yml | 8 +-- roles/kubernetes/secrets/tasks/gen_tokens.yml | 3 +- roles/network_plugin/calico/tasks/main.yml | 11 +++- roles/network_plugin/canal/tasks/main.yml | 10 +++- roles/network_plugin/cloud/tasks/main.yml | 10 +++- 13 files changed, 69 insertions(+), 43 deletions(-) diff --git a/roles/download/tasks/set_docker_image_facts.yml b/roles/download/tasks/set_docker_image_facts.yml index 0efda4d09..4ae81d954 100644 --- a/roles/download/tasks/set_docker_image_facts.yml +++ b/roles/download/tasks/set_docker_image_facts.yml @@ -13,6 +13,7 @@ no_log: true register: docker_images_raw failed_when: false + changed_when: false check_mode: no when: not download_always_pull|bool diff --git a/roles/etcd/tasks/check_certs.yml b/roles/etcd/tasks/check_certs.yml index 9bb32f162..dda255b68 100644 --- a/roles/etcd/tasks/check_certs.yml +++ b/roles/etcd/tasks/check_certs.yml @@ -3,6 +3,7 @@ find: paths: "{{ etcd_cert_dir }}" patterns: "ca.pem,node*.pem" + get_checksum: true delegate_to: "{{groups['etcd'][0]}}" register: etcdcert_master run_once: true @@ -58,7 +59,7 @@ sync_certs: true when: >- {%- set certs = {'sync': False} -%} - {% if gen_node_certs[inventory_hostname] or + {% if gen_node_certs[inventory_hostname] or (not etcdcert_node.results[0].stat.exists|default(False)) or (not etcdcert_node.results[1].stat.exists|default(False)) or (etcdcert_node.results[1].stat.checksum|default('') != etcdcert_master.files|selectattr("path", "equalto", etcdcert_node.results[1].stat.path)|first|map(attribute="checksum")|default('')) -%} diff --git a/roles/etcd/tasks/gen_certs_script.yml b/roles/etcd/tasks/gen_certs_script.yml index 06d86257c..f54791ed9 100644 --- a/roles/etcd/tasks/gen_certs_script.yml +++ b/roles/etcd/tasks/gen_certs_script.yml @@ -107,38 +107,38 @@ sync_certs|default(false) and inventory_hostname not in groups['etcd'] notify: set etcd_secret_changed -#NOTE(mattymo): Use temporary file to copy master certs because we have a ~200k -#char limit when using shell command - -#FIXME(mattymo): Use tempfile module in ansible 2.3 -- name: Gen_certs | Prepare tempfile for unpacking certs - shell: mktemp /tmp/certsXXXXX.tar.gz - register: cert_tempfile - -- name: Gen_certs | Write master certs to tempfile - copy: - content: "{{etcd_master_cert_data.stdout}}" - dest: "{{cert_tempfile.stdout}}" - owner: root - mode: "0600" +#NOTE(mattymo): Use temporary file to copy master certs because we have a ~200k +#char limit when using shell command + +#FIXME(mattymo): Use tempfile module in ansible 2.3 +- name: Gen_certs | Prepare tempfile for unpacking certs + shell: mktemp /tmp/certsXXXXX.tar.gz + register: cert_tempfile + +- name: Gen_certs | Write master certs to tempfile + copy: + content: "{{etcd_master_cert_data.stdout}}" + dest: "{{cert_tempfile.stdout}}" + owner: root + mode: "0600" when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and - inventory_hostname != groups['etcd'][0] - -- name: Gen_certs | Unpack certs on masters + inventory_hostname != groups['etcd'][0] + +- name: Gen_certs | Unpack certs on masters shell: "base64 -d < {{ cert_tempfile.stdout }} | tar xz -C {{ etcd_cert_dir }}" - no_log: true - changed_when: false - check_mode: no + no_log: true + changed_when: false + check_mode: no when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and - inventory_hostname != groups['etcd'][0] - notify: set secret_changed - -- name: Gen_certs | Cleanup tempfile - file: - path: "{{cert_tempfile.stdout}}" - state: absent + inventory_hostname != groups['etcd'][0] + notify: set secret_changed + +- name: Gen_certs | Cleanup tempfile + file: + path: "{{cert_tempfile.stdout}}" + state: absent when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and - inventory_hostname != groups['etcd'][0] + inventory_hostname != groups['etcd'][0] - name: Gen_certs | Copy certs on nodes shell: "base64 -d <<< '{{etcd_node_cert_data.stdout|quote}}' | tar xz -C {{ etcd_cert_dir }}" diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index 02737ea31..d917b56ac 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -16,7 +16,7 @@ tags: etcd-secrets - include: sync_etcd_node_certs.yml - when: cert_management == "vault" and inventory_hostname in etcd_node_cert_hosts + when: cert_management == "vault" and inventory_hostname in etcd_node_cert_hosts tags: etcd-secrets - include: gen_certs_vault.yml diff --git a/roles/etcd/tasks/pre_upgrade.yml b/roles/etcd/tasks/pre_upgrade.yml index d498a0336..0f171094a 100644 --- a/roles/etcd/tasks/pre_upgrade.yml +++ b/roles/etcd/tasks/pre_upgrade.yml @@ -28,6 +28,7 @@ - name: "Pre-upgrade | find etcd-proxy container" command: "{{ docker_bin_dir }}/docker ps -aq --filter 'name=etcd-proxy*'" register: etcd_proxy_container + changed_when: false failed_when: false - name: "Pre-upgrade | remove etcd-proxy if it exists" @@ -47,6 +48,7 @@ until: etcd_member_list.rc != 2 run_once: true when: etcdctl_installed.stat.exists + changed_when: false failed_when: false - name: "Pre-upgrade | change peer names to SSL" diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index 67a64d4a6..fd894124a 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -13,7 +13,6 @@ - name: Install kubectl bash completion shell: "{{ bin_dir }}/kubectl completion bash >/etc/bash_completion.d/kubectl.sh" - #no_log: true when: ansible_os_family in ["Debian","RedHat"] tags: kubectl diff --git a/roles/kubernetes/secrets/tasks/check-certs.yml b/roles/kubernetes/secrets/tasks/check-certs.yml index 41cef85c1..9ac877e9a 100644 --- a/roles/kubernetes/secrets/tasks/check-certs.yml +++ b/roles/kubernetes/secrets/tasks/check-certs.yml @@ -3,6 +3,7 @@ find: paths: "{{ kube_cert_dir }}" patterns: "*.pem" + get_checksum: true delegate_to: "{{groups['kube-master'][0]}}" register: kubecert_master run_once: true @@ -58,7 +59,7 @@ {% if gen_node_certs[inventory_hostname] or (not kubecert_node.results[0].stat.exists|default(False)) or (not kubecert_node.results[1].stat.exists|default(False)) or - (kubecert_node.results[1].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[1].stat.path)|first|map(attribute="checksum")|default('')) -%} + (kubecert_node.results[1].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[1].stat.path)|map(attribute="checksum")|first|default('')) -%} {%- set _ = certs.update({'sync': True}) -%} {% endif %} {{ certs.sync }} diff --git a/roles/kubernetes/secrets/tasks/check-tokens.yml b/roles/kubernetes/secrets/tasks/check-tokens.yml index 14cfbb124..eff617408 100644 --- a/roles/kubernetes/secrets/tasks/check-tokens.yml +++ b/roles/kubernetes/secrets/tasks/check-tokens.yml @@ -19,7 +19,7 @@ - name: "Check tokens | check if a cert already exists" stat: - path: "{{ kube_cert_dir }}/ca.pem" + path: "{{ kube_token_dir }}/known_tokens.csv" register: known_tokens - name: "Check_tokens | Set 'sync_tokens' to true" diff --git a/roles/kubernetes/secrets/tasks/gen_certs_script.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml index 4a9188065..8df2195bf 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs_script.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml @@ -106,6 +106,8 @@ - name: Gen_certs | Prepare tempfile for unpacking certs shell: mktemp /tmp/certsXXXXX.tar.gz register: cert_tempfile + when: inventory_hostname in groups['kube-master'] and sync_certs|default(false) and + inventory_hostname != groups['kube-master'][0] - name: Gen_certs | Write master certs to tempfile copy: @@ -149,13 +151,9 @@ path: "{{ kube_cert_dir }}" group: "{{ kube_cert_group }}" owner: kube + mode: "u=rwX,g-rwx,o-rwx" recurse: yes -- name: Gen_certs | set permissions on keys - shell: chmod 0600 {{ kube_cert_dir}}/*key.pem - when: inventory_hostname in groups['kube-master'] - changed_when: false - - name: Gen_certs | target ca-certificates path set_fact: ca_cert_path: |- diff --git a/roles/kubernetes/secrets/tasks/gen_tokens.yml b/roles/kubernetes/secrets/tasks/gen_tokens.yml index 35a8196ac..a4cc0f69b 100644 --- a/roles/kubernetes/secrets/tasks/gen_tokens.yml +++ b/roles/kubernetes/secrets/tasks/gen_tokens.yml @@ -39,9 +39,9 @@ - name: Gen_tokens | Get list of tokens from first master shell: "(find {{ kube_token_dir }} -maxdepth 1 -type f)" register: tokens_list - changed_when: false check_mode: no delegate_to: "{{groups['kube-master'][0]}}" + run_once: true when: sync_tokens|default(false) - name: Gen_tokens | Gather tokens @@ -54,6 +54,5 @@ - name: Gen_tokens | Copy tokens on masters shell: "echo '{{ tokens_data.stdout|quote }}' | base64 -d | tar xz -C /" - changed_when: false when: inventory_hostname in groups['kube-master'] and sync_tokens|default(false) and inventory_hostname != groups['kube-master'][0] diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index 549ece3b3..2f3096bf3 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -41,7 +41,7 @@ notify: restart calico-node - name: Calico | Copy cni plugins from hyperkube - command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/" + command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -ac /opt/cni/bin/ /cnibindir/" register: cni_task_result until: cni_task_result.rc == 0 retries: 4 @@ -59,6 +59,14 @@ when: "{{ overwrite_hyperkube_cni|bool }}" tags: [hyperkube, upgrade] +- name: Calico | Set cni directory permissions + file: + path: /opt/cni/bin + state: directory + owner: kube + recurse: true + mode: 0755 + - name: Calico | wait for etcd uri: url: https://localhost:2379/health @@ -80,6 +88,7 @@ register: calico_conf delegate_to: "{{groups['etcd'][0]}}" run_once: true + changed_when: false - name: Calico | Configure calico network pool shell: > diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml index 9402d390c..ea67e20cd 100644 --- a/roles/network_plugin/canal/tasks/main.yml +++ b/roles/network_plugin/canal/tasks/main.yml @@ -44,7 +44,7 @@ register: canal_node_manifest - name: Canal | Copy cni plugins from hyperkube - command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/" + command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -ac /opt/cni/bin/ /cnibindir/" register: cni_task_result until: cni_task_result.rc == 0 retries: 4 @@ -61,6 +61,14 @@ changed_when: false tags: [hyperkube, upgrade] +- name: Canal | Set cni directory permissions + file: + path: /opt/cni/bin + state: directory + owner: kube + recurse: true + mode: 0755 + - name: Canal | Install calicoctl container script template: src: calicoctl-container.j2 diff --git a/roles/network_plugin/cloud/tasks/main.yml b/roles/network_plugin/cloud/tasks/main.yml index 346a57969..36fa8e57d 100644 --- a/roles/network_plugin/cloud/tasks/main.yml +++ b/roles/network_plugin/cloud/tasks/main.yml @@ -1,5 +1,4 @@ --- - - name: Cloud | Copy cni plugins from hyperkube command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/" register: cni_task_result @@ -7,3 +6,12 @@ retries: 4 delay: "{{ retry_stagger | random + 3 }}" changed_when: false + +- name: Cloud | Set cni directory permissions + file: + path: /opt/cni/bin + state: directory + owner: kube + recurse: true + mode: "u=rwX,g-rwx,o-rwx" +