Fix cert and netchecker upgrade issues (#1543)

* Bump tag for upgrade CI, fix netchecker upgrade

netchecker-server was changed from pod to deployment, so
we need an upgrade hook for it.

CI now uses v2.1.1 as a basis for upgrade.

* Fix upgrades for certs from non-rbac to rbac

.gitlab-ci.yml

@@ -110,7 +110,7 @@ before_script:
     # Check out latest tag if testing upgrade
     # Uncomment when gitlab kargo repo has tags
     #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
-    - test "${UPGRADE_TEST}" != "false" && git checkout 031cf565ec3ccd3ebbe80eeef3454c3780e5c598 && pip install ansible==2.2.0
+    - test "${UPGRADE_TEST}" != "false" && git checkout acae0fe4a36bd1d3cd267e72ad01126a72d1458a
 
 
     # Create cluster
@@ -142,7 +142,6 @@ before_script:
       if [ "${UPGRADE_TEST}" != "false" ]; then
       test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml";
       test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml";
-      pip install ansible==2.3.0;
       git checkout "${CI_BUILD_REF}";
       ansible-playbook -i inventory/inventory.ini -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER
       ${SSH_ARGS}
@@ -281,7 +280,6 @@ before_script:
   CLOUD_IMAGE: ubuntu-1604-xenial
   CLOUD_REGION: europe-west1-b
   CLOUD_MACHINE_TYPE: "n1-standard-2"
-  UPGRADE_TEST: "basic"
   CLUSTER_MODE: ha
   UPGRADE_TEST: "graceful"
   STARTUP_SCRIPT: ""

roles/kubernetes-apps/ansible/tasks/netchecker.yml

@@ -15,6 +15,15 @@
     - inventory_hostname == groups['kube-master'][0]
     - rbac_enabled or item.type not in rbac_resources
 
+- name: Kubernetes Apps | Purge old Netchecker server
+  kube:
+    name: "netchecker-server"
+    namespace: "{{ netcheck_namespace }}"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "po"
+    state: absent
+  when: inventory_hostname == groups['kube-master'][0]
+
 #FIXME: remove if kubernetes/features#124 is implemented
 - name: Kubernetes Apps | Purge old Netchecker daemonsets
   kube:

roles/kubernetes/secrets/files/make-ssl.sh

@@ -80,7 +80,9 @@ gen_key_and_cert() {
     openssl x509 -req -in ${name}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${name}.pem -days 3650 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
 }
 
-if [ ! -e "$SSLDIR/ca-key.pem" ]; then
+# Admins
+if [ -n "$MASTERS" ]; then
+    # If any host requires new certs, just regenerate all master certs
     # kube-apiserver
     gen_key_and_cert "apiserver" "/CN=kube-apiserver"
     cat ca.pem >> apiserver.pem
@@ -88,10 +90,7 @@ if [ ! -e "$SSLDIR/ca-key.pem" ]; then
     gen_key_and_cert "kube-scheduler" "/CN=system:kube-scheduler"
     # kube-controller-manager
     gen_key_and_cert "kube-controller-manager" "/CN=system:kube-controller-manager"
-fi
 
-# Admins
-if [ -n "$MASTERS" ]; then
     for host in $MASTERS; do
         cn="${host%%.*}"
         # admin

roles/kubernetes/secrets/tasks/check-certs.yml

@@ -20,7 +20,18 @@
   register: kubecert_node
   with_items:
     - ca.pem
+    - apiserver.pem
+    - apiserver-key.pem
+    - kube-scheduler.pem
+    - kube-scheduler-key.pem
+    - kube-controller-manager.pem
+    - kube-controller-manager-key.pem
+    - admin-{{ inventory_hostname }}.pem
+    - admin-{{ inventory_hostname }}-key.pem
+    - node-{{ inventory_hostname }}.pem
     - node-{{ inventory_hostname }}-key.pem
+    - kube-proxy-{{ inventory_hostname }}.pem
+    - kube-proxy-{{ inventory_hostname }}-key.pem
 
 - name: "Check_certs | Set 'gen_certs' to true"
   set_fact:
@@ -29,10 +40,41 @@
   run_once: true
   with_items: >-
        ['{{ kube_cert_dir }}/ca.pem',
-       {% for host in groups['k8s-cluster'] %}
+       '{{ kube_cert_dir }}/apiserver.pem',
-       '{{ kube_cert_dir }}/node-{{ host }}-key.pem'
+       '{{ kube_cert_dir }}/apiserver-key.pem',
+       '{{ kube_cert_dir }}/kube-scheduler.pem',
+       '{{ kube_cert_dir }}/kube-scheduler-key.pem',
+       '{{ kube_cert_dir }}/kube-controller-manager.pem',
+       '{{ kube_cert_dir }}/kube-controller-manager-key.pem',
+       {% for host in groups['kube-master'] %}
+       '{{ kube_cert_dir }}/admin-{{ host }}.pem'
+       '{{ kube_cert_dir }}/admin-{{ host }}-key.pem'
        {% if not loop.last %}{{','}}{% endif %}
        {% endfor %}]
+       {% for host in groups['k8s-cluster'] %}
+       '{{ kube_cert_dir }}/node-{{ host }}.pem'
+       '{{ kube_cert_dir }}/node-{{ host }}-key.pem'
+       '{{ kube_cert_dir }}/kube-proxy-{{ host }}.pem'
+       '{{ kube_cert_dir }}/kube-proxy-{{ host }}-key.pem'
+       {% if not loop.last %}{{','}}{% endif %}
+       {% endfor %}]
+
+- name: "Check_certs | Set 'gen_master_certs' to true"
+  set_fact:
+    gen_master_certs: |-
+      {%- set gen = False -%}
+      {% set existing_certs = kubecert_master.files|map(attribute='path')|list|sort %}
+      {% for cert in ['apiserver.pem', 'apiserver-key.pem', 'kube-scheduler.pem',
+                      'kube-scheduler-key.pem', 'kube-controller-manager.pem',
+                      'kube-controller-manager-key.pem'] -%}
+        {% set cert_file = "%s/%s.pem"|format(kube_cert_dir, cert) %}
+        {% if not cert_file in existing_certs -%}
+        {%- set gen = True -%}
+        {% endif -%}
+      {% endfor %}
+      {{ gen }}
+  run_once: true
+
 
 - name: "Check_certs | Set 'gen_node_certs' to true"
   set_fact:
@@ -41,7 +83,8 @@
       {% set existing_certs = kubecert_master.files|map(attribute='path')|list|sort %}
       {% for host in groups['k8s-cluster'] -%}
         {% set host_cert = "%s/node-%s-key.pem"|format(kube_cert_dir, host) %}
-        {% if host_cert in existing_certs -%}
+        {% set kube_proxy_cert = "%s/kube-proxy-%s-key.pem"|format(kube_cert_dir, host) %}
+        {% if host_cert in existing_certs and kube_proxy_cert in existing_certs -%}
         "{{ host }}": False,
         {% else -%}
         "{{ host }}": True,
@@ -50,7 +93,6 @@
       }
   run_once: true
 
-
 - name: "Check_certs | Set 'sync_certs' to true"
   set_fact:
     sync_certs: true
@@ -58,8 +100,8 @@
       {%- set certs = {'sync': False} -%}
       {% if gen_node_certs[inventory_hostname] or
         (not kubecert_node.results[0].stat.exists|default(False)) or
-          (not kubecert_node.results[1].stat.exists|default(False)) or
+          (not kubecert_node.results[10].stat.exists|default(False)) or
-            (kubecert_node.results[1].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[1].stat.path)|map(attribute="checksum")|first|default('')) -%}
+            (kubecert_node.results[10].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[10].stat.path)|map(attribute="checksum")|first|default('')) -%}
               {%- set _ = certs.update({'sync': True}) -%}
       {% endif %}
       {{ certs.sync }}

roles/kubernetes/secrets/tasks/gen_certs_script.yml

@@ -40,7 +40,7 @@
   command: "{{ kube_script_dir }}/make-ssl.sh -f {{ kube_config_dir }}/openssl.conf -d {{ kube_cert_dir }}"
   environment:
     - MASTERS: "{% for m in groups['kube-master'] %}
-                  {% if gen_node_certs[m]|default(false) %}
+                  {% if gen_master_certs|default(false) %}
                     {{ m }}
                   {% endif %}
                 {% endfor %}"