From d39c273d96afe610fed03a2558ffc1beec64c114 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Thu, 11 Apr 2019 22:52:43 +0300 Subject: [PATCH] Revert "Use K8s 1.14 and add kubeadm experimental control plane mode (#4317)" (#4510) This reverts commit 316508626d317991e8ac5547d097da24a65f4d68. --- README.md | 2 +- .../group_vars/k8s-cluster/k8s-cluster.yml | 2 +- roles/download/defaults/main.yml | 2 +- roles/kubernetes/client/tasks/main.yml | 23 +- roles/kubernetes/kubeadm/tasks/main.yml | 17 +- .../master/defaults/main/kube-proxy.yml | 2 +- .../kubernetes/master/defaults/main/main.yml | 11 +- .../master/tasks/kubeadm-certificate.yml | 30 +++ .../master/tasks/kubeadm-kubeconfig.yml | 34 +++ .../master/tasks/kubeadm-migrate-certs.yml | 4 +- .../tasks/kubeadm-secondary-experimental.yml | 45 ---- .../master/tasks/kubeadm-secondary-legacy.yml | 43 ---- .../kubernetes/master/tasks/kubeadm-setup.yml | 119 +++++----- .../master/tasks/kubeadm-version.yml | 5 + roles/kubernetes/master/tasks/pre-upgrade.yml | 4 +- .../templates/kubeadm-config.v1alpha1.yaml.j2 | 204 ++++++++++++++++++ .../templates/kubeadm-config.v1alpha2.yaml.j2 | 10 +- .../templates/kubeadm-config.v1alpha3.yaml.j2 | 10 +- .../templates/kubeadm-config.v1beta1.yaml.j2 | 10 +- .../kubeadm-controlplane.v1beta1.yaml.j2 | 26 --- .../preinstall/tasks/0020-verify-settings.yml | 1 - roles/kubespray-defaults/defaults/main.yaml | 7 +- roles/network_plugin/calico/defaults/main.yml | 4 - roles/network_plugin/calico/handlers/main.yml | 19 +- .../calico/rr/defaults/main.yml | 4 - roles/network_plugin/calico/rr/tasks/main.yml | 6 +- roles/network_plugin/calico/tasks/check.yml | 3 +- roles/network_plugin/calico/tasks/install.yml | 8 +- roles/network_plugin/calico/tasks/pre.yml | 2 +- .../calico/templates/etcdv2-store.yml.j2 | 6 +- .../calico/templates/etcdv3-store.yml.j2 | 6 +- roles/network_plugin/canal/defaults/main.yml | 5 - roles/network_plugin/canal/tasks/main.yml | 6 +- roles/network_plugin/cilium/defaults/main.yml | 3 - roles/network_plugin/cilium/handlers/main.yml | 2 +- roles/network_plugin/cilium/tasks/main.yml | 6 +- tests/files/gce_centos7-flannel-addons.yml | 1 - 37 files changed, 398 insertions(+), 294 deletions(-) create mode 100644 roles/kubernetes/master/tasks/kubeadm-kubeconfig.yml delete mode 100644 roles/kubernetes/master/tasks/kubeadm-secondary-experimental.yml delete mode 100644 roles/kubernetes/master/tasks/kubeadm-secondary-legacy.yml create mode 100644 roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 delete mode 100644 roles/kubernetes/master/templates/kubeadm-controlplane.v1beta1.yaml.j2 diff --git a/README.md b/README.md index ee7630a78..df7da6d6d 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ Supported Components -------------------- - Core - - [kubernetes](https://github.com/kubernetes/kubernetes) v1.14.0 + - [kubernetes](https://github.com/kubernetes/kubernetes) v1.13.5 - [etcd](https://github.com/coreos/etcd) v3.2.26 - [docker](https://www.docker.com/) v18.06 (see note) - [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2) diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index 139dde74b..11a22af3a 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -20,7 +20,7 @@ kube_users_dir: "{{ kube_config_dir }}/users" kube_api_anonymous_auth: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.14.0 +kube_version: v1.13.5 # kubernetes image repo define kube_image_repo: "gcr.io/google-containers" diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 586c66387..352309311 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -35,7 +35,7 @@ download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube image_arch: "{{host_architecture | default('amd64')}}" # Versions -kube_version: v1.14.0 +kube_version: v1.13.5 kubeadm_version: "{{ kube_version }}" etcd_version: v3.2.26 diff --git a/roles/kubernetes/client/tasks/main.yml b/roles/kubernetes/client/tasks/main.yml index 19d0f019d..71b505a47 100644 --- a/roles/kubernetes/client/tasks/main.yml +++ b/roles/kubernetes/client/tasks/main.yml @@ -40,35 +40,18 @@ run_once: yes when: kubeconfig_localhost|default(false) -# NOTE(mattymo): Please forgive this workaround - name: Generate admin kubeconfig with external api endpoint shell: >- - {% if kubeadm_version is version('v1.14.0', '>=') %} - mkdir -p {{ kube_config_dir }}/external_kubeconfig && + {{ bin_dir }}/kubeadm alpha + {% if kubeadm_version is version('v1.13.0', '<') %} + phase {% endif %} - {{ bin_dir }}/kubeadm - {% if kubeadm_version is version('v1.14.0', '>=') %} - init phase - {% elif kubeadm_version is version('v1.13.0', '>=') %} - alpha - {% else %} - alpha phase - {% endif %} - {% if kubeadm_version is version('v1.14.0', '>=') %} - kubeconfig admin - --kubeconfig-dir {{ kube_config_dir }}/external_kubeconfig - {% else %} kubeconfig user --client-name kubernetes-admin --org system:masters - {% endif %} --cert-dir {{ kube_config_dir }}/ssl --apiserver-advertise-address {{ external_apiserver_address }} --apiserver-bind-port {{ external_apiserver_port }} - {% if kubeadm_version is version('v1.14.0', '>=') %} - && cat {{ kube_config_dir }}/external_kubeconfig/admin.conf && - rm -rf {{ kube_config_dir }}/external_kubeconfig - {% endif %} environment: "{{ proxy_env }}" run_once: yes register: admin_kubeconfig diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 7f0f4b5d1..2d60876ff 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -26,12 +26,6 @@ run_once: true register: temp_token delegate_to: "{{ groups['kube-master'][0] }}" - when: kubeadm_token is not defined - -- name: Set kubeadm_token to generated token - set_fact: - kubeadm_token: "{{ temp_token.stdout }}" - when: kubeadm_token is not defined - name: gets the kubeadm version command: "{{ bin_dir }}/kubeadm version -o short" @@ -67,6 +61,8 @@ dest: "{{ kube_config_dir }}/kubeadm-client.conf" backup: yes when: not is_kube_master + vars: + kubeadm_token: "{{ temp_token.stdout }}" - name: Join to cluster if needed environment: @@ -126,10 +122,11 @@ {{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf get configmap kube-proxy -n kube-system -o yaml | sed 's#server:.*#server:\ {{ kube_apiserver_endpoint }}#g' | {{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf replace -f - + delegate_to: "{{groups['kube-master']|first}}" run_once: true when: - - inventory_hostname == groups['kube-master']|first - kubeadm_config_api_fqdn is not defined + - is_kube_master - kubeadm_discovery_address != kube_apiserver_endpoint - not kube_proxy_remove tags: @@ -137,10 +134,11 @@ - name: Restart all kube-proxy pods to ensure that they load the new configmap shell: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf delete pod -n kube-system -l k8s-app=kube-proxy --force --grace-period=0" + delegate_to: "{{groups['kube-master']|first}}" run_once: true when: - - inventory_hostname == groups['kube-master']|first - kubeadm_config_api_fqdn is not defined + - is_kube_master - kubeadm_discovery_address != kube_apiserver_endpoint - not kube_proxy_remove tags: @@ -161,10 +159,11 @@ # is fixed - name: Delete kube-proxy daemonset if kube_proxy_remove set, e.g. kube_network_plugin providing proxy services shell: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf delete daemonset -n kube-system kube-proxy" + delegate_to: "{{groups['kube-master']|first}}" run_once: true when: - - inventory_hostname == groups['kube-master']|first - kube_proxy_remove + - is_kube_master - kubeadm_discovery_address != kube_apiserver_endpoint tags: - kube-proxy diff --git a/roles/kubernetes/master/defaults/main/kube-proxy.yml b/roles/kubernetes/master/defaults/main/kube-proxy.yml index d5bcdb16f..6dc8d4610 100644 --- a/roles/kubernetes/master/defaults/main/kube-proxy.yml +++ b/roles/kubernetes/master/defaults/main/kube-proxy.yml @@ -107,4 +107,4 @@ kube_proxy_resource_container: /kube-proxy # udpIdleTimeout is how long an idle UDP connection will be kept open (e.g. '250ms', '2s'). # Must be greater than 0. Only applicable for proxyMode=userspace. -kube_proxy_udp_idle_timeout: 250ms +kube_proxy_udp_idle_timeout: 250ms \ No newline at end of file diff --git a/roles/kubernetes/master/defaults/main/main.yml b/roles/kubernetes/master/defaults/main/main.yml index 3109038e8..3fb9c5582 100644 --- a/roles/kubernetes/master/defaults/main/main.yml +++ b/roles/kubernetes/master/defaults/main/main.yml @@ -23,18 +23,11 @@ kube_apiserver_storage_backend: etcd3 # By default, force back to etcd2. Set to true to force etcd3 (experimental!) force_etcd3: false -kube_etcd_cacert_file: ca.pem -kube_etcd_cert_file: node-{{ inventory_hostname }}.pem -kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem - # Associated interfaces must be reachable by the rest of the cluster, and by # CLI/web clients. kube_controller_manager_bind_address: 0.0.0.0 kube_scheduler_bind_address: 0.0.0.0 -# discovery_timeout modifies the discovery timeout -discovery_timeout: 5m0s - # audit support kubernetes_audit: false # path to audit log file @@ -85,6 +78,7 @@ kube_apiserver_request_timeout: "1m0s" # 1.9 and below Admission control plug-ins kube_apiserver_admission_control: + - Initializers - NamespaceLifecycle - LimitRanger - ServiceAccount @@ -105,7 +99,8 @@ kube_apiserver_enable_admission_plugins: [] kube_apiserver_disable_admission_plugins: [] # extra runtime config -kube_api_runtime_config: [] +kube_api_runtime_config: + - admissionregistration.k8s.io/v1alpha1 ## Enable/Disable Kube API Server Authentication Methods kube_basic_auth: false diff --git a/roles/kubernetes/master/tasks/kubeadm-certificate.yml b/roles/kubernetes/master/tasks/kubeadm-certificate.yml index c3d486b83..6072085e0 100644 --- a/roles/kubernetes/master/tasks/kubeadm-certificate.yml +++ b/roles/kubernetes/master/tasks/kubeadm-certificate.yml @@ -12,3 +12,33 @@ - {src: front-proxy-client.crt, dest: front-proxy-client.crt.old} - {src: front-proxy-client.key, dest: front-proxy-client.key.old} ignore_errors: yes + +- name: Remove old certs and keys + file: + path: "{{ kube_cert_dir }}/{{ item }}" + state: absent + with_items: + - apiserver.crt + - apiserver.key + - apiserver-kubelet-client.crt + - apiserver-kubelet-client.key + - front-proxy-client.crt + - front-proxy-client.key + +- name: Generate new certs and keys + command: "{{ bin_dir }}/kubeadm init phase certs {{ item }} --config={{ kube_config_dir }}/kubeadm-config.yaml" + environment: "{{ proxy_env }}" + with_items: + - apiserver + - apiserver-kubelet-client + - front-proxy-client + when: inventory_hostname == groups['kube-master']|first and kubeadm_version is version('v1.13.0', '>=') + +- name: Generate new certs and keys + command: "{{ bin_dir }}/kubeadm alpha phase certs {{ item }} --config={{ kube_config_dir }}/kubeadm-config.yaml" + environment: "{{ proxy_env }}" + with_items: + - apiserver + - apiserver-kubelet-client + - front-proxy-client + when: inventory_hostname == groups['kube-master']|first and kubeadm_version is version('v1.13.0', '<') diff --git a/roles/kubernetes/master/tasks/kubeadm-kubeconfig.yml b/roles/kubernetes/master/tasks/kubeadm-kubeconfig.yml new file mode 100644 index 000000000..7f4bfbf56 --- /dev/null +++ b/roles/kubernetes/master/tasks/kubeadm-kubeconfig.yml @@ -0,0 +1,34 @@ +--- +- name: Backup old configuration files + copy: + src: "{{ kube_config_dir }}/{{ item.src }}" + dest: "{{ kube_config_dir }}/{{ item.dest }}" + remote_src: yes + with_items: + - {src: admin.conf, dest: admin.conf.old} + - {src: kubelet.conf, dest: kubelet.conf.old} + - {src: controller-manager.conf, dest: controller-manager.conf.old} + - {src: scheduler.conf, dest: scheduler.conf.old} + ignore_errors: yes + +- name: Remove old configuration files + file: + path: "{{ kube_config_dir }}/{{ item }}" + state: absent + with_items: + - admin.conf + - kubelet.conf + - controller-manager.conf + - scheduler.conf + +- name: Generate new configuration files + command: "{{ bin_dir }}/kubeadm init phase kubeconfig all --config={{ kube_config_dir }}/kubeadm-config.yaml" + environment: "{{ proxy_env }}" + when: kubeadm_version is version('v1.13.0', '>=') + ignore_errors: yes + +- name: Generate new configuration files + command: "{{ bin_dir }}/kubeadm alpha phase kubeconfig all --config={{ kube_config_dir }}/kubeadm-config.yaml" + environment: "{{ proxy_env }}" + when: kubeadm_version is version('v1.13.0', '<') + ignore_errors: yes diff --git a/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml b/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml index 043530c4a..3a3a45a8e 100644 --- a/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml +++ b/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml @@ -15,6 +15,6 @@ - {src: front-proxy-client-key.pem, dest: front-proxy-client.key} - {src: service-account-key.pem, dest: sa.pub} - {src: service-account-key.pem, dest: sa.key} - - {src: "node-{{ inventory_hostname }}.pem", dest: apiserver-kubelet-client.crt} - - {src: "node-{{ inventory_hostname }}-key.pem", dest: apiserver-kubelet-client.key} + - {src: "node-{{ inventory_hostname }}.pem", dest: apiserver-kubelet-client.crt } + - {src: "node-{{ inventory_hostname }}-key.pem", dest: apiserver-kubelet-client.key } register: kubeadm_copy_old_certs diff --git a/roles/kubernetes/master/tasks/kubeadm-secondary-experimental.yml b/roles/kubernetes/master/tasks/kubeadm-secondary-experimental.yml deleted file mode 100644 index f7f098d35..000000000 --- a/roles/kubernetes/master/tasks/kubeadm-secondary-experimental.yml +++ /dev/null @@ -1,45 +0,0 @@ ---- -- name: Set kubeadm_discovery_address - set_fact: - kubeadm_discovery_address: >- - {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%} - {{ first_kube_master }}:{{ kube_apiserver_port }} - {%- else -%} - {{ kube_apiserver_endpoint }} - {%- endif %} - tags: - - facts - -- name: Create kubeadm ControlPlane config - template: - src: "kubeadm-controlplane.{{ kubeadmConfig_api_version }}.yaml.j2" - dest: "{{ kube_config_dir }}/kubeadm-controlplane.yaml" - backup: yes - when: - - inventory_hostname != groups['kube-master']|first - - not kubeadm_already_run.stat.exists - -- name: Wait for k8s apiserver - wait_for: - host: "{{kubeadm_discovery_address.split(':')[0]}}" - port: "{{kubeadm_discovery_address.split(':')[1]}}" - timeout: 180 - -- name: Joining control plane node to the cluster. - command: >- - {{ bin_dir }}/kubeadm join - --config {{ kube_config_dir}}/kubeadm-controlplane.yaml - --ignore-preflight-errors=all - {% if kubeadm_certificate_key is defined %} - --certificate-key={{ kubeadm_certificate_key }} - {% endif %} - register: kubeadm_join_control_plane - when: - - inventory_hostname != groups['kube-master']|first - - not kubeadm_already_run.stat.exists - environment: - PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" - -- name: Set secret_changed to false to avoid extra token rotation - set_fact: - secret_changed: false diff --git a/roles/kubernetes/master/tasks/kubeadm-secondary-legacy.yml b/roles/kubernetes/master/tasks/kubeadm-secondary-legacy.yml deleted file mode 100644 index 52b9de22a..000000000 --- a/roles/kubernetes/master/tasks/kubeadm-secondary-legacy.yml +++ /dev/null @@ -1,43 +0,0 @@ -- name: slurp kubeadm certs - slurp: - src: "{{ item }}" - with_items: - - "{{ kube_cert_dir }}/apiserver.crt" - - "{{ kube_cert_dir }}/apiserver.key" - - "{{ kube_cert_dir }}/apiserver-kubelet-client.crt" - - "{{ kube_cert_dir }}/apiserver-kubelet-client.key" - - "{{ kube_cert_dir }}/ca.crt" - - "{{ kube_cert_dir }}/ca.key" - - "{{ kube_cert_dir }}/front-proxy-ca.crt" - - "{{ kube_cert_dir }}/front-proxy-ca.key" - - "{{ kube_cert_dir }}/front-proxy-client.crt" - - "{{ kube_cert_dir }}/front-proxy-client.key" - - "{{ kube_cert_dir }}/sa.key" - - "{{ kube_cert_dir }}/sa.pub" - register: kubeadm_certs - delegate_to: "{{ groups['kube-master']|first }}" - -- name: kubeadm | write out kubeadm certs - copy: - dest: "{{ item.item }}" - content: "{{ item.content | b64decode }}" - owner: root - group: root - mode: 0600 - no_log: true - register: copy_kubeadm_certs - with_items: "{{ kubeadm_certs.results }}" - when: inventory_hostname != groups['kube-master']|first - -- name: kubeadm | Init other uninitialized masters - command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all - register: kubeadm_init - retries: 10 - until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr - when: - - inventory_hostname != groups['kube-master']|first - - not kubeadm_already_run.stat.exists - failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr - environment: - PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" - notify: Master | restart kubelet diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index 9265b1ff9..e10847c9b 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -10,10 +10,11 @@ import_tasks: kubeadm-migrate-certs.yml when: old_apiserver_cert.stat.exists -- name: kubeadm | Check serviceaccount key +- name: kubeadm | Check apiserver key stat: path: "{{ kube_cert_dir }}/apiserver.key" - register: sa_key_before + register: apiserver_key_before + delegate_to: "{{groups['kube-master']|first}}" run_once: true - name: kubeadm | Check if kubeadm has already run @@ -61,6 +62,10 @@ sans_address: "{{ groups['kube-master'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list | select('defined') | list }}" tags: facts +- name: kubeadm | Copy etcd cert dir under k8s cert dir + command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd" + changed_when: false + - name: Create audit-policy directory file: path: "{{ audit_policy_file | dirname }}" @@ -89,18 +94,7 @@ - kubeadm_already_run.stat.exists - name: kubeadm | Initialize first master - command: >- - timeout -k 600s 600s - {{ bin_dir }}/kubeadm init - --config={{ kube_config_dir }}/kubeadm-config.yaml - --ignore-preflight-errors=all - {% if kubeadm_version is version('v1.14.0', '>=') %} - --experimental-upload-certs - {% endif %} - --skip-phases=addon/coredns - {% if kubeadm_certificate_key is defined %} - --certificate-key={{ kubeadm_certificate_key }} - {% endif %} + command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all register: kubeadm_init # Retry is because upload config sometimes fails retries: 3 @@ -111,75 +105,76 @@ PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" notify: Master | restart kubelet -- name: set kubeadm certificate key - set_fact: - kubeadm_certificate_key: "{{ item | regex_search('--certificate-key ([^ ]+)','\\1') | first }}" - with_items: "{{ (kubeadm_init|default({'stdout_lines': []}))['stdout_lines'] }}" - run_once: yes - when: - - kubeadm_version is version('v1.14.0', '>=') - - kubeadm_certificate_key is not defined - - kubeadm_init is defined - - item | trim | match('.*--certificate-key .*') - -- name: Create kubeadm token for joining nodes with 24h expiration (default) - command: "{{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token create" +- name: slurp kubeadm certs + slurp: + src: "{{ item }}" + with_items: + - "{{ kube_cert_dir }}/apiserver.crt" + - "{{ kube_cert_dir }}/apiserver.key" + - "{{ kube_cert_dir }}/apiserver-kubelet-client.crt" + - "{{ kube_cert_dir }}/apiserver-kubelet-client.key" + - "{{ kube_cert_dir }}/ca.crt" + - "{{ kube_cert_dir }}/ca.key" + - "{{ kube_cert_dir }}/front-proxy-ca.crt" + - "{{ kube_cert_dir }}/front-proxy-ca.key" + - "{{ kube_cert_dir }}/front-proxy-client.crt" + - "{{ kube_cert_dir }}/front-proxy-client.key" + - "{{ kube_cert_dir }}/sa.key" + - "{{ kube_cert_dir }}/sa.pub" + register: kubeadm_certs + delegate_to: "{{ groups['kube-master']|first }}" run_once: true - register: temp_token - retries: 5 - delay: 5 - until: temp_token is succeeded - when: kubeadm_token is not defined - tags: - - kubeadm_token -- name: Set kubeadm_token - set_fact: - kubeadm_token: "{{ temp_token.stdout }}" - when: kubeadm_token is not defined - tags: - - kubeadm_token +- name: kubeadm | write out kubeadm certs + copy: + dest: "{{ item.item }}" + content: "{{ item.content | b64decode }}" + owner: root + group: root + mode: 0600 + no_log: true + register: copy_kubeadm_certs + with_items: "{{ kubeadm_certs.results }}" + when: inventory_hostname != groups['kube-master']|first -- name: Create hardcoded kubeadm token for joining nodes with 24h expiration (if defined) - shell: >- - {{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token delete {{ kubeadm_token }} || :; - {{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token create {{ kubeadm_token }} - run_once: true +- name: kubeadm | Kubeconfig management with kubeadm + import_tasks: kubeadm-kubeconfig.yml when: - - inventory_hostname == groups['kube-master']|first - - kubeadm_token is defined - tags: - - kubeadm_token + - not upgrade_cluster_setup + - kubeadm_already_run.stat.exists -- name: kubeadm | Initialize other masters (experimental control plane) - include: kubeadm-secondary-experimental.yml - when: kubeadm_control_plane +- name: kubeadm | Init other uninitialized masters + command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all + register: kubeadm_init + retries: 10 + until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr + when: inventory_hostname != groups['kube-master']|first and not kubeadm_already_run.stat.exists + failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr + environment: + PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" + notify: Master | restart kubelet -- name: kubeadm | Initialize other masters (experimental control plane) - include: kubeadm-secondary-legacy.yml - when: not kubeadm_control_plane - -- name: kubeadm | upgrade kubernetes cluster +- name: kubeadm | upgrage kubernetes cluster import_tasks: kubeadm-upgrade.yml when: upgrade_cluster_setup -- name: kubeadm | Check serviceaccount key again +- name: kubeadm | Check apiserver key again stat: - path: "{{ kube_cert_dir }}/sa.key" - register: sa_key_after + path: "{{ kube_cert_dir }}/apiserver.key" + register: apiserver_key_after + delegate_to: "{{groups['kube-master']|first}}" run_once: true - name: kubeadm | Set secret_changed if service account key was updated command: /bin/true notify: Master | set secret_changed - when: sa_key_before.stat.checksum|default("") != sa_key_after.stat.checksum + when: apiserver_key_before.stat.checksum|default("") != apiserver_key_after.stat.checksum - name: kubeadm | cleanup old certs if necessary import_tasks: kubeadm-cleanup-old-certs.yml when: - old_apiserver_cert.stat.exists -# FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file. - name: kubeadm | Remove taint for master with node role command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node.kubernetes.io/master:NoSchedule-" delegate_to: "{{groups['kube-master']|first}}" diff --git a/roles/kubernetes/master/tasks/kubeadm-version.yml b/roles/kubernetes/master/tasks/kubeadm-version.yml index f61657f38..971e7930f 100644 --- a/roles/kubernetes/master/tasks/kubeadm-version.yml +++ b/roles/kubernetes/master/tasks/kubeadm-version.yml @@ -3,6 +3,11 @@ command: "{{ bin_dir }}/kubeadm version -o short" register: kubeadm_output +- name: sets kubeadm api version to v1alpha1 + set_fact: + kubeadmConfig_api_version: v1alpha1 + when: kubeadm_output.stdout is version('v1.11.0', '<') + - name: sets kubeadm api version to v1alpha2 set_fact: kubeadmConfig_api_version: v1alpha2 diff --git a/roles/kubernetes/master/tasks/pre-upgrade.yml b/roles/kubernetes/master/tasks/pre-upgrade.yml index 371f03847..7a36ebc89 100644 --- a/roles/kubernetes/master/tasks/pre-upgrade.yml +++ b/roles/kubernetes/master/tasks/pre-upgrade.yml @@ -3,8 +3,8 @@ command: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} ls /registry/minions" environment: ETCDCTL_API: 2 - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/{{ kube_etcd_key_file }}" + ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" + ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" register: old_data_exists delegate_to: "{{groups['etcd'][0]}}" changed_when: false diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 new file mode 100644 index 000000000..ec157e980 --- /dev/null +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 @@ -0,0 +1,204 @@ +apiVersion: kubeadm.k8s.io/v1alpha1 +kind: MasterConfiguration +api: +{% if kubeadm_config_api_fqdn is defined %} + controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }} + bindPort: {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} +{% else %} + advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }} + bindPort: {{ kube_apiserver_port }} +{% endif %} +etcd: + endpoints: +{% for endpoint in etcd_access_addresses.split(',') %} + - {{ endpoint }} +{% endfor %} + caFile: {{ etcd_cert_dir }}/ca.pem + certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem + keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem +networking: + dnsDomain: {{ dns_domain }} + serviceSubnet: {{ kube_service_addresses }} + podSubnet: {{ kube_pods_subnet }} +kubernetesVersion: {{ kube_version }} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} +cloudProvider: {{cloud_provider}} +cloudConfig: {{ kube_config_dir }}/cloud_config +{% elif cloud_provider is defined and cloud_provider in ["external"] %} +cloudConfig: {{ kube_config_dir }}/cloud_config +{% endif %} +{% if kube_proxy_mode == 'ipvs' %} +kubeProxy: + config: +{% if kube_version is version('v1.10', '<') %} + featureGates: SupportIPVSProxyMode=true +{% endif %} +{% if kube_version is version('v1.10', '>=') %} + featureGates: + SupportIPVSProxyMode: true +{% endif %} + mode: ipvs +{% endif %} +{% if kube_proxy_nodeport_addresses %} + nodePortAddresses: {{ kube_proxy_nodeport_addresses }} +{% endif %} +resourceContainer: "" +authorizationModes: +{% for mode in authorization_modes %} +- {{ mode }} +{% endfor %} +selfHosted: false +apiServerExtraArgs: + bind-address: {{ kube_apiserver_bind_address }} +{% if kube_apiserver_insecure_port|string != "0" %} + insecure-bind-address: {{ kube_apiserver_insecure_bind_address }} +{% endif %} + insecure-port: "{{ kube_apiserver_insecure_port }}" +{% if kube_version is version('v1.10', '<') %} + admission-control: {{ kube_apiserver_admission_control | join(',') }} +{% else %} +{% if kube_apiserver_enable_admission_plugins|length > 0 %} + enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }} +{% endif %} +{% if kube_apiserver_disable_admission_plugins|length > 0 %} + disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }} +{% endif %} +{% endif %} + apiserver-count: "{{ kube_apiserver_count }}" +{% if kube_version is version('v1.9', '>=') %} + endpoint-reconciler-type: lease +{% endif %} +{% if etcd_events_cluster_enabled %} + etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}" +{% endif %} + service-node-port-range: {{ kube_apiserver_node_port_range }} + kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" + profiling: "{{ kube_profiling }}" + request-timeout: "{{ kube_apiserver_request_timeout }}" + repair-malformed-updates: "false" + enable-aggregator-routing: "{{ kube_api_aggregator_routing }}" +{% if kube_api_anonymous_auth is defined and kube_version is version('v1.5', '>=') %} + anonymous-auth: "{{ kube_api_anonymous_auth }}" +{% endif %} +{% if kube_basic_auth|default(true) %} + basic-auth-file: {{ kube_users_dir }}/known_users.csv +{% endif %} +{% if kube_token_auth|default(true) %} + token-auth-file: {{ kube_token_dir }}/known_tokens.csv +{% endif %} +{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %} + oidc-issuer-url: {{ kube_oidc_url }} + oidc-client-id: {{ kube_oidc_client_id }} +{% if kube_oidc_ca_file is defined %} + oidc-ca-file: {{ kube_oidc_ca_file }} +{% endif %} +{% if kube_oidc_username_claim is defined %} + oidc-username-claim: {{ kube_oidc_username_claim }} +{% endif %} +{% if kube_oidc_groups_claim is defined %} + oidc-groups-claim: {{ kube_oidc_groups_claim }} +{% endif %} +{% if kube_oidc_username_prefix is defined %} + oidc-username-prefix: "{{ kube_oidc_username_prefix }}" +{% endif %} +{% if kube_oidc_groups_prefix is defined %} + oidc-groups-prefix: "{{ kube_oidc_groups_prefix }}" +{% endif %} +{% endif %} +{% if kube_webhook_token_auth|default(false) %} + authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml +{% endif %} +{% if kube_encrypt_secret_data %} + experimental-encryption-provider-config: {{ kube_cert_dir }}/secrets_encryption.yaml +{% endif %} + storage-backend: {{ kube_apiserver_storage_backend }} +{% if kube_api_runtime_config is defined %} + runtime-config: {{ kube_api_runtime_config | join(',') }} +{% endif %} + allow-privileged: "true" +{% for key in kube_kubeadm_apiserver_extra_args %} + {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}" +{% endfor %} +{% if kube_feature_gates %} + feature-gates: {{ kube_feature_gates|join(',') }} +{% endif %} +{% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %} + configure-cloud-routes: "true" +{% endif %} +controllerManagerExtraArgs: + node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} + node-monitor-period: {{ kube_controller_node_monitor_period }} + pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }} + node-cidr-mask-size: "{{ kube_network_node_prefix }}" + profiling: "{{ kube_profiling }}" + terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}" +{% if kube_feature_gates %} + feature-gates: {{ kube_feature_gates|join(',') }} +{% endif %} +{% for key in kube_kubeadm_controller_extra_args %} + {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" +{% endfor %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} +controllerManagerExtraVolumes: +{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %} +- name: openstackcacert + hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" + mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" +{% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} +- name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config +{% endif %} +{% endif %} +schedulerExtraArgs: + profiling: "{{ kube_profiling }}" +{% if kube_feature_gates %} + feature-gates: {{ kube_feature_gates|join(',') }} +{% endif %} +{% if kube_kubeadm_scheduler_extra_args|length > 0 %} +{% for key in kube_kubeadm_scheduler_extra_args %} + {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}" +{% endfor %} +{% endif %} +{% if kube_basic_auth|default(true) or kube_token_auth|default(true) or kube_webhook_token_auth|default(false) or ssl_ca_dirs|length %} +apiServerExtraVolumes: +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} +- name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config +{% endif %} +{% if kube_basic_auth|default(true) %} +- name: basic-auth-config + hostPath: {{ kube_users_dir }} + mountPath: {{ kube_users_dir }} +{% endif %} +{% if kube_token_auth|default(true) %} +- name: token-auth-config + hostPath: {{ kube_token_dir }} + mountPath: {{ kube_token_dir }} +{% endif %} +{% if kube_webhook_token_auth|default(false) %} +- name: webhook-token-auth-config + hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml + mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml +{% endif %} +{% if ssl_ca_dirs|length %} +{% for dir in ssl_ca_dirs %} +- name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }} + hostPath: {{ dir }} + mountPath: {{ dir }} + writable: false +{% endfor %} +{% endif %} +{% endif %} +apiServerCertSANs: +{% for san in apiserver_sans %} + - {{ san }} +{% endfor %} +certificatesDir: {{ kube_cert_dir }} +imageRepository: {{ kube_image_repo }} +unifiedControlPlaneImage: "" +{% if kube_override_hostname|default('') %} +nodeName: {{ kube_override_hostname }} +{% endif %} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index df419a069..9281b4c45 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -14,9 +14,9 @@ etcd: {% for endpoint in etcd_access_addresses.split(',') %} - {{ endpoint }} {% endfor %} - caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }} - certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }} - keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }} + caFile: {{ etcd_cert_dir }}/ca.pem + certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem + keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem networking: dnsDomain: {{ dns_domain }} serviceSubnet: {{ kube_service_addresses }} @@ -221,12 +221,10 @@ nodeRegistration: {% if kube_override_hostname|default('') %} name: {{ kube_override_hostname }} {% endif %} -{% if inventory_hostname not in groups['kube-node'] %} +{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %} taints: - effect: NoSchedule key: node.kubernetes.io/master -{% else %} - taints: {} {% endif %} {% if container_manager == 'crio' %} criSocket: /var/run/crio/crio.sock diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 index e2ccbbbed..33c768bec 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 @@ -7,12 +7,10 @@ nodeRegistration: {% if kube_override_hostname|default('') %} name: {{ kube_override_hostname }} {% endif %} -{% if inventory_hostname not in groups['kube-node'] %} +{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %} taints: - effect: NoSchedule key: node.kubernetes.io/master -{% else %} - taints: {} {% endif %} {% if container_manager == 'crio' %} criSocket: /var/run/crio/crio.sock @@ -31,9 +29,9 @@ etcd: {% for endpoint in etcd_access_addresses.split(',') %} - {{ endpoint }} {% endfor %} - caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }} - certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }} - keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }} + caFile: {{ etcd_cert_dir }}/ca.pem + certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem + keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem networking: dnsDomain: {{ dns_domain }} serviceSubnet: {{ kube_service_addresses }} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 index d49a71b77..cf9707246 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 @@ -7,12 +7,10 @@ nodeRegistration: {% if kube_override_hostname|default('') %} name: {{ kube_override_hostname }} {% endif %} -{% if inventory_hostname not in groups['kube-node'] %} +{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %} taints: - effect: NoSchedule key: node.kubernetes.io/master -{% else %} - taints: [] {% endif %} {% if container_manager == 'crio' %} criSocket: /var/run/crio/crio.sock @@ -31,9 +29,9 @@ etcd: {% for endpoint in etcd_access_addresses.split(',') %} - {{ endpoint }} {% endfor %} - caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }} - certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }} - keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }} + caFile: {{ etcd_cert_dir }}/ca.pem + certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem + keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem networking: dnsDomain: {{ dns_domain }} serviceSubnet: {{ kube_service_addresses }} diff --git a/roles/kubernetes/master/templates/kubeadm-controlplane.v1beta1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-controlplane.v1beta1.yaml.j2 deleted file mode 100644 index e9eb6134c..000000000 --- a/roles/kubernetes/master/templates/kubeadm-controlplane.v1beta1.yaml.j2 +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: kubeadm.k8s.io/v1beta1 -kind: JoinConfiguration -discovery: - bootstrapToken: -{% if kubeadm_config_api_fqdn is defined %} - apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} -{% else %} - apiServerEndpoint: {{ kubeadm_discovery_address | replace("https://", "")}} -{% endif %} - token: {{ kubeadm_token }} - unsafeSkipCAVerification: true - timeout: {{ discovery_timeout }} - tlsBootstrapToken: {{ kubeadm_token }} -controlPlane: - localAPIEndpoint: - advertiseAddress: {{ kube_apiserver_address }} - bindPort: {{ kube_apiserver_port }} -nodeRegistration: - name: {{ inventory_hostname }} -{% if container_manager == 'crio' %} - criSocket: /var/run/crio/crio.sock -{% elif container_manager == 'rkt' %} - criSocket: /var/run/rkt.sock -{% else %} - criSocket: /var/run/dockershim.sock -{% endif %} diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index b9120347b..599484878 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -137,7 +137,6 @@ run_once: yes delegate_to: "{{ groups['kube-master'][0] }}" changed_when: false - failed_when: false when: - kube_network_plugin == 'calico' diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 49e1d3b35..205362c3e 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -12,7 +12,7 @@ is_atomic: false disable_swap: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.14.0 +kube_version: v1.13.5 ## Kube Proxy mode One of ['iptables','ipvs'] kube_proxy_mode: ipvs @@ -334,9 +334,6 @@ kube_feature_gates: |- {{ feature_gate_v1_12 }} {%- endif %} -# Enable kubeadm experimental control plane -kubeadm_control_plane: false - # Local volume provisioner storage classes # Levarages Ansibles string to Python datatype casting. Otherwise the dict_key isn't substituted # see https://github.com/ansible/ansible/issues/17324 @@ -385,7 +382,7 @@ no_proxy: >- {%- endif -%} {%- for item in (groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([]))|unique -%} {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(fallback_ips[item])) }}, - {%- if item != hostvars[item].get('ansible_hostname', '') -%} + {%- if item != hostvars[item].get('ansible_hostname', "") -%} {{ hostvars[item]['ansible_hostname'] }}, {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }}, {%- endif -%} diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml index 3ee4b1b29..ae564b1f3 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico/defaults/main.yml @@ -61,7 +61,3 @@ calico_baremetal_nodename: "{{ kube_override_hostname | default(inventory_hostna ### do not enable this, this is detected in scope of tasks, this is just a default value calico_upgrade_needed: false - -kube_etcd_cacert_file: ca.pem -kube_etcd_cert_file: node-{{ inventory_hostname }}.pem -kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem diff --git a/roles/network_plugin/calico/handlers/main.yml b/roles/network_plugin/calico/handlers/main.yml index 64218c040..05cc73289 100644 --- a/roles/network_plugin/calico/handlers/main.yml +++ b/roles/network_plugin/calico/handlers/main.yml @@ -1,14 +1,15 @@ --- -- name: reset_calico_cni +- name: restart calico-node command: /bin/true notify: - - delete 10-calico.conflist - - delete calico-node containers + - Calico | reload systemd + - Calico | reload calico-node -- name: delete 10-calico.conflist - file: - path: /etc/calico/10-calico.conflist - state: absent +- name: Calico | reload systemd + shell: systemctl daemon-reload -- name: delete calico-node containers - shell: "docker ps -af name=k8s_POD_calico-node* -q | xargs --no-run-if-empty docker rm -f" +- name: Calico | reload calico-node + service: + name: calico-node + state: restarted + sleep: 10 diff --git a/roles/network_plugin/calico/rr/defaults/main.yml b/roles/network_plugin/calico/rr/defaults/main.yml index 4871f34a8..bdc2d9f10 100644 --- a/roles/network_plugin/calico/rr/defaults/main.yml +++ b/roles/network_plugin/calico/rr/defaults/main.yml @@ -10,7 +10,3 @@ calico_rr_memory_limit: 1000M calico_rr_cpu_limit: 300m calico_rr_memory_requests: 128M calico_rr_cpu_requests: 150m - -kube_etcd_cacert_file: ca.pem -kube_etcd_cert_file: node-{{ inventory_hostname }}.pem -kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem diff --git a/roles/network_plugin/calico/rr/tasks/main.yml b/roles/network_plugin/calico/rr/tasks/main.yml index d7f02588c..1c41fdf7a 100644 --- a/roles/network_plugin/calico/rr/tasks/main.yml +++ b/roles/network_plugin/calico/rr/tasks/main.yml @@ -22,9 +22,9 @@ state: hard force: yes with_items: - - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"} - - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} - - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} + - {s: "ca.pem", d: "ca_cert.crt"} + - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"} + - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"} - name: Calico-rr | Create dir for logs file: diff --git a/roles/network_plugin/calico/tasks/check.yml b/roles/network_plugin/calico/tasks/check.yml index aaa206519..88fbe63a2 100644 --- a/roles/network_plugin/calico/tasks/check.yml +++ b/roles/network_plugin/calico/tasks/check.yml @@ -13,7 +13,8 @@ shell: "{{ bin_dir }}/calicoctl version | grep 'Cluster Version:' | awk '{ print $3}'" register: calico_version_on_server run_once: yes - async: 20 + delegate_to: "{{ groups['kube-master'][0] }}" + async: 10 poll: 3 changed_when: false diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index c31ec98d7..acf3fd761 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -4,8 +4,6 @@ src: "cni-calico.conflist.j2" dest: "/etc/cni/net.d/{% if calico_version is version('v3.3.0', '>=') %}calico.conflist.template{% else %}10-calico.conflist{% endif %}" owner: kube - register: calico_conflist - notify: reset_calico_cni - name: Calico | Create calico certs directory file: @@ -22,9 +20,9 @@ state: hard force: yes with_items: - - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"} - - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} - - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} + - {s: "ca.pem", d: "ca_cert.crt"} + - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"} + - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"} - name: Calico | Install calicoctl container script template: diff --git a/roles/network_plugin/calico/tasks/pre.yml b/roles/network_plugin/calico/tasks/pre.yml index b843a92ad..4781541bd 100644 --- a/roles/network_plugin/calico/tasks/pre.yml +++ b/roles/network_plugin/calico/tasks/pre.yml @@ -13,4 +13,4 @@ register: calico_kubelet_name delegate_to: "{{ groups['kube-master'][0] }}" when: - - "cloud_provider is defined" + - "cloud_provider is defined" \ No newline at end of file diff --git a/roles/network_plugin/calico/templates/etcdv2-store.yml.j2 b/roles/network_plugin/calico/templates/etcdv2-store.yml.j2 index 41b68d742..c65728838 100644 --- a/roles/network_plugin/calico/templates/etcdv2-store.yml.j2 +++ b/roles/network_plugin/calico/templates/etcdv2-store.yml.j2 @@ -4,6 +4,6 @@ metadata: spec: datastoreType: "etcdv2" etcdEndpoints: "{{ etcd_access_addresses }}" - etcdKeyFile: "{{ etcd_cert_dir }}/{{ kube_etcd_key_file }}" - etcdCertFile: "{{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}" - etcdCACertFile: "{{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}" + etcdKeyFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + etcdCertFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" + etcdCACertFile: "{{ etcd_cert_dir }}/ca.pem" diff --git a/roles/network_plugin/calico/templates/etcdv3-store.yml.j2 b/roles/network_plugin/calico/templates/etcdv3-store.yml.j2 index 17ee75511..3dc566d0f 100644 --- a/roles/network_plugin/calico/templates/etcdv3-store.yml.j2 +++ b/roles/network_plugin/calico/templates/etcdv3-store.yml.j2 @@ -4,6 +4,6 @@ metadata: spec: datastoreType: "etcdv3" etcdEndpoints: "{{ etcd_access_addresses }}" - etcdKeyFile: "{{ etcd_cert_dir }}/{{ kube_etcd_key_file }}" - etcdCertFile: "{{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}" - etcdCACertFile: "{{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}" + etcdKeyFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + etcdCertFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" + etcdCACertFile: "{{ etcd_cert_dir }}/ca.pem" \ No newline at end of file diff --git a/roles/network_plugin/canal/defaults/main.yml b/roles/network_plugin/canal/defaults/main.yml index bed66333e..0be0f14fc 100644 --- a/roles/network_plugin/canal/defaults/main.yml +++ b/roles/network_plugin/canal/defaults/main.yml @@ -30,8 +30,3 @@ calicoctl_memory_limit: 170M calicoctl_cpu_limit: 100m calicoctl_memory_requests: 32M calicoctl_cpu_requests: 25m - -# etcd cert filenames -kube_etcd_cacert_file: ca.pem -kube_etcd_cert_file: node-{{ inventory_hostname }}.pem -kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml index 178ca288c..945274937 100644 --- a/roles/network_plugin/canal/tasks/main.yml +++ b/roles/network_plugin/canal/tasks/main.yml @@ -20,9 +20,9 @@ state: hard force: yes with_items: - - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"} - - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} - - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} + - {s: "ca.pem", d: "ca_cert.crt"} + - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"} + - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"} - name: Canal | Set Flannel etcd configuration command: |- diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml index 95cfdfcf2..e97364644 100755 --- a/roles/network_plugin/cilium/defaults/main.yml +++ b/roles/network_plugin/cilium/defaults/main.yml @@ -5,9 +5,6 @@ cilium_disable_ipv4: false # Etcd SSL dirs cilium_cert_dir: /etc/cilium/certs -kube_etcd_cacert_file: ca.pem -kube_etcd_cert_file: node-{{ inventory_hostname }}.pem -kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem # Cilium Network Policy directory cilium_policy_dir: /etc/kubernetes/policy diff --git a/roles/network_plugin/cilium/handlers/main.yml b/roles/network_plugin/cilium/handlers/main.yml index 00525b995..039c3469f 100644 --- a/roles/network_plugin/cilium/handlers/main.yml +++ b/roles/network_plugin/cilium/handlers/main.yml @@ -11,4 +11,4 @@ - name: Kubelet | reload kubelet service: name: kubelet - state: restarted + state: restarted \ No newline at end of file diff --git a/roles/network_plugin/cilium/tasks/main.yml b/roles/network_plugin/cilium/tasks/main.yml index 44ab4ae57..6c55be663 100755 --- a/roles/network_plugin/cilium/tasks/main.yml +++ b/roles/network_plugin/cilium/tasks/main.yml @@ -21,9 +21,9 @@ state: hard force: yes with_items: - - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"} - - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} - - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} + - {s: "ca.pem", d: "ca_cert.crt"} + - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"} + - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"} - name: Cilium | Create Cilium node manifests template: diff --git a/tests/files/gce_centos7-flannel-addons.yml b/tests/files/gce_centos7-flannel-addons.yml index 8c1a91970..b4984c636 100644 --- a/tests/files/gce_centos7-flannel-addons.yml +++ b/tests/files/gce_centos7-flannel-addons.yml @@ -6,7 +6,6 @@ cloud_machine_type: "n1-standard-2" mode: ha # Deployment settings -kubeadm_control_plane: true kube_network_plugin: flannel helm_enabled: true kubernetes_audit: true