From 352297cf8df0e9356a95939309d6f18c28168224 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Tue, 9 Jul 2019 15:41:59 +0300 Subject: [PATCH] Fixup deploy of kubeadm etcd for Kubernetes v1.15.0 (#4952) * Fixup deploy of kubeadm etcd for Kubernetes v1.15.0 Change-Id: If42c2c75c4d278ba9475ebf76c243f3e6ee4d02e * undo renaming cloud config file Change-Id: Iafbd27c3887d6a2a6d0819c711f150ecf70c515d --- roles/kubernetes/kubeadm/defaults/main.yml | 5 +- roles/kubernetes/kubeadm/tasks/main.yml | 23 ++++---- .../templates/kubeadm-client.conf.v1beta1.j2 | 4 ++ .../kubernetes/master/tasks/kubeadm-setup.yml | 2 +- .../master/tasks/kubeadm-upgrade.yml | 1 + .../templates/kubeadm-config.v1beta1.yaml.j2 | 6 +++ .../templates/kubeadm-config.v1beta2.yaml.j2 | 52 ++++++++++++++++++- .../preinstall/tasks/0070-system-packages.yml | 2 +- 8 files changed, 79 insertions(+), 16 deletions(-) diff --git a/roles/kubernetes/kubeadm/defaults/main.yml b/roles/kubernetes/kubeadm/defaults/main.yml index fb2a02baa..988cbc594 100644 --- a/roles/kubernetes/kubeadm/defaults/main.yml +++ b/roles/kubernetes/kubeadm/defaults/main.yml @@ -1,6 +1,9 @@ --- # discovery_timeout modifies the discovery timeout -discovery_timeout: 5m0s +# This value must be smaller than kubeadm_join_timeout +discovery_timeout: 60s +kubeadm_join_timeout: 120s + # Optionally remove kube_proxy installed by kubeadm kube_proxy_remove: false diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 110de5f55..af5a0855d 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -10,15 +10,24 @@ tags: - facts - - name: Check if kubelet.conf exists stat: path: "{{ kube_config_dir }}/kubelet.conf" register: kubelet_conf +- name: Check if kubeadm CA cert is accessible + stat: + path: "{{ kube_cert_dir }}/ca.crt" + register: kubeadm_ca_stat + delegate_to: "{{ groups['kube-master'][0] }}" + run_once: true + - name: Calculate kubeadm CA cert hash shell: openssl x509 -pubkey -in {{ kube_cert_dir }}/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //' register: kubeadm_ca_hash + when: + - kubeadm_ca_stat.stat is defined + - kubeadm_ca_stat.stat.exists delegate_to: "{{ groups['kube-master'][0] }}" run_once: true @@ -58,23 +67,21 @@ - name: Join to cluster command: >- + timeout -k {{ kubeadm_join_timeout }} {{ kubeadm_join_timeout }} {{ bin_dir }}/kubeadm join --config {{ kube_config_dir }}/kubeadm-client.conf --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests register: kubeadm_join - async: 120 - poll: 15 rescue: - name: Join to cluster with ignores command: >- + timeout -k {{ kubeadm_join_timeout }} {{ kubeadm_join_timeout }} {{ bin_dir }}/kubeadm join --config {{ kube_config_dir }}/kubeadm-client.conf --ignore-preflight-errors=all register: kubeadm_join - async: 180 - poll: 15 always: @@ -85,12 +92,6 @@ Joined with warnings {{ kubeadm_join.stderr_lines }} -- name: Wait for kubelet bootstrap to create config - wait_for: - path: "{{ kube_config_dir }}/kubelet.conf" - delay: 1 - timeout: 60 - - name: Update server field in kubelet kubeconfig lineinfile: dest: "{{ kube_config_dir }}/kubelet.conf" diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 index 93ea517d3..66583412b 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 @@ -9,8 +9,12 @@ discovery: apiServerEndpoint: {{ kubeadm_discovery_address }} {% endif %} token: {{ kubeadm_token }} +{% if kubeadm_ca_hash.stdout is defined %} caCertHashes: - sha256:{{ kubeadm_ca_hash.stdout }} +{% else %} + unsafeSkipCAVerification: true +{% endif %} timeout: {{ discovery_timeout }} tlsBootstrapToken: {{ kubeadm_token }} caCertPath: {{ kube_cert_dir }}/ca.crt diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index 8a00efa49..7b57c94e0 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -103,7 +103,7 @@ - name: kubeadm | Initialize first master command: >- - timeout -k 600s 600s + timeout -k 300s 300s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all diff --git a/roles/kubernetes/master/tasks/kubeadm-upgrade.yml b/roles/kubernetes/master/tasks/kubeadm-upgrade.yml index 5eaefbe3e..b231528fb 100644 --- a/roles/kubernetes/master/tasks/kubeadm-upgrade.yml +++ b/roles/kubernetes/master/tasks/kubeadm-upgrade.yml @@ -29,6 +29,7 @@ --allow-experimental-upgrades --allow-release-candidate-upgrades --etcd-upgrade=false + --force register: kubeadm_upgrade when: inventory_hostname != groups['kube-master']|first failed_when: diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 index 4f5a4bcba..61df859d9 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 @@ -69,6 +69,12 @@ etcd: - {{ san }} {% endfor %} {% endif %} +{% if dns_mode in ['coredns', 'coredns_dual'] %} +dns: + type: CoreDNS + imageRepository: {{ coredns_image_repo | regex_replace('/coredns$','') }} + imageTag: {{ coredns_image_tag }} +{% endif %} networking: dnsDomain: {{ dns_domain }} serviceSubnet: {{ kube_service_addresses }} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2 index b0a44270b..25638e142 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2 @@ -27,6 +27,7 @@ apiVersion: kubeadm.k8s.io/v1beta2 kind: ClusterConfiguration clusterName: {{ cluster_name }} etcd: +{% if not etcd_kubeadm_enabled %} external: endpoints: {% for endpoint in etcd_access_addresses.split(',') %} @@ -35,6 +36,53 @@ etcd: caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }} certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }} keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }} +{% elif etcd_kubeadm_enabled %} + local: + imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}" + imageTag: "{{ etcd_image_tag }}" + dataDir: "/var/lib/etcd" + extraArgs: + metrics: {{ etcd_metrics }} + election-timeout: "{{ etcd_election_timeout }}" + heartbeat-interval: "{{ etcd_heartbeat_interval }}" + auto-compaction-retention: "{{ etcd_compaction_retention }}" +{% if etcd_snapshot_count is defined %} + snapshot-count: "{{ etcd_snapshot_count }}" +{% endif %} +{% if etcd_quota_backend_bytes is defined %} + quota-backend-bytes: "{{ etcd_quota_backend_bytes }}" +{% endif %} +{% if etcd_log_package_levels is defined %} + log-package_levels: "{{ etcd_log_package_levels }}" +{% endif %} +{% for key, value in etcd_extra_vars.items() %} + {{ key }}: "{{ value }}" +{% endfor %} +{% if host_architecture != "amd64" -%} + etcd-unsupported-arch: {{host_architecture}} +{% endif %} + serverCertSANs: +{% for san in etcd_cert_alt_names %} + - {{ san }} +{% endfor %} +{% for san in etcd_cert_alt_ips %} + - {{ san }} +{% endfor %} + peerCertSANs: +{% for san in etcd_cert_alt_names %} + - {{ san }} +{% endfor %} +{% for san in etcd_cert_alt_ips %} + - {{ san }} +{% endfor %} +{% endif %} + +{% if dns_mode in ['coredns', 'coredns_dual'] %} +dns: + type: CoreDNS + imageRepository: {{ coredns_image_repo | regex_replace('/coredns$','') }} + imageTag: {{ coredns_image_tag }} +{% endif %} networking: dnsDomain: {{ dns_domain }} serviceSubnet: {{ kube_service_addresses }} @@ -127,7 +175,7 @@ apiServer: feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} - cloud-provider: {{cloud_provider}} + cloud-provider: {{ cloud_provider }} cloud-config: {{ kube_config_dir }}/cloud_config {% elif cloud_provider is defined and cloud_provider in ["external"] %} cloud-config: {{ kube_config_dir }}/cloud_config @@ -201,7 +249,7 @@ controllerManager: {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" {% endfor %} {% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} - cloud-provider: {{cloud_provider}} + cloud-provider: {{ cloud_provider }} cloud-config: {{ kube_config_dir }}/cloud_config {% elif cloud_provider is defined and cloud_provider in ["external"] %} cloud-config: {{ kube_config_dir }}/cloud_config diff --git a/roles/kubernetes/preinstall/tasks/0070-system-packages.yml b/roles/kubernetes/preinstall/tasks/0070-system-packages.yml index 65cacc656..b3b34b5e6 100644 --- a/roles/kubernetes/preinstall/tasks/0070-system-packages.yml +++ b/roles/kubernetes/preinstall/tasks/0070-system-packages.yml @@ -44,7 +44,7 @@ - name: Update common_required_pkgs with ipvsadm when kube_proxy_mode is ipvs set_fact: - common_required_pkgs: "{{ common_required_pkgs|default([]) + ['ipvsadm'] }}" + common_required_pkgs: "{{ common_required_pkgs|default([]) + ['ipvsadm', 'ipset'] }}" when: kube_proxy_mode == 'ipvs' - name: Install packages requirements