From 0cfcd39d559f3f33fe582e5c577abae2f3b7e5d2 Mon Sep 17 00:00:00 2001 From: Rong Zhang Date: Wed, 21 Nov 2018 17:35:40 +0800 Subject: [PATCH] Switch to kubeadm deployment mode (#3461) * Switch to kubeadm deployment mode Discuss:https://github.com/kubernetes-incubator/kubespray/issues/3301 * Add non-kubeadm upgrage to kubeadm cluster --- .gitlab-ci.yml | 48 +++++++++---------- inventory/sample/group_vars/all/all.yml | 4 +- roles/kubernetes/kubeadm/tasks/main.yml | 6 +-- .../templates/kubeadm-client.conf.v1alpha2.j2 | 4 ++ .../templates/kubeadm-client.conf.v1alpha3.j2 | 2 + .../master/tasks/kubeadm-migrate-certs.yml | 2 + .../kubernetes/master/tasks/kubeadm-setup.yml | 6 ++- .../templates/kubeadm-config.v1alpha2.yaml.j2 | 6 +++ .../templates/kubeadm-config.v1alpha3.yaml.j2 | 4 ++ roles/kubespray-defaults/defaults/main.yaml | 4 +- tests/files/gce_coreos-calico-aio.yml | 2 +- ...ade.yml => gce_debian9-calico-upgrade.yml} | 2 +- tests/files/gce_ubuntu-canal-ha.yml | 4 +- ...nnel-sep.yml => gce_ubuntu-flannel-ha.yml} | 6 ++- tests/files/gce_ubuntu18-flannel-aio.yml | 3 +- 15 files changed, 64 insertions(+), 39 deletions(-) rename tests/files/{gce_debian8-calico-upgrade.yml => gce_debian9-calico-upgrade.yml} (84%) rename tests/files/{gce_ubuntu-flannel-sep.yml => gce_ubuntu-flannel-ha.yml} (64%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3325fdc35..edb9af764 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -244,10 +244,6 @@ before_script: # stage: deploy-part1 MOVED_TO_GROUP_VARS: "true" -.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables -# stage: deploy-part1 - UPGRADE_TEST: "graceful" - .centos_weave_kubeadm_variables: ¢os_weave_kubeadm_variables # stage: deploy-part1 UPGRADE_TEST: "graceful" @@ -256,6 +252,10 @@ before_script: # stage: deploy-part1 MOVED_TO_GROUP_VARS: "true" +.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables +# stage: deploy-special + MOVED_TO_GROUP_VARS: "true" + .ubuntu_contiv_sep_variables: &ubuntu_contiv_sep_variables # stage: deploy-special MOVED_TO_GROUP_VARS: "true" @@ -276,7 +276,7 @@ before_script: # stage: deploy-part2 MOVED_TO_GROUP_VARS: "true" -.debian8_calico_variables: &debian8_calico_variables +.debian9_calico_variables: &debian9_calico_variables # stage: deploy-part2 MOVED_TO_GROUP_VARS: "true" @@ -302,7 +302,7 @@ before_script: .centos7_multus_calico_variables: ¢os7_multus_calico_variables # stage: deploy-part2 - MOVED_TO_GROUP_VARS: "true" + UPGRADE_TEST: "graceful" .coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables # stage: deploy-special @@ -317,7 +317,7 @@ before_script: MOVED_TO_GROUP_VARS: "true" .ubuntu_flannel_variables: &ubuntu_flannel_variables -# stage: deploy-special +# stage: deploy-part2 MOVED_TO_GROUP_VARS: "true" .ubuntu_kube_router_variables: &ubuntu_kube_router_variables @@ -378,6 +378,17 @@ gce_centos-weave-kubeadm-sep: except: ['triggers'] only: [/^pr-.*$/] +gce_ubuntu-flannel-ha: + stage: deploy-part2 + <<: *job + <<: *gce + variables: + <<: *gce_variables + <<: *ubuntu_flannel_variables + when: on_success + except: ['triggers'] + only: [/^pr-.*$/] + ### MANUAL JOBS gce_ubuntu-weave-sep: @@ -402,7 +413,7 @@ gce_coreos-calico-sep-triggers: only: ['triggers'] gce_ubuntu-canal-ha-triggers: - stage: deploy-part2 + stage: deploy-special <<: *job <<: *gce variables: @@ -444,7 +455,7 @@ do_ubuntu-canal-ha: only: ['master', /^pr-.*$/] gce_ubuntu-canal-ha: - stage: deploy-part2 + stage: deploy-special <<: *job <<: *gce variables: @@ -539,24 +550,24 @@ gce_rhel7-weave-triggers: when: on_success only: ['triggers'] -gce_debian8-calico-upgrade: +gce_debian9-calico-upgrade: stage: deploy-part2 <<: *job <<: *gce variables: <<: *gce_variables - <<: *debian8_calico_variables + <<: *debian9_calico_variables when: manual except: ['triggers'] only: ['master', /^pr-.*$/] -gce_debian8-calico-triggers: +gce_debian9-calico-triggers: stage: deploy-part2 <<: *job <<: *gce variables: <<: *gce_variables - <<: *debian8_calico_variables + <<: *debian9_calico_variables when: on_success only: ['triggers'] @@ -690,17 +701,6 @@ gce_ubuntu-rkt-sep: except: ['triggers'] only: ['master', /^pr-.*$/] -gce_ubuntu-flannel-sep: - stage: deploy-special - <<: *job - <<: *gce - variables: - <<: *gce_variables - <<: *ubuntu_flannel_variables - when: manual - except: ['triggers'] - only: ['master', /^pr-.*$/] - gce_ubuntu-kube-router-sep: stage: deploy-special <<: *job diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml index 368b94448..49bf3077c 100644 --- a/inventory/sample/group_vars/all/all.yml +++ b/inventory/sample/group_vars/all/all.yml @@ -45,8 +45,8 @@ bin_dir: /usr/local/bin #cloud_provider: -## Uncomment to enable experimental kubeadm deployment mode -#kubeadm_enabled: false +## kubeadm deployment mode +kubeadm_enabled: true ## Set these proxy values in order to update package manager and docker daemon to use proxies #http_proxy: "" diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 84fd31f69..1a2470d46 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -87,9 +87,9 @@ - name: Update server field in kube-proxy kubeconfig shell: >- - {{ bin_dir }}/kubectl get configmap kube-proxy -n kube-system -o yaml + {{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf get configmap kube-proxy -n kube-system -o yaml | sed 's#server:.*#server:\ {{ kube_apiserver_endpoint }}#g' - | {{ bin_dir }}/kubectl replace -f - + | {{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf replace -f - delegate_to: "{{groups['kube-master']|first}}" run_once: true when: @@ -110,7 +110,7 @@ when: kube_network_plugin in ['calico','canal'] - name: Restart all kube-proxy pods to ensure that they load the new configmap - shell: "{{ bin_dir }}/kubectl delete pod -n kube-system -l k8s-app=kube-proxy" + shell: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf delete pod -n kube-system -l k8s-app=kube-proxy" delegate_to: "{{groups['kube-master']|first}}" run_once: true when: diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 index 62105fbde..eebcdf7c0 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 @@ -18,4 +18,8 @@ nodeRegistration: name: {{ inventory_hostname }} {% if container_manager == 'crio' %} criSocket: /var/run/crio/crio.sock +{% elif container_manager == 'rkt' %} + criSocket: /var/run/rkt.sock +{% else %} + criSocket: /var/run/dockershim.sock {% endif %} diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 index 61707ea85..a1e0887e0 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 @@ -18,6 +18,8 @@ nodeRegistration: name: {{ inventory_hostname }} {% if container_manager == 'crio' %} criSocket: /var/run/crio/crio.sock +{% elif container_manager == 'rkt' %} + criSocket: /var/run/rkt.sock {% else %} criSocket: /var/run/dockershim.sock {% endif %} diff --git a/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml b/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml index 83bfbb22a..3a3a45a8e 100644 --- a/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml +++ b/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml @@ -15,4 +15,6 @@ - {src: front-proxy-client-key.pem, dest: front-proxy-client.key} - {src: service-account-key.pem, dest: sa.pub} - {src: service-account-key.pem, dest: sa.key} + - {src: "node-{{ inventory_hostname }}.pem", dest: apiserver-kubelet-client.crt } + - {src: "node-{{ inventory_hostname }}-key.pem", dest: apiserver-kubelet-client.key } register: kubeadm_copy_old_certs diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index 21a03c963..e0c13fefa 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -6,6 +6,10 @@ delegate_to: "{{groups['kube-master']|first}}" run_once: true +- name: kubeadm | Migrate old certs if necessary + import_tasks: kubeadm-migrate-certs.yml + when: old_apiserver_cert.stat.exists + - name: kubeadm | Check service account key stat: path: "{{ kube_cert_dir }}/sa.key" @@ -219,7 +223,7 @@ when: old_apiserver_cert.stat.exists - name: kubeadm | Remove taint for master with node role - command: "{{ bin_dir }}/kubectl taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-" + command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-" delegate_to: "{{groups['kube-master']|first}}" when: inventory_hostname in groups['kube-node'] failed_when: false diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index c418d5aec..5f1bcab2b 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -171,11 +171,17 @@ nodeRegistration: {% if kube_override_hostname|default('') %} name: {{ kube_override_hostname }} {% endif %} +{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %} taints: - effect: NoSchedule key: node-role.kubernetes.io/master +{% endif %} {% if container_manager == 'crio' %} criSocket: /var/run/crio/crio.sock +{% elif container_manager == 'rkt' %} + criSocket: /var/run/rkt.sock +{% else %} + criSocket: /var/run/dockershim.sock {% endif %} {% if dynamic_kubelet_configuration %} featureGates: diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 index 84d59c93e..4adbe2b8b 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 @@ -7,11 +7,15 @@ nodeRegistration: {% if kube_override_hostname|default('') %} name: {{ kube_override_hostname }} {% endif %} +{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %} taints: - effect: NoSchedule key: node-role.kubernetes.io/master +{% endif %} {% if container_manager == 'crio' %} criSocket: /var/run/crio/crio.sock +{% elif container_manager == 'rkt' %} + criSocket: /var/run/rkt.sock {% else %} criSocket: /var/run/dockershim.sock {% endif %} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 307ad2a2c..c9c8c5575 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -231,8 +231,8 @@ cert_management: script helm_deployment_type: host -# Enable kubeadm deployment (experimental) -kubeadm_enabled: false +# Enable kubeadm deployment +kubeadm_enabled: true # Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts kubeconfig_localhost: false diff --git a/tests/files/gce_coreos-calico-aio.yml b/tests/files/gce_coreos-calico-aio.yml index 7430f5620..51a7c686d 100644 --- a/tests/files/gce_coreos-calico-aio.yml +++ b/tests/files/gce_coreos-calico-aio.yml @@ -1,7 +1,7 @@ # Instance settings cloud_image_family: coreos-stable cloud_region: us-central1-a -cloud_machine_type: "n1-standard-1" +cloud_machine_type: "n1-standard-2" mode: aio ##user-data to simply turn off coreos upgrades startup_script: 'systemctl disable locksmithd && systemctl stop locksmithd' diff --git a/tests/files/gce_debian8-calico-upgrade.yml b/tests/files/gce_debian9-calico-upgrade.yml similarity index 84% rename from tests/files/gce_debian8-calico-upgrade.yml rename to tests/files/gce_debian9-calico-upgrade.yml index 1230bfffd..b129904ff 100644 --- a/tests/files/gce_debian8-calico-upgrade.yml +++ b/tests/files/gce_debian9-calico-upgrade.yml @@ -1,5 +1,5 @@ # Instance settings -cloud_image: debian-8-kubespray +cloud_image: debian-9-kubespray cloud_region: us-central1-b mode: default diff --git a/tests/files/gce_ubuntu-canal-ha.yml b/tests/files/gce_ubuntu-canal-ha.yml index 63f4179fb..419c0426f 100644 --- a/tests/files/gce_ubuntu-canal-ha.yml +++ b/tests/files/gce_ubuntu-canal-ha.yml @@ -1,7 +1,7 @@ # Instance settings cloud_image_family: ubuntu-1604-lts -cloud_region: us-central1-c -mode: ha +cloud_region: us-central1-b +mode: separate # Deployment settings kube_network_plugin: canal diff --git a/tests/files/gce_ubuntu-flannel-sep.yml b/tests/files/gce_ubuntu-flannel-ha.yml similarity index 64% rename from tests/files/gce_ubuntu-flannel-sep.yml rename to tests/files/gce_ubuntu-flannel-ha.yml index e34137852..03076e26a 100644 --- a/tests/files/gce_ubuntu-flannel-sep.yml +++ b/tests/files/gce_ubuntu-flannel-ha.yml @@ -1,10 +1,12 @@ # Instance settings cloud_image_family: ubuntu-1604-lts -cloud_region: us-central1-a -mode: separate +cloud_region: us-central1-b +cloud_machine_type: "n1-standard-1" +mode: ha # Deployment settings kube_network_plugin: flannel +kubeadm_enabled: false deploy_netchecker: true dns_min_replicas: 1 cloud_provider: gce diff --git a/tests/files/gce_ubuntu18-flannel-aio.yml b/tests/files/gce_ubuntu18-flannel-aio.yml index cd3a95534..c6638b6d6 100644 --- a/tests/files/gce_ubuntu18-flannel-aio.yml +++ b/tests/files/gce_ubuntu18-flannel-aio.yml @@ -1,10 +1,11 @@ # Instance settings cloud_image_family: ubuntu-1804-lts cloud_region: us-central1-a -cloud_machine_type: "n1-standard-1" +cloud_machine_type: "n1-standard-2" mode: aio # Deployment settings +kubeadm_enabled: true kube_network_plugin: flannel dynamic_kubelet_configuration: true deploy_netchecker: true