Switch to kubeadm deployment mode (#3461)

* Switch to kubeadm deployment mode

Discuss:https://github.com/kubernetes-incubator/kubespray/issues/3301

* Add non-kubeadm upgrage to kubeadm cluster

.gitlab-ci.yml

@@ -244,10 +244,6 @@ before_script:
 # stage: deploy-part1
   MOVED_TO_GROUP_VARS: "true"
 
-.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
-# stage: deploy-part1
-  UPGRADE_TEST: "graceful"
-
 .centos_weave_kubeadm_variables: &centos_weave_kubeadm_variables
 # stage: deploy-part1
   UPGRADE_TEST: "graceful"
@@ -256,6 +252,10 @@ before_script:
 # stage: deploy-part1
   MOVED_TO_GROUP_VARS: "true"
 
+.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
 .ubuntu_contiv_sep_variables: &ubuntu_contiv_sep_variables
 # stage: deploy-special
   MOVED_TO_GROUP_VARS: "true"
@@ -276,7 +276,7 @@ before_script:
 # stage: deploy-part2
   MOVED_TO_GROUP_VARS: "true"
 
-.debian8_calico_variables: &debian8_calico_variables
+.debian9_calico_variables: &debian9_calico_variables
 # stage: deploy-part2
   MOVED_TO_GROUP_VARS: "true"
 
@@ -302,7 +302,7 @@ before_script:
 
 .centos7_multus_calico_variables: &centos7_multus_calico_variables
 # stage: deploy-part2
-  MOVED_TO_GROUP_VARS: "true"
+  UPGRADE_TEST: "graceful"
 
 .coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables
 # stage: deploy-special
@@ -317,7 +317,7 @@ before_script:
   MOVED_TO_GROUP_VARS: "true"
 
 .ubuntu_flannel_variables: &ubuntu_flannel_variables
-# stage: deploy-special
+# stage: deploy-part2
   MOVED_TO_GROUP_VARS: "true"
 
 .ubuntu_kube_router_variables: &ubuntu_kube_router_variables
@@ -378,6 +378,17 @@ gce_centos-weave-kubeadm-sep:
   except: ['triggers']
   only: [/^pr-.*$/]
 
+gce_ubuntu-flannel-ha:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_flannel_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
 ### MANUAL JOBS
 
 gce_ubuntu-weave-sep:
@@ -402,7 +413,7 @@ gce_coreos-calico-sep-triggers:
   only: ['triggers']
 
 gce_ubuntu-canal-ha-triggers:
-  stage: deploy-part2
+  stage: deploy-special
   <<: *job
   <<: *gce
   variables:
@@ -444,7 +455,7 @@ do_ubuntu-canal-ha:
   only: ['master', /^pr-.*$/]
 
 gce_ubuntu-canal-ha:
-  stage: deploy-part2
+  stage: deploy-special
   <<: *job
   <<: *gce
   variables:
@@ -539,24 +550,24 @@ gce_rhel7-weave-triggers:
   when: on_success
   only: ['triggers']
 
-gce_debian8-calico-upgrade:
+gce_debian9-calico-upgrade:
   stage: deploy-part2
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *debian8_calico_variables
+    <<: *debian9_calico_variables
   when: manual
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
-gce_debian8-calico-triggers:
+gce_debian9-calico-triggers:
   stage: deploy-part2
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *debian8_calico_variables
+    <<: *debian9_calico_variables
   when: on_success
   only: ['triggers']
 
@@ -690,17 +701,6 @@ gce_ubuntu-rkt-sep:
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
-gce_ubuntu-flannel-sep:
-  stage: deploy-special
-  <<: *job
-  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu_flannel_variables
-  when: manual
-  except: ['triggers']
-  only: ['master', /^pr-.*$/]
-
 gce_ubuntu-kube-router-sep:
   stage: deploy-special
   <<: *job

inventory/sample/group_vars/all/all.yml

@@ -45,8 +45,8 @@ bin_dir: /usr/local/bin
 #cloud_provider:
 
 
-## Uncomment to enable experimental kubeadm deployment mode
+## kubeadm deployment mode
-#kubeadm_enabled: false
+kubeadm_enabled: true
 
 ## Set these proxy values in order to update package manager and docker daemon to use proxies
 #http_proxy: ""

roles/kubernetes/kubeadm/tasks/main.yml

@@ -87,9 +87,9 @@
 
 - name: Update server field in kube-proxy kubeconfig
   shell: >-
-    {{ bin_dir }}/kubectl get configmap kube-proxy -n kube-system -o yaml
+    {{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf get configmap kube-proxy -n kube-system -o yaml
     | sed 's#server:.*#server:\ {{ kube_apiserver_endpoint }}#g'
-    | {{ bin_dir }}/kubectl replace -f -
+    | {{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf replace -f -
   delegate_to: "{{groups['kube-master']|first}}"
   run_once: true
   when:
@@ -110,7 +110,7 @@
   when: kube_network_plugin in ['calico','canal']
 
 - name: Restart all kube-proxy pods to ensure that they load the new configmap
-  shell: "{{ bin_dir }}/kubectl delete pod -n kube-system -l k8s-app=kube-proxy"
+  shell: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf delete pod -n kube-system -l k8s-app=kube-proxy"
   delegate_to: "{{groups['kube-master']|first}}"
   run_once: true
   when:

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2

@@ -18,4 +18,8 @@ nodeRegistration:
   name: {{ inventory_hostname  }}
 {% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+  criSocket: /var/run/rkt.sock
+{% else %}
+  criSocket: /var/run/dockershim.sock
 {% endif %}

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2

@@ -18,6 +18,8 @@ nodeRegistration:
   name: {{ inventory_hostname  }}
 {% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+  criSocket: /var/run/rkt.sock
 {% else %}
   criSocket: /var/run/dockershim.sock
 {% endif %}

roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml

@@ -15,4 +15,6 @@
     - {src: front-proxy-client-key.pem, dest: front-proxy-client.key}
     - {src: service-account-key.pem, dest: sa.pub}
     - {src: service-account-key.pem, dest: sa.key}
+    - {src: "node-{{ inventory_hostname }}.pem", dest: apiserver-kubelet-client.crt }
+    - {src: "node-{{ inventory_hostname }}-key.pem", dest: apiserver-kubelet-client.key }
   register: kubeadm_copy_old_certs

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -6,6 +6,10 @@
   delegate_to: "{{groups['kube-master']|first}}"
   run_once: true
 
+- name: kubeadm | Migrate old certs if necessary
+  import_tasks: kubeadm-migrate-certs.yml
+  when: old_apiserver_cert.stat.exists
+
 - name: kubeadm | Check service account key
   stat:
     path: "{{ kube_cert_dir }}/sa.key"
@@ -219,7 +223,7 @@
   when: old_apiserver_cert.stat.exists
 
 - name: kubeadm | Remove taint for master with node role
-  command: "{{ bin_dir }}/kubectl taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-"
+  command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-"
   delegate_to: "{{groups['kube-master']|first}}"
   when: inventory_hostname in groups['kube-node']
   failed_when: false

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -171,11 +171,17 @@ nodeRegistration:
 {% if kube_override_hostname|default('') %}
   name: {{ kube_override_hostname }}
 {% endif %}
+{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %}
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
+{% endif %}
 {% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+  criSocket: /var/run/rkt.sock
+{% else %}
+  criSocket: /var/run/dockershim.sock
 {% endif %}
 {% if dynamic_kubelet_configuration %}
 featureGates:

roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2

@@ -7,11 +7,15 @@ nodeRegistration:
 {% if kube_override_hostname|default('') %}
   name: {{ kube_override_hostname }}
 {% endif %}
+{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %}
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
+{% endif %}
 {% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+  criSocket: /var/run/rkt.sock
 {% else %}
   criSocket: /var/run/dockershim.sock
 {% endif %}

roles/kubespray-defaults/defaults/main.yaml

@@ -231,8 +231,8 @@ cert_management: script
 
 helm_deployment_type: host
 
-# Enable kubeadm deployment (experimental)
+# Enable kubeadm deployment
-kubeadm_enabled: false
+kubeadm_enabled: true
 
 # Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts
 kubeconfig_localhost: false

tests/files/gce_coreos-calico-aio.yml

@@ -1,7 +1,7 @@
 # Instance settings
 cloud_image_family: coreos-stable
 cloud_region: us-central1-a
-cloud_machine_type: "n1-standard-1"
+cloud_machine_type: "n1-standard-2"
 mode: aio
 ##user-data to simply turn off coreos upgrades
 startup_script: 'systemctl disable locksmithd && systemctl stop locksmithd'

tests/files/gce_debian9-calico-upgrade.yml

@@ -1,5 +1,5 @@
 # Instance settings
-cloud_image: debian-8-kubespray
+cloud_image: debian-9-kubespray
 cloud_region: us-central1-b
 mode: default
 

tests/files/gce_ubuntu-canal-ha.yml

@@ -1,7 +1,7 @@
 # Instance settings
 cloud_image_family: ubuntu-1604-lts
-cloud_region: us-central1-c
+cloud_region: us-central1-b
-mode: ha
+mode: separate
 
 # Deployment settings
 kube_network_plugin: canal

tests/files/gce_ubuntu-flannel-ha.yml

@@ -1,10 +1,12 @@
 # Instance settings
 cloud_image_family: ubuntu-1604-lts
-cloud_region: us-central1-a
+cloud_region: us-central1-b
-mode: separate
+cloud_machine_type: "n1-standard-1"
+mode: ha
 
 # Deployment settings
 kube_network_plugin: flannel
+kubeadm_enabled: false
 deploy_netchecker: true
 dns_min_replicas: 1
 cloud_provider: gce

tests/files/gce_ubuntu18-flannel-aio.yml

@@ -1,10 +1,11 @@
 # Instance settings
 cloud_image_family: ubuntu-1804-lts
 cloud_region: us-central1-a
-cloud_machine_type: "n1-standard-1"
+cloud_machine_type: "n1-standard-2"
 mode: aio
 
 # Deployment settings
+kubeadm_enabled: true
 kube_network_plugin: flannel
 dynamic_kubelet_configuration: true
 deploy_netchecker: true