add strategy mitogen_linear when installed mitogen (#5985)

* add strategy mitogen_linear when installed mitogen

* add small docs

Rename playbook file

The raw action executes as a regular Mitogen connection, which requires Python on the target, so add strategy: linear to bootstrap-os role playbook.

* add mitogen to  CI test
fix typo

* enable mitogen test on deploy-part1 tests
change version from master to release
download tar.gz archive

* run all CI tests with mitogen

* disable mitogen with upgrade CI tests

* enable mitogen on CI tests via env vars

* disable mitogen on CI test by default, enable on some different OS

* disable mitogen CI test on centos8
(get error  /usr/bin/python: No such file or directory)

.gitlab-ci.yml

@@ -26,6 +26,7 @@ variables:
   IDEMPOT_CHECK: "false"
   RESET_CHECK: "false"
   UPGRADE_TEST: "false"
+  MITOGEN_ENABLE: "false"
   ANSIBLE_LOG_LEVEL: "-vv"
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:],kube-master[1:]"

.gitlab-ci/packet.yml

@@ -26,11 +26,15 @@ packet_centos7-flannel-containerd-addons-ha:
   extends: .packet
   stage: deploy-part2
   when: on_success
+  variables:
+    MITOGEN_ENABLE: "true"
 
 packet_ubuntu18-crio:
   extends: .packet
   stage: deploy-part2
   when: on_success
+  variables:
+    MITOGEN_ENABLE: "true"
 
 # ### MANUAL JOBS
 
@@ -40,6 +44,7 @@ packet_centos7-weave-upgrade-ha:
   when: on_success
   variables:
     UPGRADE_TEST: basic
+    MITOGEN_ENABLE: "false"
 
 packet_ubuntu16-weave-sep:
   stage: deploy-part2
@@ -95,11 +100,14 @@ packet_debian9-calico-upgrade-once:
   when: on_success
   variables:
     UPGRADE_TEST: graceful
+    MITOGEN_ENABLE: "false"
 
 packet_debian10-containerd:
   stage: deploy-part2
   extends: .packet
   when: on_success
+  variables:
+    MITOGEN_ENABLE: "true"
 
 packet_centos7-calico-ha:
   stage: deploy-part2
@@ -127,6 +135,8 @@ packet_fedora31-flannel:
   stage: deploy-part2
   extends: .packet
   when: on_success
+  variables:
+    MITOGEN_ENABLE: "true"
 
 packet_centos7-kube-router:
   stage: deploy-part2
@@ -172,6 +182,7 @@ packet_debian9-calico-upgrade:
   when: on_success
   variables:
     UPGRADE_TEST: graceful
+    MITOGEN_ENABLE: "false"
 
 packet_ubuntu18-calico-ha-recover:
   stage: deploy-part3

cluster.yml

@@ -23,6 +23,7 @@
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"] }
 
 - hosts: k8s-cluster:etcd
+  strategy: linear
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   gather_facts: false
   roles:

docs/ansible.md

@@ -183,3 +183,7 @@ bastion ansible_host=x.x.x.x
 
 For more information about Ansible and bastion hosts, read
 [Running Ansible Through an SSH Bastion Host](https://blog.scottlowe.org/2015/12/24/running-ansible-through-ssh-bastion-host/)
+
+## Mitogen
+
+You can use [mitogen](mitogen.md) to speed up kubespray.

docs/mitogen.md

@@ -0,0 +1,13 @@
+# Mitogen
+
+[Mitogen for Ansible](https://mitogen.networkgenomics.com/ansible_detailed.html) allow a 1.25x - 7x speedup and a CPU usage reduction of at least 2x, depending on network conditions, modules executed, and time already spent by targets on useful work. Mitogen cannot improve a module once it is executing, it can only ensure the module executes as quickly as possible.
+
+## Install
+
+```ShellSession
+ansible-playbook mitogen.yml
+```
+
+## Limitation
+
+If you are experiencing problems, please see the [documentation](https://mitogen.networkgenomics.com/ansible_detailed.html#noteworthy-differences).

mitogen.yml

@@ -5,8 +5,8 @@
 - hosts: localhost
   strategy: linear
   vars:
-    mitogen_version: master
-    mitogen_url: https://github.com/dw/mitogen/archive/{{ mitogen_version }}.zip
+    mitogen_version: 0.2.9
+    mitogen_url: https://github.com/dw/mitogen/archive/v{{ mitogen_version }}.tar.gz
     ansible_connection: local
   tasks:
     - name: Create mitogen plugin dir
@@ -21,15 +21,22 @@
     - name: download mitogen release
       get_url:
         url: "{{ mitogen_url }}"
-        dest: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.zip"
+        dest: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
         validate_certs: true
 
-    - name: extract zip
+    - name: extract archive
       unarchive:
-        src: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.zip"
+        src: "{{ playbook_dir }}/dist/mitogen_{{ mitogen_version }}.tar.gz"
         dest: "{{ playbook_dir }}/dist/"
 
     - name: copy plugin
       synchronize:
         src: "{{ playbook_dir }}/dist/mitogen-{{ mitogen_version }}/"
         dest: "{{ playbook_dir }}/plugins/mitogen"
+
+    - name: add strategy to ansible.cfg
+      ini_file:
+        path: ansible.cfg
+        section: defaults
+        option: strategy
+        value: mitogen_linear

scale.yml

@@ -24,6 +24,7 @@
 
 - name: Bootstrap any new workers
   hosts: kube-node
+  strategy: linear
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   gather_facts: false
   roles:

tests/scripts/testcases_run.sh

@@ -43,6 +43,13 @@ test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout "$KUBESPRAY
 # Checkout the CI vars file so it is available
 test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml tests/testcases/*.yml
 
+# Install mitogen ansible plugin
+if [ "${MITOGEN_ENABLE}" = "true" ]; then
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} mitogen.yml
+  export ANSIBLE_STRATEGY=mitogen_linear
+  export ANSIBLE_STRATEGY_PLUGINS=plugins/mitogen/ansible_mitogen/plugins/strategy
+fi
+
 # Create cluster
 ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 

upgrade-cluster.yml

@@ -23,6 +23,7 @@
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"] }
 
 - hosts: k8s-cluster:etcd:calico-rr
+  strategy: linear
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   gather_facts: false
   vars: