Better fix for openstack cinder zone issue using ignore-volume-az option (#2980)

* Better fix for openstack cinder zone issue[1][2]
using ignore-volume-az option[3].
[1]: https://github.com/kubernetes-incubator/kubespray/pull/2155
[2]: https://github.com/kubernetes-incubator/kubespray/pull/2346
[3]: https://github.com/kubernetes/kubernetes/pull/53523

* Remove kube-scheduler-policy.yaml

inventory/sample/group_vars/all/openstack.yml

@@ -1,5 +1,6 @@
 ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461)
 #openstack_blockstorage_version: "v1/v2/auto (default)"
+#openstack_blockstorage_ignore_volume_az: yes
 ## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables.
 #openstack_lbaas_enabled: True
 #openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP"

roles/kubernetes/master/defaults/main.yml

@@ -129,9 +129,6 @@ kube_kubeadm_apiserver_extra_args: {}
 kube_kubeadm_controller_extra_args: {}
 kube_kubeadm_scheduler_extra_args: {}
 
-## Variable for influencing kube-scheduler behaviour
-volume_cross_zone_attachment: false
-
 ## Encrypting Secret Data at Rest
 kube_encrypt_secret_data: false
 kube_encrypt_token: "{{ lookup('password', credentials_dir + '/kube_encrypt_token.creds length=32 chars=ascii_letters,digits') }}"

roles/kubernetes/master/tasks/static-pod-setup.yml

@@ -26,14 +26,6 @@
 
 - meta: flush_handlers
 
-- name: Write kube-scheduler policy file
-  template:
-    src: kube-scheduler-policy.yaml.j2
-    dest: "{{ kube_config_dir }}/kube-scheduler-policy.yaml"
-  notify: Master | Restart kube-scheduler
-  tags:
-    - kube-scheduler
-
 - name: Write kube-scheduler kubeconfig
   template:
     src: kube-scheduler-kubeconfig.yaml.j2

roles/kubernetes/master/templates/kube-scheduler-policy.yaml.j2

@@ -1,27 +0,0 @@
-{
-"kind" : "Policy",
-"apiVersion" : "v1",
-"predicates" : [
-    {"name" : "MaxEBSVolumeCount"},
-    {"name" : "MaxGCEPDVolumeCount"},
-    {"name" : "MaxAzureDiskVolumeCount"},
-    {"name" : "MatchInterPodAffinity"},
-    {"name" : "NoDiskConflict"},
-    {"name" : "GeneralPredicates"},
-    {"name" : "CheckNodeMemoryPressure"},
-    {"name" : "CheckNodeDiskPressure"},
-    {"name" : "CheckNodeCondition"},
-    {"name" : "PodToleratesNodeTaints"},
-    {"name" : "CheckVolumeBinding"}
-    ],
-"priorities" : [
-    {"name" : "SelectorSpreadPriority", "weight" : 1},
-    {"name" : "InterPodAffinityPriority", "weight" : 1},
-    {"name" : "LeastRequestedPriority", "weight" : 1},
-    {"name" : "BalancedResourceAllocation", "weight" : 1},
-    {"name" : "NodePreferAvoidPodsPriority", "weight" : 1},
-    {"name" : "NodeAffinityPriority", "weight" : 1},
-    {"name" : "TaintTolerationPriority", "weight" : 1}
-    ],
-"hardPodAffinitySymmetricWeight" : 10
-}

roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2

@@ -31,10 +31,6 @@ spec:
     - scheduler
     - --leader-elect=true
     - --kubeconfig={{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml
-{% if volume_cross_zone_attachment %}
-    - --use-legacy-policy-config
-    - --policy-config-file={{ kube_config_dir }}/kube-scheduler-policy.yaml
-{% endif %}
     - --profiling={{ kube_profiling }}
     - --v={{ kube_log_level }}
 {% if kube_feature_gates %}
@@ -69,11 +65,6 @@ spec:
     - mountPath: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
       name: kubeconfig
       readOnly: true
-{% if volume_cross_zone_attachment %}
-    - mountPath: "{{ kube_config_dir }}/kube-scheduler-policy.yaml"
-      name: kube-scheduler-policy
-      readOnly: true
-{% endif %}
   volumes:
   - name: ssl-certs-host
     hostPath:
@@ -89,8 +80,3 @@ spec:
   - name: kubeconfig
     hostPath:
       path: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
-{% if volume_cross_zone_attachment %}
-  - name: kube-scheduler-policy
-    hostPath:
-      path: "{{ kube_config_dir }}/kube-scheduler-policy.yaml"
-{% endif %}

roles/kubernetes/node/templates/openstack-cloud-config.j2

@@ -20,10 +20,13 @@ domain-id ="{{ openstack_domain_id }}"
 ca-file="{{ kube_config_dir }}/openstack-cacert.pem"
 {% endif %}
 
-{% if openstack_blockstorage_version is defined %}
 [BlockStorage]
+{% if openstack_blockstorage_version is defined %}
 bs-version={{ openstack_blockstorage_version }}
 {% endif %}
+{% if openstack_blockstorage_ignore_volume_az is defined and openstack_blockstorage_ignore_volume_az|bool %}
+ignore-volume-az={{ openstack_blockstorage_ignore_volume_az }}
+{% endif %}
 
 {% if openstack_lbaas_enabled and openstack_lbaas_subnet_id is defined %}
 [LoadBalancer]

roles/kubespray-defaults/defaults/main.yaml

@@ -249,6 +249,7 @@ cert_manager_enabled: false
 
 ## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (Fixed in 1.9: https://github.com/kubernetes/kubernetes/issues/50461)
 # openstack_blockstorage_version: "v1/v2/auto (default)"
+openstack_blockstorage_ignore_volume_az: "{{ volume_cross_zone_attachment | default('false') }}"
 ## When OpenStack is used, if LBaaSv2 is available you can enable it with the following 2 variables.
 openstack_lbaas_enabled: false
 # openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP"
@@ -293,11 +294,6 @@ vault_secrets_dir: "{{ vault_base_dir }}/secrets"
 local_volume_provisioner_base_dir: /mnt/disks
 local_volume_provisioner_mount_dir: /mnt/disks
 
-## Running on top of openstack vms with cinder enabled may lead to unschedulable pods due to NoVolumeZoneConflict restriction in kube-scheduler.
-## See https://github.com/kubernetes-incubator/kubespray/issues/2141
-## Set this variable to true to get rid of this issue
-volume_cross_zone_attachment: false
-
 # weave's network password for encryption
 # if null then no network encryption
 # you can use --extra-vars to pass the password in command line