Make any_errors_fatal configurable

Useful at scale when 1 or 2 noes my fail and you can proceed with
the majority and go back and fix the others later.

cluster.yml

@@ -6,7 +6,7 @@
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"]}
 
 - hosts: k8s-cluster:etcd:calico-rr
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   gather_facts: false
   vars:
     # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining
@@ -17,13 +17,13 @@
     - { role: bootstrap-os, tags: bootstrap-os}
 
 - hosts: k8s-cluster:etcd:calico-rr
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   vars:
     ansible_ssh_pipelining: true
   gather_facts: true
 
 - hosts: k8s-cluster:etcd:calico-rr
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: kernel-upgrade, tags: kernel-upgrade, when: kernel_upgrade is defined and kernel_upgrade }
@@ -34,38 +34,38 @@
       when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
 
 - hosts: etcd:k8s-cluster:vault
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults, when: "cert_management == 'vault'" }
     - { role: vault, tags: vault, vault_bootstrap: true, when: "cert_management == 'vault'" }
 
 - hosts: etcd
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: etcd, tags: etcd, etcd_cluster_setup: true }
 
 - hosts: k8s-cluster
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: etcd, tags: etcd, etcd_cluster_setup: false }
 
 - hosts: etcd:k8s-cluster:vault
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: vault, tags: vault, when: "cert_management == 'vault'"}
 
 - hosts: k8s-cluster
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: kubernetes/node, tags: node }
     - { role: network_plugin, tags: network }
 
 - hosts: kube-master
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: kubernetes/master, tags: master }
@@ -73,20 +73,20 @@
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
 
 - hosts: calico-rr
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: network_plugin/calico/rr, tags: network }
 
 - hosts: k8s-cluster
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: dnsmasq, when: "dns_mode == 'dnsmasq_kubedns'", tags: dnsmasq }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf }
 
 - hosts: kube-master[0]
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: kubernetes-apps, tags: apps }

upgrade-cluster.yml

@@ -6,7 +6,7 @@
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"]}
 
 - hosts: k8s-cluster:etcd:calico-rr
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   gather_facts: false
   vars:
     # Need to disable pipelining for bootstrap-os as some systems have requiretty in sudoers set, which makes pipelining
@@ -17,13 +17,13 @@
     - { role: bootstrap-os, tags: bootstrap-os}
 
 - hosts: k8s-cluster:etcd:calico-rr
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   vars:
     ansible_ssh_pipelining: true
   gather_facts: true
 
 - hosts: k8s-cluster:etcd:calico-rr
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: kernel-upgrade, tags: kernel-upgrade, when: kernel_upgrade is defined and kernel_upgrade }
@@ -34,32 +34,32 @@
       when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"
 
 - hosts: etcd:k8s-cluster:vault
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults, when: "cert_management == 'vault'" }
     - { role: vault, tags: vault, vault_bootstrap: true, when: "cert_management == 'vault'" }
 
 - hosts: etcd
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: etcd, tags: etcd, etcd_cluster_setup: true }
 
 - hosts: k8s-cluster
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: etcd, tags: etcd, etcd_cluster_setup: false }
 
 - hosts: etcd:k8s-cluster:vault
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults, when: "cert_management == 'vault'"}
     - { role: vault, tags: vault, when: "cert_management == 'vault'"}
 
 #Handle upgrades to master components first to maintain backwards compat.
 - hosts: kube-master
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   serial: 1
   roles:
     - { role: kargo-defaults}
@@ -71,7 +71,7 @@
 
 #Finally handle worker upgrades, based on given batch size
 - hosts: kube-node:!kube-master
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   serial: "{{ serial | default('20%') }}"
   roles:
     - { role: kargo-defaults}
@@ -82,20 +82,20 @@
     - { role: kubernetes-apps/network_plugin, tags: network }
 
 - hosts: calico-rr
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: network_plugin/calico/rr, tags: network }
 
 - hosts: k8s-cluster
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: dnsmasq, when: "dns_mode == 'dnsmasq_kubedns'", tags: dnsmasq }
     - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf }
 
 - hosts: kube-master[0]
-  any_errors_fatal: true
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kargo-defaults}
     - { role: kubernetes-apps, tags: apps }