From e8ccbebd6f567cdfd4adefb270bc57d1a7ec4200 Mon Sep 17 00:00:00 2001 From: Samuel Liu Date: Wed, 29 Jun 2022 02:55:35 +0800 Subject: [PATCH] add ingress nginx webhook (#9033) * add ingress nginx webhook * fix ingress nginx template --- .../sample/group_vars/k8s_cluster/addons.yml | 1 + roles/download/defaults/main.yml | 2 + .../ingress_nginx/defaults/main.yml | 2 + .../ingress_nginx/tasks/main.yml | 13 +++ .../admission-webhook-configuration.yml.j2 | 29 +++++++ .../templates/admission-webhook-job.yml.j2 | 86 +++++++++++++++++++ .../clusterrole-admission-webhook.yml.j2 | 15 ++++ .../clusterrole-ingress-nginx.yml.j2 | 1 - ...lusterrolebinding-admission-webhook.yml.j2 | 16 ++++ .../ds-ingress-nginx-controller.yml.j2 | 22 +++++ .../templates/role-admission-webhook.ym.j2 | 17 ++++ .../templates/role-admission-webhook.yml.j2 | 17 ++++ .../rolebinding-admission-webhook.yml.j2 | 17 ++++ .../templates/sa-admission-webhook.yml.j2 | 8 ++ .../packet_centos7-flannel-addons-ha.yml | 2 + tests/files/packet_ubuntu18-flannel-ha.yml | 2 + 16 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-configuration.yml.j2 create mode 100644 roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-job.yml.j2 create mode 100644 roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-admission-webhook.yml.j2 create mode 100644 roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-admission-webhook.yml.j2 create mode 100644 roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.ym.j2 create mode 100644 roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.yml.j2 create mode 100644 roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-admission-webhook.yml.j2 create mode 100644 roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-admission-webhook.yml.j2 diff --git a/inventory/sample/group_vars/k8s_cluster/addons.yml b/inventory/sample/group_vars/k8s_cluster/addons.yml index e7f64ce2e..088a1d072 100644 --- a/inventory/sample/group_vars/k8s_cluster/addons.yml +++ b/inventory/sample/group_vars/k8s_cluster/addons.yml @@ -123,6 +123,7 @@ ingress_publish_status_address: "" # - --default-ssl-certificate=default/foo-tls # ingress_nginx_termination_grace_period_seconds: 300 # ingress_nginx_class: nginx +# ingress_nginx_webhook_enabled: false # ALB ingress controller deployment ingress_alb_enabled: false diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 4ecd31cfc..f5119d1bd 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -893,6 +893,8 @@ local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-p local_path_provisioner_image_tag: "v0.0.21" ingress_nginx_controller_image_repo: "{{ kube_image_repo }}/ingress-nginx/controller" ingress_nginx_controller_image_tag: "v1.2.1" +ingress_nginx_kube_webhook_certgen_imae_repo: "{{ kube_image_repo }}/ingress-nginx/kube-webhook-certgen" +ingress_nginx_kube_webhook_certgen_imae_tag: "v1.1.1" alb_ingress_image_repo: "{{ docker_image_repo }}/amazon/aws-alb-ingress-controller" alb_ingress_image_tag: "v1.1.9" cert_manager_version: "v1.8.2" diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml b/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml index 06ed7215b..21ea68c9d 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml @@ -14,3 +14,5 @@ ingress_nginx_configmap_udp_services: {} ingress_nginx_extra_args: [] ingress_nginx_termination_grace_period_seconds: 300 # ingress_nginx_class: nginx +ingress_nginx_webhook_enabled: false +ingress_nginx_webhook_job_ttl: 1800 diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml b/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml index 100420121..363835d7d 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml @@ -25,12 +25,25 @@ - { name: ds-ingress-nginx-controller, file: ds-ingress-nginx-controller.yml, type: ds } ingress_nginx_templates_for_psp: - { name: psp-ingress-nginx, file: psp-ingress-nginx.yml, type: podsecuritypolicy } + ingress_nginx_templates_for_webhook: + - { name: admission-webhook-configuration, file: admission-webhook-configuration.yml, type: sa } + - { name: sa-admission-webhook, file: sa-admission-webhook.yml, type: sa } + - { name: clusterrole-admission-webhook, file: clusterrole-admission-webhook.yml, type: clusterrole } + - { name: clusterrolebinding-admission-webhook, file: clusterrolebinding-admission-webhook.yml, type: clusterrolebinding } + - { name: role-admission-webhook, file: role-admission-webhook.yml, type: role } + - { name: rolebinding-admission-webhook, file: rolebinding-admission-webhook.yml, type: rolebinding } + - { name: admission-webhook-job, file: admission-webhook-job.yml, type: job } - name: NGINX Ingress Controller | Append extra templates to NGINX Ingress Templates list for PodSecurityPolicy set_fact: ingress_nginx_templates: "{{ ingress_nginx_templates_for_psp + ingress_nginx_templates }}" when: podsecuritypolicy_enabled +- name: NGINX Ingress Controller | Append extra templates to NGINX Ingress Templates list for PodSecurityPolicy + set_fact: + ingress_nginx_templates: "{{ ingress_nginx_templates + ingress_nginx_templates_for_webhook }}" + when: ingress_nginx_webhook_enabled + - name: NGINX Ingress Controller | Create manifests template: src: "{{ item.file }}.j2" diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-configuration.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-configuration.yml.j2 new file mode 100644 index 000000000..d6878a01b --- /dev/null +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-configuration.yml.j2 @@ -0,0 +1,29 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: ingress-nginx-controller-admission + namespace: {{ ingress_nginx_namespace }} + path: /networking/v1/ingresses + failurePolicy: Fail + matchPolicy: Equivalent + name: validate.nginx.ingress.kubernetes.io + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + sideEffects: None diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-job.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-job.yml.j2 new file mode 100644 index 000000000..03a84203c --- /dev/null +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/admission-webhook-job.yml.j2 @@ -0,0 +1,86 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission-create + namespace: {{ ingress_nginx_namespace }} +spec: + template: + metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission-create + spec: + containers: + - args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: "{{ ingress_nginx_kube_webhook_certgen_imae_repo }}:{{ ingress_nginx_kube_webhook_certgen_imae_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + name: create + securityContext: + allowPrivilegeEscalation: false + nodeSelector: + kubernetes.io/os: linux + restartPolicy: OnFailure + securityContext: + fsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + serviceAccountName: ingress-nginx-admission + ttlSecondsAfterFinished: {{ ingress_nginx_webhook_job_ttl }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission-patch + namespace: {{ ingress_nginx_namespace }} +spec: + template: + metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission-patch + spec: + containers: + - args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: "{{ ingress_nginx_kube_webhook_certgen_imae_repo }}:{{ ingress_nginx_kube_webhook_certgen_imae_tag }}" + imagePullPolicy: {{ k8s_image_pull_policy }} + name: patch + securityContext: + allowPrivilegeEscalation: false + nodeSelector: + kubernetes.io/os: linux + restartPolicy: OnFailure + securityContext: + fsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 + serviceAccountName: ingress-nginx-admission + ttlSecondsAfterFinished: {{ ingress_nginx_webhook_job_ttl }} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-admission-webhook.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-admission-webhook.yml.j2 new file mode 100644 index 000000000..daa47539a --- /dev/null +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-admission-webhook.yml.j2 @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission +rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-ingress-nginx.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-ingress-nginx.yml.j2 index 400bbf443..5d1e57081 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-ingress-nginx.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-ingress-nginx.yml.j2 @@ -28,4 +28,3 @@ rules: - apiGroups: ["networking.k8s.io"] resources: ["ingressclasses"] verbs: ["get", "list", "watch"] - diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-admission-webhook.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-admission-webhook.yml.j2 new file mode 100644 index 000000000..87915946e --- /dev/null +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-admission-webhook.yml.j2 @@ -0,0 +1,16 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: {{ ingress_nginx_namespace }} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 index b9c9ee693..dcec79bf3 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 @@ -65,6 +65,11 @@ spec: {% for extra_arg in ingress_nginx_extra_args %} - {{ extra_arg }} {% endfor %} +{% if ingress_nginx_webhook_enabled %} + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key +{% endif %} securityContext: capabilities: drop: @@ -96,6 +101,11 @@ spec: containerPort: 10254 {% if not ingress_nginx_host_network %} hostPort: {{ ingress_nginx_metrics_port }} +{% endif %} +{% if ingress_nginx_webhook_enabled %} + - name: webhook + containerPort: 8443 + protocol: TCP {% endif %} livenessProbe: failureThreshold: 3 @@ -118,3 +128,15 @@ spec: timeoutSeconds: 5 successThreshold: 1 failureThreshold: 3 +{% if ingress_nginx_webhook_enabled %} + volumeMounts: + - mountPath: /usr/local/certificates/ + name: webhook-cert + readOnly: true +{% endif %} +{% if ingress_nginx_webhook_enabled %} + volumes: + - name: webhook-cert + secret: + secretName: ingress-nginx-admission +{% endif %} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.ym.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.ym.j2 new file mode 100644 index 000000000..5d1bb0172 --- /dev/null +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.ym.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission + namespace: {{ ingress_nginx_namespace }} +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.yml.j2 new file mode 100644 index 000000000..5d1bb0172 --- /dev/null +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-admission-webhook.yml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission + namespace: {{ ingress_nginx_namespace }} +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-admission-webhook.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-admission-webhook.yml.j2 new file mode 100644 index 000000000..671912db3 --- /dev/null +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-admission-webhook.yml.j2 @@ -0,0 +1,17 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + name: ingress-nginx-admission + namespace: {{ ingress_nginx_namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: +- kind: ServiceAccount + name: ingress-nginx-admission + namespace: {{ ingress_nginx_namespace }} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-admission-webhook.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-admission-webhook.yml.j2 new file mode 100644 index 000000000..488a04523 --- /dev/null +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-admission-webhook.yml.j2 @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + namespace: {{ ingress_nginx_namespace }} + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx diff --git a/tests/files/packet_centos7-flannel-addons-ha.yml b/tests/files/packet_centos7-flannel-addons-ha.yml index c1526d686..4d060a7c2 100644 --- a/tests/files/packet_centos7-flannel-addons-ha.yml +++ b/tests/files/packet_centos7-flannel-addons-ha.yml @@ -16,6 +16,8 @@ etcd_events_cluster_enabled: true local_volume_provisioner_enabled: true kube_encrypt_secret_data: true ingress_nginx_enabled: true +ingress_nginx_webhook_enabled: true +ingress_nginx_webhook_job_ttl: 30 cert_manager_enabled: true # Disable as health checks are still unstable and slow to respond. metrics_server_enabled: false diff --git a/tests/files/packet_ubuntu18-flannel-ha.yml b/tests/files/packet_ubuntu18-flannel-ha.yml index fe672891d..cc513d010 100644 --- a/tests/files/packet_ubuntu18-flannel-ha.yml +++ b/tests/files/packet_ubuntu18-flannel-ha.yml @@ -14,6 +14,8 @@ etcd_events_cluster_enabled: true local_volume_provisioner_enabled: true kube_encrypt_secret_data: true ingress_nginx_enabled: true +ingress_nginx_webhook_enabled: true +ingress_nginx_webhook_job_ttl: 30 cert_manager_enabled: true # Disable as health checks are still unstable and slow to respond. metrics_server_enabled: false