ingress-nginx: Upgrade to 0.16.2

ingress-nginx 0.16.2 (https://github.com/kubernetes/ingress-nginx/releases/tag/nginx-0.16.2)

This patch simplify ingress-nginx deployment by default deploy on
master, with customizable options; on the other hand, remove the
additional Ansible group "kube-ingress" and its k8s node label
injection.

Reference to https://kubernetes.io/docs/concepts/services-networking/ingress/#prerequisites:

    GCE/Google Kubernetes Engine deploys an ingress controller on the master.

By changing `ingress_nginx_nodeselector` plus custom k8s node
label, user could customize the DaemonSet deployment target.

If `ingress_nginx_nodeselector` is empty, will deploy DaemonSet on
every k8s node.

README.md

@@ -104,7 +104,7 @@ Supported Components
 -   Application
     -   [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v1.1.0-k8s1.10
     -   [cert-manager](https://github.com/jetstack/cert-manager) v0.3.2
-    -   [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.15.0
+    -   [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.16.2
 
 Note: kubernetes doesn't support newer docker versions. Among other things kubelet currently breaks on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
 

inventory/sample/group_vars/k8s-cluster.yml

@@ -208,6 +208,8 @@ cephfs_provisioner_enabled: false
 # Nginx ingress controller deployment
 ingress_nginx_enabled: false
 # ingress_nginx_host_network: false
+# ingress_nginx_nodeselector:
+#   node-role.kubernetes.io/master: "true"
 # ingress_nginx_namespace: "ingress-nginx"
 # ingress_nginx_insecure_port: 80
 # ingress_nginx_secure_port: 443

inventory/sample/hosts.ini

@@ -26,11 +26,6 @@
 # node5
 # node6
 
-# [kube-ingress]
-# node2
-# node3
-
 # [k8s-cluster:children]
 # kube-master
 # kube-node
-# kube-ingress

roles/download/defaults/main.yml

@@ -157,7 +157,7 @@ local_volume_provisioner_image_tag: "v2.0.0"
 cephfs_provisioner_image_repo: "quay.io/external_storage/cephfs-provisioner"
 cephfs_provisioner_image_tag: "v1.1.0-k8s1.10"
 ingress_nginx_controller_image_repo: "quay.io/kubernetes-ingress-controller/nginx-ingress-controller"
-ingress_nginx_controller_image_tag: "0.15.0"
+ingress_nginx_controller_image_tag: "0.16.2"
 ingress_nginx_default_backend_image_repo: "gcr.io/google_containers/defaultbackend"
 ingress_nginx_default_backend_image_tag: "1.4"
 cert_manager_version: "v0.3.2"
@@ -564,7 +564,7 @@ downloads:
     tag: "{{ ingress_nginx_controller_image_tag }}"
     sha256: "{{ ingress_nginx_controller_digest_checksum|default(None) }}"
     groups:
-      - kube-ingress
+      - kube-node
   ingress_nginx_default_backend:
     enabled: "{{ ingress_nginx_enabled }}"
     container: true
@@ -572,7 +572,7 @@ downloads:
     tag: "{{ ingress_nginx_default_backend_image_tag }}"
     sha256: "{{ ingress_nginx_default_backend_digest_checksum|default(None) }}"
     groups:
-      - kube-ingress
+      - kube-node
   cert_manager_controller:
     enabled: "{{ cert_manager_enabled }}"
     container: true

roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml

@@ -1,6 +1,8 @@
 ---
 ingress_nginx_namespace: "ingress-nginx"
 ingress_nginx_host_network: false
+ingress_nginx_nodeselector:
+  node-role.kubernetes.io/master: "true"
 ingress_nginx_insecure_port: 80
 ingress_nginx_secure_port: 443
 ingress_nginx_configmap: {}

roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml

@@ -1,5 +1,23 @@
 ---
 
+- name: NGINX Ingress Controller | Remove legacy addon dir and manifests
+  file:
+    path: "{{ kube_config_dir }}/addons/ingress_nginx"
+    state: absent
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+  tags:
+    - upgrade
+
+- name: NGINX Ingress Controller | Remove legacy namespace
+  shell: |
+    {{ bin_dir }}/kubectl delete namespace {{ ingress_nginx_namespace }}
+  ignore_errors: yes
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+  tags:
+    - upgrade
+
 - name: NGINX Ingress Controller | Create addon dir
   file:
     path: "{{ kube_config_dir }}/addons/ingress_nginx"
@@ -7,24 +25,26 @@
     owner: root
     group: root
     mode: 0755
+  when:
+    - inventory_hostname == groups['kube-master'][0]
 
 - name: NGINX Ingress Controller | Create manifests
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/ingress_nginx/{{ item.file }}"
   with_items:
-    - { name: ingress-nginx-ns, file: ingress-nginx-ns.yml, type: ns }
-    - { name: ingress-nginx-sa, file: ingress-nginx-sa.yml, type: sa }
-    - { name: ingress-nginx-role, file: ingress-nginx-role.yml, type: role }
-    - { name: ingress-nginx-rolebinding, file: ingress-nginx-rolebinding.yml, type: rolebinding }
-    - { name: ingress-nginx-clusterrole, file: ingress-nginx-clusterrole.yml, type: clusterrole }
-    - { name: ingress-nginx-clusterrolebinding, file: ingress-nginx-clusterrolebinding.yml, type: clusterrolebinding }
-    - { name: ingress-nginx-cm, file: ingress-nginx-cm.yml, type: cm }
-    - { name: ingress-nginx-tcp-servicecs-cm, file: ingress-nginx-tcp-servicecs-cm.yml, type: cm }
-    - { name: ingress-nginx-udp-servicecs-cm, file: ingress-nginx-udp-servicecs-cm.yml, type: cm }
-    - { name: ingress-nginx-default-backend-svc, file: ingress-nginx-default-backend-svc.yml, type: svc }
-    - { name: ingress-nginx-default-backend-rs, file: ingress-nginx-default-backend-rs.yml, type: rs }
-    - { name: ingress-nginx-controller-ds, file: ingress-nginx-controller-ds.yml, type: ds }
+    - { name: 00-namespace, file: 00-namespace.yml, type: ns }
+    - { name: deploy-default-backend, file: deploy-default-backend.yml, type: deploy }
+    - { name: svc-default-backend, file: svc-default-backend.yml, type: svc }
+    - { name: cm-ingress-nginx, file: cm-ingress-nginx.yml, type: cm }
+    - { name: cm-tcp-services, file: cm-tcp-services.yml, type: cm }
+    - { name: cm-udp-services, file: cm-udp-services.yml, type: cm }
+    - { name: sa-ingress-nginx, file: sa-ingress-nginx.yml, type: sa }
+    - { name: clusterrole-ingress-nginx, file: clusterrole-ingress-nginx.yml, type: clusterrole }
+    - { name: clusterrolebinding-ingress-nginx, file: clusterrolebinding-ingress-nginx.yml, type: clusterrolebinding }
+    - { name: role-ingress-nginx, file: role-ingress-nginx.yml, type: role }
+    - { name: rolebinding-ingress-nginx, file: rolebinding-ingress-nginx.yml, type: rolebinding }
+    - { name: ds-ingress-nginx-controller, file: ds-ingress-nginx-controller.yml, type: ds }
   register: ingress_nginx_manifests
   when:
     - inventory_hostname == groups['kube-master'][0]

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-ingress-nginx.yml.j2

@@ -6,5 +6,7 @@ metadata:
   namespace: {{ ingress_nginx_namespace }}
   labels:
     k8s-app: ingress-nginx
+{% if ingress_nginx_configmap %}
 data:
   {{ ingress_nginx_configmap | to_nice_yaml | indent(2) }}
+{%- endif %}

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-tcp-services.yml.j2

@@ -2,9 +2,11 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: ingress-nginx-tcp-services
+  name: tcp-services
   namespace: {{ ingress_nginx_namespace }}
   labels:
     k8s-app: ingress-nginx
+{% if ingress_nginx_configmap_tcp_services %}
 data:
   {{ ingress_nginx_configmap_tcp_services | to_nice_yaml | indent(2) }}
+{%- endif %}

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-udp-services.yml.j2

@@ -2,9 +2,11 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: ingress-nginx-udp-services
+  name: udp-services
   namespace: {{ ingress_nginx_namespace }}
   labels:
     k8s-app: ingress-nginx
+{% if ingress_nginx_configmap_udp_services %}
 data:
   {{ ingress_nginx_configmap_udp_services | to_nice_yaml | indent(2) }}
+{%- endif %}

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2

@@ -1,27 +1,27 @@
 ---
 apiVersion: apps/v1
-kind: ReplicaSet
+kind: Deployment
 metadata:
-  name: ingress-nginx-default-backend-v{{ ingress_nginx_default_backend_image_tag }}
+  name: default-backend-v{{ ingress_nginx_default_backend_image_tag }}
   namespace: {{ ingress_nginx_namespace }}
   labels:
-    k8s-app: ingress-nginx-default-backend
+    k8s-app: default-backend
     version: v{{ ingress_nginx_default_backend_image_tag }}
 spec:
   replicas: 1
   selector:
     matchLabels:
-      k8s-app: ingress-nginx-default-backend
+      k8s-app: default-backend
       version: v{{ ingress_nginx_default_backend_image_tag }}
   template:
     metadata:
       labels:
-        k8s-app: ingress-nginx-default-backend
+        k8s-app: default-backend
         version: v{{ ingress_nginx_default_backend_image_tag }}
     spec:
       terminationGracePeriodSeconds: 60
       containers:
-        - name: ingress-nginx-default-backend
+        - name: default-backend
           # Any image is permissible as long as:
           # 1. It serves a 404 page at /
           # 2. It serves 200 on a /healthz endpoint
@@ -35,3 +35,10 @@ spec:
             timeoutSeconds: 5
           ports:
             - containerPort: 8080
+          resources:
+            limits:
+              cpu: 10m
+              memory: 20Mi
+            requests:
+              cpu: 10m
+              memory: 20Mi

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2

@@ -7,9 +7,6 @@ metadata:
   labels:
     k8s-app: ingress-nginx
     version: v{{ ingress_nginx_controller_image_tag }}
-  annotations:
-    prometheus.io/port: '10254'
-    prometheus.io/scrape: 'true'
 spec:
   selector:
     matchLabels:
@@ -24,23 +21,36 @@ spec:
         prometheus.io/port: '10254'
         prometheus.io/scrape: 'true'
     spec:
+{% if rbac_enabled %}
+      serviceAccountName: ingress-nginx
+{% endif %}
 {% if ingress_nginx_host_network %}
       hostNetwork: true
 {% endif %}
+{% if ingress_nginx_nodeselector %}
       nodeSelector:
-        node-role.kubernetes.io/ingress: "true"
-      terminationGracePeriodSeconds: 60
+        {{ ingress_nginx_nodeselector | to_nice_yaml }}
+{%- endif %}
       containers:
         - name: ingress-nginx-controller
           image: {{ ingress_nginx_controller_image_repo }}:{{ ingress_nginx_controller_image_tag }}
           imagePullPolicy: {{ k8s_image_pull_policy }}
           args:
             - /nginx-ingress-controller
-            - --default-backend-service=$(POD_NAMESPACE)/ingress-nginx-default-backend
+            - --default-backend-service=$(POD_NAMESPACE)/default-backend
             - --configmap=$(POD_NAMESPACE)/ingress-nginx
-            - --tcp-services-configmap=$(POD_NAMESPACE)/ingress-nginx-tcp-services
-            - --udp-services-configmap=$(POD_NAMESPACE)/ingress-nginx-udp-services
+            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
+            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
+            - --publish-service=$(POD_NAMESPACE)/ingress-nginx
             - --annotations-prefix=nginx.ingress.kubernetes.io
+          securityContext:
+            capabilities:
+                drop:
+                  - ALL
+                add:
+                  - NET_BIND_SERVICE
+            # www-data -> 33
+            runAsUser: 33
           env:
             - name: POD_NAME
               valueFrom:
@@ -78,7 +88,3 @@ spec:
             timeoutSeconds: 1
           securityContext:
             runAsNonRoot: false
-{% if rbac_enabled %}
-      serviceAccountName: ingress-nginx
-{% endif %}
-

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-default-backend.yml.j2

@@ -2,13 +2,13 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: ingress-nginx-default-backend
+  name: default-backend
   namespace: {{ ingress_nginx_namespace }}
   labels:
-    k8s-app: ingress-nginx-default-backend
+    k8s-app: default-backend
 spec:
   ports:
     - port: 80
       targetPort: 8080
   selector:
-    k8s-app: ingress-nginx-default-backend
+    k8s-app: default-backend

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -75,9 +75,6 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% else %}
 {%   set dummy = role_node_labels.append('node-role.kubernetes.io/node=true') %}
 {% endif %}
-{% if inventory_hostname in groups['kube-ingress']|default([]) %}
-{%   set dummy = role_node_labels.append('node-role.kubernetes.io/ingress=true') %}
-{% endif %}
 {% set inventory_node_labels = [] %}
 {% if node_labels is defined %}
 {%   for labelname, labelvalue in node_labels.iteritems() %}

roles/kubernetes/node/templates/kubelet.standard.env.j2

@@ -91,9 +91,6 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 {% else %}
 {%   set dummy = role_node_labels.append('node-role.kubernetes.io/node=true') %}
 {% endif %}
-{% if inventory_hostname in groups['kube-ingress']|default([]) %}
-{%   set dummy = role_node_labels.append('node-role.kubernetes.io/ingress=true') %}
-{% endif %}
 {% set inventory_node_labels = [] %}
 {% if node_labels is defined %}
 {%   for labelname, labelvalue in node_labels.iteritems() %}