diff --git a/README.md b/README.md index 9a234fd0c..9685f375a 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ Supported Components - Application - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v1.1.0-k8s1.10 - [cert-manager](https://github.com/jetstack/cert-manager) v0.3.2 - - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.15.0 + - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.16.2 Note: kubernetes doesn't support newer docker versions. Among other things kubelet currently breaks on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin). diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index 20805d0c1..cc77d5008 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -208,6 +208,8 @@ cephfs_provisioner_enabled: false # Nginx ingress controller deployment ingress_nginx_enabled: false # ingress_nginx_host_network: false +# ingress_nginx_nodeselector: +# node-role.kubernetes.io/master: "true" # ingress_nginx_namespace: "ingress-nginx" # ingress_nginx_insecure_port: 80 # ingress_nginx_secure_port: 443 diff --git a/inventory/sample/hosts.ini b/inventory/sample/hosts.ini index bddfa2f80..ad38aedf2 100644 --- a/inventory/sample/hosts.ini +++ b/inventory/sample/hosts.ini @@ -26,11 +26,6 @@ # node5 # node6 -# [kube-ingress] -# node2 -# node3 - # [k8s-cluster:children] # kube-master # kube-node -# kube-ingress diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 597eea501..2e7937f98 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -157,7 +157,7 @@ local_volume_provisioner_image_tag: "v2.0.0" cephfs_provisioner_image_repo: "quay.io/external_storage/cephfs-provisioner" cephfs_provisioner_image_tag: "v1.1.0-k8s1.10" ingress_nginx_controller_image_repo: "quay.io/kubernetes-ingress-controller/nginx-ingress-controller" -ingress_nginx_controller_image_tag: "0.15.0" +ingress_nginx_controller_image_tag: "0.16.2" ingress_nginx_default_backend_image_repo: "gcr.io/google_containers/defaultbackend" ingress_nginx_default_backend_image_tag: "1.4" cert_manager_version: "v0.3.2" @@ -564,7 +564,7 @@ downloads: tag: "{{ ingress_nginx_controller_image_tag }}" sha256: "{{ ingress_nginx_controller_digest_checksum|default(None) }}" groups: - - kube-ingress + - kube-node ingress_nginx_default_backend: enabled: "{{ ingress_nginx_enabled }}" container: true @@ -572,7 +572,7 @@ downloads: tag: "{{ ingress_nginx_default_backend_image_tag }}" sha256: "{{ ingress_nginx_default_backend_digest_checksum|default(None) }}" groups: - - kube-ingress + - kube-node cert_manager_controller: enabled: "{{ cert_manager_enabled }}" container: true diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml b/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml index ff1217809..8acee53eb 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml @@ -1,6 +1,8 @@ --- ingress_nginx_namespace: "ingress-nginx" ingress_nginx_host_network: false +ingress_nginx_nodeselector: + node-role.kubernetes.io/master: "true" ingress_nginx_insecure_port: 80 ingress_nginx_secure_port: 443 ingress_nginx_configmap: {} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml b/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml index 0a37e94cd..eff3c7ed8 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml @@ -1,5 +1,23 @@ --- +- name: NGINX Ingress Controller | Remove legacy addon dir and manifests + file: + path: "{{ kube_config_dir }}/addons/ingress_nginx" + state: absent + when: + - inventory_hostname == groups['kube-master'][0] + tags: + - upgrade + +- name: NGINX Ingress Controller | Remove legacy namespace + shell: | + {{ bin_dir }}/kubectl delete namespace {{ ingress_nginx_namespace }} + ignore_errors: yes + when: + - inventory_hostname == groups['kube-master'][0] + tags: + - upgrade + - name: NGINX Ingress Controller | Create addon dir file: path: "{{ kube_config_dir }}/addons/ingress_nginx" @@ -7,24 +25,26 @@ owner: root group: root mode: 0755 + when: + - inventory_hostname == groups['kube-master'][0] - name: NGINX Ingress Controller | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/ingress_nginx/{{ item.file }}" with_items: - - { name: ingress-nginx-ns, file: ingress-nginx-ns.yml, type: ns } - - { name: ingress-nginx-sa, file: ingress-nginx-sa.yml, type: sa } - - { name: ingress-nginx-role, file: ingress-nginx-role.yml, type: role } - - { name: ingress-nginx-rolebinding, file: ingress-nginx-rolebinding.yml, type: rolebinding } - - { name: ingress-nginx-clusterrole, file: ingress-nginx-clusterrole.yml, type: clusterrole } - - { name: ingress-nginx-clusterrolebinding, file: ingress-nginx-clusterrolebinding.yml, type: clusterrolebinding } - - { name: ingress-nginx-cm, file: ingress-nginx-cm.yml, type: cm } - - { name: ingress-nginx-tcp-servicecs-cm, file: ingress-nginx-tcp-servicecs-cm.yml, type: cm } - - { name: ingress-nginx-udp-servicecs-cm, file: ingress-nginx-udp-servicecs-cm.yml, type: cm } - - { name: ingress-nginx-default-backend-svc, file: ingress-nginx-default-backend-svc.yml, type: svc } - - { name: ingress-nginx-default-backend-rs, file: ingress-nginx-default-backend-rs.yml, type: rs } - - { name: ingress-nginx-controller-ds, file: ingress-nginx-controller-ds.yml, type: ds } + - { name: 00-namespace, file: 00-namespace.yml, type: ns } + - { name: deploy-default-backend, file: deploy-default-backend.yml, type: deploy } + - { name: svc-default-backend, file: svc-default-backend.yml, type: svc } + - { name: cm-ingress-nginx, file: cm-ingress-nginx.yml, type: cm } + - { name: cm-tcp-services, file: cm-tcp-services.yml, type: cm } + - { name: cm-udp-services, file: cm-udp-services.yml, type: cm } + - { name: sa-ingress-nginx, file: sa-ingress-nginx.yml, type: sa } + - { name: clusterrole-ingress-nginx, file: clusterrole-ingress-nginx.yml, type: clusterrole } + - { name: clusterrolebinding-ingress-nginx, file: clusterrolebinding-ingress-nginx.yml, type: clusterrolebinding } + - { name: role-ingress-nginx, file: role-ingress-nginx.yml, type: role } + - { name: rolebinding-ingress-nginx, file: rolebinding-ingress-nginx.yml, type: rolebinding } + - { name: ds-ingress-nginx-controller, file: ds-ingress-nginx-controller.yml, type: ds } register: ingress_nginx_manifests when: - inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-ns.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/00-namespace.yml.j2 similarity index 100% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-ns.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/00-namespace.yml.j2 diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-clusterrole.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-ingress-nginx.yml.j2 similarity index 100% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-clusterrole.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-ingress-nginx.yml.j2 diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-clusterrolebinding.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-ingress-nginx.yml.j2 similarity index 100% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-clusterrolebinding.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-ingress-nginx.yml.j2 diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-ingress-nginx.yml.j2 similarity index 82% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-ingress-nginx.yml.j2 index 7e47e81b1..00c44a97b 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-ingress-nginx.yml.j2 @@ -6,5 +6,7 @@ metadata: namespace: {{ ingress_nginx_namespace }} labels: k8s-app: ingress-nginx +{% if ingress_nginx_configmap %} data: {{ ingress_nginx_configmap | to_nice_yaml | indent(2) }} +{%- endif %} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-tcp-services.yml.j2 similarity index 71% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-tcp-services.yml.j2 index 0a87e91b7..d97c42d97 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-tcp-services.yml.j2 @@ -2,9 +2,11 @@ apiVersion: v1 kind: ConfigMap metadata: - name: ingress-nginx-tcp-services + name: tcp-services namespace: {{ ingress_nginx_namespace }} labels: k8s-app: ingress-nginx +{% if ingress_nginx_configmap_tcp_services %} data: {{ ingress_nginx_configmap_tcp_services | to_nice_yaml | indent(2) }} +{%- endif %} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-udp-services.yml.j2 similarity index 71% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-udp-services.yml.j2 index d943e5718..b343869b7 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-udp-services.yml.j2 @@ -2,9 +2,11 @@ apiVersion: v1 kind: ConfigMap metadata: - name: ingress-nginx-udp-services + name: udp-services namespace: {{ ingress_nginx_namespace }} labels: k8s-app: ingress-nginx +{% if ingress_nginx_configmap_udp_services %} data: {{ ingress_nginx_configmap_udp_services | to_nice_yaml | indent(2) }} +{%- endif %} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-default-backend-rs.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2 similarity index 71% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-default-backend-rs.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2 index c0bed920b..eca5a5084 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-default-backend-rs.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2 @@ -1,27 +1,27 @@ --- apiVersion: apps/v1 -kind: ReplicaSet +kind: Deployment metadata: - name: ingress-nginx-default-backend-v{{ ingress_nginx_default_backend_image_tag }} + name: default-backend-v{{ ingress_nginx_default_backend_image_tag }} namespace: {{ ingress_nginx_namespace }} labels: - k8s-app: ingress-nginx-default-backend + k8s-app: default-backend version: v{{ ingress_nginx_default_backend_image_tag }} spec: replicas: 1 selector: matchLabels: - k8s-app: ingress-nginx-default-backend + k8s-app: default-backend version: v{{ ingress_nginx_default_backend_image_tag }} template: metadata: labels: - k8s-app: ingress-nginx-default-backend + k8s-app: default-backend version: v{{ ingress_nginx_default_backend_image_tag }} spec: terminationGracePeriodSeconds: 60 containers: - - name: ingress-nginx-default-backend + - name: default-backend # Any image is permissible as long as: # 1. It serves a 404 page at / # 2. It serves 200 on a /healthz endpoint @@ -35,3 +35,10 @@ spec: timeoutSeconds: 5 ports: - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 similarity index 79% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 index 40e1d4715..5d141d4ff 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 @@ -7,9 +7,6 @@ metadata: labels: k8s-app: ingress-nginx version: v{{ ingress_nginx_controller_image_tag }} - annotations: - prometheus.io/port: '10254' - prometheus.io/scrape: 'true' spec: selector: matchLabels: @@ -24,23 +21,36 @@ spec: prometheus.io/port: '10254' prometheus.io/scrape: 'true' spec: +{% if rbac_enabled %} + serviceAccountName: ingress-nginx +{% endif %} {% if ingress_nginx_host_network %} hostNetwork: true {% endif %} +{% if ingress_nginx_nodeselector %} nodeSelector: - node-role.kubernetes.io/ingress: "true" - terminationGracePeriodSeconds: 60 + {{ ingress_nginx_nodeselector | to_nice_yaml }} +{%- endif %} containers: - name: ingress-nginx-controller image: {{ ingress_nginx_controller_image_repo }}:{{ ingress_nginx_controller_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - /nginx-ingress-controller - - --default-backend-service=$(POD_NAMESPACE)/ingress-nginx-default-backend + - --default-backend-service=$(POD_NAMESPACE)/default-backend - --configmap=$(POD_NAMESPACE)/ingress-nginx - - --tcp-services-configmap=$(POD_NAMESPACE)/ingress-nginx-tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/ingress-nginx-udp-services + - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services + - --udp-services-configmap=$(POD_NAMESPACE)/udp-services + - --publish-service=$(POD_NAMESPACE)/ingress-nginx - --annotations-prefix=nginx.ingress.kubernetes.io + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + # www-data -> 33 + runAsUser: 33 env: - name: POD_NAME valueFrom: @@ -78,7 +88,3 @@ spec: timeoutSeconds: 1 securityContext: runAsNonRoot: false -{% if rbac_enabled %} - serviceAccountName: ingress-nginx -{% endif %} - diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-role.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-ingress-nginx.yml.j2 similarity index 100% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-role.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-ingress-nginx.yml.j2 diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-rolebinding.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-ingress-nginx.yml.j2 similarity index 100% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-rolebinding.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-ingress-nginx.yml.j2 diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-sa.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-ingress-nginx.yml.j2 similarity index 100% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-sa.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-ingress-nginx.yml.j2 diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-default-backend-svc.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-default-backend.yml.j2 similarity index 56% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-default-backend-svc.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-default-backend.yml.j2 index ab23f3799..326cc8843 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-default-backend-svc.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-default-backend.yml.j2 @@ -2,13 +2,13 @@ apiVersion: v1 kind: Service metadata: - name: ingress-nginx-default-backend + name: default-backend namespace: {{ ingress_nginx_namespace }} labels: - k8s-app: ingress-nginx-default-backend + k8s-app: default-backend spec: ports: - port: 80 targetPort: 8080 selector: - k8s-app: ingress-nginx-default-backend + k8s-app: default-backend diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index 4ca17ef53..e313161a0 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -75,9 +75,6 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% else %} {% set dummy = role_node_labels.append('node-role.kubernetes.io/node=true') %} {% endif %} -{% if inventory_hostname in groups['kube-ingress']|default([]) %} -{% set dummy = role_node_labels.append('node-role.kubernetes.io/ingress=true') %} -{% endif %} {% set inventory_node_labels = [] %} {% if node_labels is defined %} {% for labelname, labelvalue in node_labels.iteritems() %} diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2 index 83d657f7e..96fad4d0e 100644 --- a/roles/kubernetes/node/templates/kubelet.standard.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2 @@ -91,9 +91,6 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% else %} {% set dummy = role_node_labels.append('node-role.kubernetes.io/node=true') %} {% endif %} -{% if inventory_hostname in groups['kube-ingress']|default([]) %} -{% set dummy = role_node_labels.append('node-role.kubernetes.io/ingress=true') %} -{% endif %} {% set inventory_node_labels = [] %} {% if node_labels is defined %} {% for labelname, labelvalue in node_labels.iteritems() %}