From 655721268d6c511d7879e344a984a4747535e287 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Wed, 5 Apr 2017 13:47:03 +0300 Subject: [PATCH] Fix flannel for 1.6 and apply fixes to enable containerized kubelet --- roles/dnsmasq/tasks/main.yml | 6 +++-- roles/docker/templates/docker-options.conf.j2 | 3 ++- .../templates/netchecker-agent-hostnet-ds.j2 | 4 +-- .../kube-controller-manager.manifest.j2 | 4 +-- .../manifests/kube-scheduler.manifest.j2 | 4 +-- roles/kubernetes/node/defaults/main.yml | 8 ++++++ roles/kubernetes/node/templates/kubelet.j2 | 4 ++- roles/kubernetes/preinstall/handlers/main.yml | 27 +++---------------- 8 files changed, 27 insertions(+), 33 deletions(-) diff --git a/roles/dnsmasq/tasks/main.yml b/roles/dnsmasq/tasks/main.yml index 259d4f50a..edc50703d 100644 --- a/roles/dnsmasq/tasks/main.yml +++ b/roles/dnsmasq/tasks/main.yml @@ -65,7 +65,8 @@ - {name: dnsmasq, file: dnsmasq-svc.yml, type: svc} - {name: dnsmasq-autoscaler, file: dnsmasq-autoscaler.yml, type: deployment} register: manifests - when: inventory_hostname == groups['kube-master'][0] + delegate_to: "{{ groups['kube-master'][0] }}" + run_once: true - name: Start Resources kube: @@ -76,7 +77,8 @@ filename: "{{kube_config_dir}}/{{item.item.file}}" state: "{{item.changed | ternary('latest','present') }}" with_items: "{{ manifests.results }}" - when: inventory_hostname == groups['kube-master'][0] + delegate_to: "{{ groups['kube-master'][0] }}" + run_once: true - name: Check for dnsmasq port (pulling image and running container) wait_for: diff --git a/roles/docker/templates/docker-options.conf.j2 b/roles/docker/templates/docker-options.conf.j2 index 0113bfc61..3f54c853d 100644 --- a/roles/docker/templates/docker-options.conf.j2 +++ b/roles/docker/templates/docker-options.conf.j2 @@ -1,2 +1,3 @@ [Service] -Environment="DOCKER_OPTS={% if docker_options is defined %}{{ docker_options }}{% endif %} --iptables={% if kube_network_plugin == 'flannel' %}true{% else %}false{% endif %}" +Environment="DOCKER_OPTS={{ docker_options | default('') }} \ +--iptables={% if kube_network_plugin == 'flannel' %}true{% else %}false{% endif %}" diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.j2 b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.j2 index 6f0c54db8..13a966c80 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.j2 +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.j2 @@ -13,9 +13,9 @@ spec: app: netchecker-agent-hostnet spec: hostNetwork: True -{%- if kube_version | version_compare('v1.6', '>=') -%} +{% if kube_version | version_compare('v1.6', '>=') %} dnsPolicy: ClusterFirstWithHostNet -{%- endif -%} +{% endif %} containers: - name: netchecker-agent image: "{{ agent_img }}" diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index c214719a3..477d6a64f 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -7,9 +7,9 @@ metadata: k8s-app: kube-controller spec: hostNetwork: true -{%- if kube_version | version_compare('v1.6', '>=') -%} +{% if kube_version | version_compare('v1.6', '>=') %} dnsPolicy: ClusterFirstWithHostNet -{%- endif -%} +{% endif %} containers: - name: kube-controller-manager image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index 77d34288b..7431ddf3d 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -7,9 +7,9 @@ metadata: k8s-app: kube-scheduler spec: hostNetwork: true -{%- if kube_version | version_compare('v1.6', '>=') -%} +{% if kube_version | version_compare('v1.6', '>=') %} dnsPolicy: ClusterFirstWithHostNet -{%- endif -%} +{% endif %} containers: - name: kube-scheduler image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index b4ca13d12..952214179 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -10,6 +10,14 @@ kube_proxy_mode: iptables # policy engine. kube_proxy_masquerade_all: false +# These options reflect limitations of running kubelet in a container. +# Modify at your own risk +kubelet_enable_cri: false +kubelet_cgroups_per_qos: false +# Set to empty to avoid cgroup creation +kubelet_enforce_node_allocatable: "" + + # Limits for kube components and nginx load balancer app kubelet_memory_limit: 512M kubelet_cpu_limit: 100m diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index 37058844d..522f3b05d 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -12,7 +12,9 @@ KUBELET_HOSTNAME="--hostname-override={{ ansible_hostname }}" {% set kubelet_args_base %}--pod-manifest-path={{ kube_manifest_dir }} \ --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \ --kube-reserved cpu={{ kubelet_cpu_limit }},memory={{ kubelet_memory_limit|regex_replace('Mi', 'M') }} \ ---node-status-update-frequency={{ kubelet_status_update_frequency }}{% endset %} +--node-status-update-frequency={{ kubelet_status_update_frequency }} \ +--enable-cri={{ kubelet_enable_cri }} --cgroups-per-qos={{ kubelet_cgroups_per_qos }} \ + --enforce-node-allocatable='{{ kubelet_enforce_node_allocatable }}'{% endset %} {# DNS settings for kubelet #} {% if dns_mode == 'kubedns' %} diff --git a/roles/kubernetes/preinstall/handlers/main.yml b/roles/kubernetes/preinstall/handlers/main.yml index b394aab76..f7e309e92 100644 --- a/roles/kubernetes/preinstall/handlers/main.yml +++ b/roles/kubernetes/preinstall/handlers/main.yml @@ -4,9 +4,7 @@ - Preinstall | reload network - Preinstall | reload kubelet - Preinstall | kube-controller configured - - Preinstall | stop controller - - Preinstall | pause for controller - - Preinstall | restart controller + - Preinstall | restart kube-controller-manager when: not ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] # FIXME(bogdando) https://github.com/projectcalico/felix/issues/1185 @@ -38,9 +36,7 @@ state: restarted notify: - Preinstall | kube-controller configured - - Preinstall | stop controller - - Preinstall | pause for controller - - Preinstall | restart controller + - Preinstall | restart kube-controller-manager when: not dns_early|bool - name: Preinstall | kube-controller configured @@ -48,21 +44,6 @@ register: kube_controller_set when: inventory_hostname in groups['kube-master'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' -- name: Preinstall | stop controller - replace: - dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest" - regexp: '(\s+)image:\s+.*?$' - replace: '\1image: kill.controller.using.fake.image.in:manifest' +- name: Preinstall | restart kube-controller-manager + shell: "docker ps -f name=k8s-controller-manager* -q | xargs --no-run-if-empty docker rm -f" when: inventory_hostname in groups['kube-master'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' and kube_controller_set.stat.exists - -- name: Preinstall | pause for controller - pause: seconds=20 - when: inventory_hostname in groups['kube-master'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' and kube_controller_set.stat.exists - -- name: Preinstall | restart controller - replace: - dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest" - regexp: '(\s+)image:\s+.*?$' - replace: '\1image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}' - when: inventory_hostname in groups['kube-master'] and dns_mode != 'none' and resolvconf_mode == 'host_resolvconf' and kube_controller_set.stat.exists -