From 1adf23151288d8176e20e98b1d2617c31a4c5b98 Mon Sep 17 00:00:00 2001 From: Aleksandr Didenko Date: Mon, 13 Mar 2017 16:04:31 +0100 Subject: [PATCH] Move calico-policy-controller into separate role By default Calico CNI does not create any network access policies or profiles if 'policy' is enabled in CNI config. And without any policies/profiles network access to/from PODs is blocked. K8s related policies are created by calico-policy-controller in such case. So we need to start it as soon as possible, before any real workloads. This patch also fixes kube-api port in calico-policy-controller yaml template. Closes #1132 --- cluster.yml | 1 + inventory/group_vars/k8s-cluster.yml | 3 +++ roles/kubernetes-apps/ansible/defaults/main.yml | 7 ------- roles/kubernetes-apps/ansible/tasks/main.yml | 5 ----- roles/kubernetes-apps/meta/main.yml | 5 ----- .../policy_controller/calico/defaults/main.yml | 9 +++++++++ .../calico/tasks/main.yml} | 5 +++-- .../templates/calico-policy-controller.yml.j2 | 2 +- .../policy_controller/meta/main.yml | 14 ++++++++++++++ .../templates/manifests/kube-apiserver.manifest.j2 | 2 +- .../calico/templates/cni-calico.conf.j2 | 2 +- 11 files changed, 33 insertions(+), 22 deletions(-) create mode 100644 roles/kubernetes-apps/policy_controller/calico/defaults/main.yml rename roles/kubernetes-apps/{ansible/tasks/calico-policy-controller.yml => policy_controller/calico/tasks/main.yml} (92%) rename roles/kubernetes-apps/{ansible => policy_controller/calico}/templates/calico-policy-controller.yml.j2 (96%) create mode 100644 roles/kubernetes-apps/policy_controller/meta/main.yml diff --git a/cluster.yml b/cluster.yml index 01b033b2f..577bc21f1 100644 --- a/cluster.yml +++ b/cluster.yml @@ -70,6 +70,7 @@ - { role: kargo-defaults} - { role: kubernetes/master, tags: master } - { role: kubernetes-apps/network_plugin, tags: network } + - { role: kubernetes-apps/policy_controller, tags: policy-controller } - hosts: calico-rr any_errors_fatal: true diff --git a/inventory/group_vars/k8s-cluster.yml b/inventory/group_vars/k8s-cluster.yml index cbd922c63..5430a5e1f 100644 --- a/inventory/group_vars/k8s-cluster.yml +++ b/inventory/group_vars/k8s-cluster.yml @@ -80,6 +80,9 @@ kube_users: # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico +# Enable kubernetes network policies +enable_network_policy: false + # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 925dd03b8..6d0562fc9 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -19,12 +19,6 @@ kubednsmasq_image_tag: "{{ kubednsmasq_version }}" exechealthz_image_repo: "gcr.io/google_containers/exechealthz-amd64" exechealthz_image_tag: "{{ exechealthz_version }}" -# Limits for calico apps -calico_policy_controller_cpu_limit: 100m -calico_policy_controller_memory_limit: 256M -calico_policy_controller_cpu_requests: 30m -calico_policy_controller_memory_requests: 64M - # Netchecker deploy_netchecker: false netchecker_port: 31081 @@ -45,5 +39,4 @@ netchecker_server_memory_requests: 64M # SSL etcd_cert_dir: "/etc/ssl/etcd/ssl" -calico_cert_dir: "/etc/calico/certs" canal_cert_dir: "/etc/canal/certs" diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index de38d28ff..ed0d11f28 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -32,11 +32,6 @@ when: dns_mode != 'none' and inventory_hostname == groups['kube-master'][0] tags: dnsmasq -- include: tasks/calico-policy-controller.yml - when: ( enable_network_policy is defined and enable_network_policy == True ) or - ( kube_network_plugin == 'canal' ) - tags: [network, canal] - - name: Kubernetes Apps | Netchecker include: tasks/netchecker.yml when: deploy_netchecker diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml index f6df2626c..75860a0ff 100644 --- a/roles/kubernetes-apps/meta/main.yml +++ b/roles/kubernetes-apps/meta/main.yml @@ -1,9 +1,4 @@ dependencies: - - role: download - file: "{{ downloads.calico_policy }}" - when: ( enable_network_policy is defined and enable_network_policy == True ) or - ( kube_network_plugin == 'canal' ) - tags: [download, network, canal] - role: download file: "{{ downloads.netcheck_server }}" when: deploy_netchecker diff --git a/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml b/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml new file mode 100644 index 000000000..7a4db0ea8 --- /dev/null +++ b/roles/kubernetes-apps/policy_controller/calico/defaults/main.yml @@ -0,0 +1,9 @@ +# Limits for calico apps +calico_policy_controller_cpu_limit: 100m +calico_policy_controller_memory_limit: 256M +calico_policy_controller_cpu_requests: 30m +calico_policy_controller_memory_requests: 64M + +# SSL +calico_cert_dir: "/etc/calico/certs" +canal_cert_dir: "/etc/canal/certs" diff --git a/roles/kubernetes-apps/ansible/tasks/calico-policy-controller.yml b/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml similarity index 92% rename from roles/kubernetes-apps/ansible/tasks/calico-policy-controller.yml rename to roles/kubernetes-apps/policy_controller/calico/tasks/main.yml index c6a6bd94d..8b4271d6a 100644 --- a/roles/kubernetes-apps/ansible/tasks/calico-policy-controller.yml +++ b/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml @@ -1,14 +1,14 @@ ---- - set_fact: calico_cert_dir: "{{ canal_cert_dir }}" when: kube_network_plugin == 'canal' - tags: facts + tags: [facts, canal] - name: Write calico-policy-controller yaml template: src: calico-policy-controller.yml.j2 dest: "{{kube_config_dir}}/calico-policy-controller.yml" when: inventory_hostname == groups['kube-master'][0] + tags: canal - name: Start of Calico policy controller kube: @@ -18,3 +18,4 @@ namespace: "{{system_namespace}}" resource: "rs" when: inventory_hostname == groups['kube-master'][0] + tags: canal diff --git a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2 similarity index 96% rename from roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 rename to roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2 index b31ae0f43..322d3a37b 100644 --- a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 +++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2 @@ -45,7 +45,7 @@ spec: # changed so long as it is used in conjunction with # CONFIGURE_ETC_HOSTS="true". - name: K8S_API - value: "https://kubernetes.default:{{ kube_apiserver_port }}" + value: "https://kubernetes.default" # Configure /etc/hosts within the container to resolve # the kubernetes.default Service to the correct clusterIP # using the environment provided by the kubelet. diff --git a/roles/kubernetes-apps/policy_controller/meta/main.yml b/roles/kubernetes-apps/policy_controller/meta/main.yml new file mode 100644 index 000000000..e678a318c --- /dev/null +++ b/roles/kubernetes-apps/policy_controller/meta/main.yml @@ -0,0 +1,14 @@ +--- +dependencies: + - role: download + file: "{{ downloads.calico_policy }}" + when: enable_network_policy and + kube_network_plugin in ['calico', 'canal'] + tags: [download, canal, policy-controller] + - role: policy_controller/calico + when: kube_network_plugin == 'calico' and + enable_network_policy + tags: policy-controller + - role: policy_controller/calico + when: kube_network_plugin == 'canal' + tags: policy-controller diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 65a30929b..600ade340 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -64,7 +64,7 @@ spec: - --runtime-config={{ conf }} {% endfor %} {% endif %} -{% if enable_network_policy is defined and enable_network_policy == True %} +{% if enable_network_policy %} - --runtime-config=extensions/v1beta1/networkpolicies=true {% endif %} - --v={{ kube_log_level }} diff --git a/roles/network_plugin/calico/templates/cni-calico.conf.j2 b/roles/network_plugin/calico/templates/cni-calico.conf.j2 index f9427e69d..7cd3c902d 100644 --- a/roles/network_plugin/calico/templates/cni-calico.conf.j2 +++ b/roles/network_plugin/calico/templates/cni-calico.conf.j2 @@ -12,7 +12,7 @@ "ipam": { "type": "calico-ipam" }, -{% if enable_network_policy is defined and enable_network_policy == True %} +{% if enable_network_policy %} "policy": { "type": "k8s" },