From fd30131dc24f1d253f6d70ce97f4444614a3a1b3 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Mon, 6 Feb 2017 15:58:54 +0300 Subject: [PATCH] Revert "Drop linux capabilities and rework users/groups" --- docs/security.md | 31 ------------------ inventory/group_vars/all.yml | 13 ++------ roles/adduser/defaults/main.yml | 28 ++++------------ roles/adduser/handlers/main.yml | 4 --- roles/adduser/tasks/main.yml | 24 +------------- roles/adduser/templates/users.j2 | 15 --------- roles/adduser/vars/coreos.yml | 8 +++++ roles/adduser/vars/debian.yml | 15 +++++++++ roles/adduser/vars/redhat.yml | 15 +++++++++ roles/dnsmasq/defaults/main.yml | 13 -------- roles/dnsmasq/templates/dnsmasq-ds.yml | 4 --- roles/etcd/defaults/main.yml | 18 +---------- roles/etcd/files/make-ssl-etcd.sh | 3 -- roles/etcd/meta/main.yml | 2 +- roles/etcd/tasks/gen_certs.yml | 14 ++++---- roles/etcd/tasks/main.yml | 2 -- roles/etcd/tasks/pre_upgrade.yml | 5 --- roles/etcd/tasks/set_facts.yml | 17 ---------- roles/etcd/templates/etcd-docker.service.j2 | 4 --- roles/etcd/templates/etcd-rkt.service.j2 | 8 ----- .../kubernetes-apps/ansible/defaults/main.yml | 15 --------- .../templates/calico-policy-controller.yml.j2 | 6 ---- .../ansible/templates/netchecker-agent-ds.yml | 6 ---- .../templates/netchecker-agent-hostnet-ds.yml | 6 ---- .../templates/netchecker-server-pod.yml | 6 ---- roles/kubernetes/master/defaults/main.yml | 15 --------- roles/kubernetes/master/tasks/main.yml | 3 -- roles/kubernetes/master/tasks/set_facts.yml | 22 ------------- .../manifests/kube-apiserver.manifest.j2 | 8 ----- .../kube-controller-manager.manifest.j2 | 11 ------- .../manifests/kube-scheduler.manifest.j2 | 11 ------- roles/kubernetes/node/defaults/main.yml | 15 --------- roles/kubernetes/node/tasks/install.yml | 2 +- roles/kubernetes/node/tasks/main.yml | 3 -- roles/kubernetes/node/tasks/pre-upgrade.yml | 4 --- .../node/templates/kubelet.rkt.service.j2 | 3 +- roles/kubernetes/preinstall/meta/main.yml | 5 +-- roles/kubernetes/preinstall/tasks/main.yml | 14 +++----- .../kubernetes/preinstall/tasks/set_facts.yml | 3 ++ .../preinstall/tasks/set_uid_facts.yml | 32 ------------------- roles/kubernetes/secrets/files/make-ssl.sh | 3 -- roles/kubernetes/secrets/tasks/gen_certs.yml | 6 ++-- roles/kubernetes/secrets/tasks/main.yml | 7 ++-- roles/network_plugin/calico/defaults/main.yml | 17 ---------- roles/network_plugin/calico/rr/tasks/main.yml | 8 ++--- .../calico/rr/templates/calico-rr.service.j2 | 6 +--- roles/network_plugin/calico/tasks/main.yml | 7 ++-- roles/network_plugin/canal/tasks/main.yml | 7 ++-- 48 files changed, 81 insertions(+), 413 deletions(-) delete mode 100644 docs/security.md delete mode 100644 roles/adduser/handlers/main.yml delete mode 100644 roles/adduser/templates/users.j2 create mode 100644 roles/adduser/vars/coreos.yml create mode 100644 roles/adduser/vars/debian.yml create mode 100644 roles/adduser/vars/redhat.yml delete mode 100644 roles/etcd/tasks/set_facts.yml delete mode 100644 roles/kubernetes/master/tasks/set_facts.yml delete mode 100644 roles/kubernetes/node/tasks/pre-upgrade.yml delete mode 100644 roles/kubernetes/preinstall/tasks/set_uid_facts.yml diff --git a/docs/security.md b/docs/security.md deleted file mode 100644 index fd45a579e..000000000 --- a/docs/security.md +++ /dev/null @@ -1,31 +0,0 @@ -Users and groups -================ - -There are following users and groups defined by the addusers role: - -* Kube user, group from the ``kubelet_user`` and ``kubelet_group`` vars. -* Etcd user, group from the ``etcd_user`` and ``etcd_group`` vars. -* Network plugin user, group from the ``netplug_user`` and ``netplug_group`` vars. - -There are additional certificate access groups for kube and etcd users defined. -For example, kubelet and network plugins require read access to the -etcd certs and keys. This is defined via the corresponding ``etcd_cert_group`` -var. Members of that group (defaults to `kube` and `netplug` users) will read -etcd secret keys and certs. Same applies to the ``kube_cert_group`` -(defaults to `kube` user) members. You may want to share kube certs via that -group with bastion proxies or the like. - -Linux capabilites -================= - -Kargo allows to control dropped Linux capabilities for unprivileged docker -containers it configures for deployments. For examle, etcd or some networking -related systemd units or k8s workloads, like kubedns, dnsmasq or netchecker apps. - -Dropped capabilites are represented by the ``apps_drop_cap``, ``dnsmasq_drop_cap``, -``etcd_drop_cap``, ``calico_drop_cap`` vars. - -Be carefull changing defaults - different kube components and k8s apps might -expect specific capabilities to be present and can only run as root! Also note -that kublet, kube-proxy and network plugins require privileged mode and ignore -dropped capabilities. diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 3c0a5b9d7..e5047ca47 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -36,19 +36,10 @@ retry_stagger: 5 # Directory where python binary is installed # ansible_python_interpreter: "/opt/bin/python" -# This is the users/groups to own files and groups that the cert creation -# scripts chgrp the cert files to -kubelet_user: kube -kubelet_group: kube +# This is the group that the cert creation scripts chgrp the +# cert files to. Not really changable... kube_cert_group: kube-cert -netplug_user: netplug -netplug_group: netplug - -etcd_user: etcd -etcd_group: etcd -etcd_cert_group: etcd-cert - # Cluster Loglevel configuration kube_log_level: 2 diff --git a/roles/adduser/defaults/main.yml b/roles/adduser/defaults/main.yml index 3766b2c0d..b3a69229c 100644 --- a/roles/adduser/defaults/main.yml +++ b/roles/adduser/defaults/main.yml @@ -1,40 +1,24 @@ --- addusers: etcd: - name: "{{ etcd_user }}" + name: etcd comment: "Etcd user" - createhome: >- - {% if ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] %}no{% else %}yes{% endif %} + createhome: yes home: "/var/lib/etcd" system: yes - shell: /usr/sbin/nologin - group: "{{ etcd_group }}" - groups: "{{ etcd_cert_group }}" - type: >- - {% if ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] %}cloud-init{% endif %} + shell: /bin/nologin kube: - name: "{{ kubelet_user }}" + name: kube comment: "Kubernetes user" - shell: /usr/sbin/nologin + shell: /sbin/nologin system: yes - group: "{{ kubelet_group }}" - groups: "{{ etcd_cert_group }},{{ kube_cert_group }}" + group: "{{ kube_cert_group }}" createhome: no - netplug: - name: "{{ netplug_user }}" - comment: "Network plugin user" - createhome: no - system: yes - shell: /usr/sbin/nologin - group: "{{ netplug_group }}" - groups: "{{ etcd_cert_group }}" adduser: name: "{{ user.name }}" group: "{{ user.name|default(None) }}" - groups: "{{ user.groups|default(None) }}" comment: "{{ user.comment|default(None) }}" shell: "{{ user.shell|default(None) }}" system: "{{ user.system|default(None) }}" createhome: "{{ user.createhome|default(None) }}" - type: "{{ user.type|default(None) }}" diff --git a/roles/adduser/handlers/main.yml b/roles/adduser/handlers/main.yml deleted file mode 100644 index 60d821f18..000000000 --- a/roles/adduser/handlers/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: User | update users for cloud-init - command: /usr/bin/coreos-cloudinit --from-file /etc/{{ user.name }}_user_cloud_init_conf - when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] diff --git a/roles/adduser/tasks/main.yml b/roles/adduser/tasks/main.yml index 8176a5c71..394ff9294 100644 --- a/roles/adduser/tasks/main.yml +++ b/roles/adduser/tasks/main.yml @@ -1,35 +1,13 @@ --- -- name: User | Create Certificate Access Groups - group: name={{ item }} system=yes - with_items: "{{ user.groups.split(',') }}" - - name: User | Create User Group group: name={{user.group|default(user.name)}} system={{user.system|default(omit)}} -- name: User | Create cloud-init user - template: - dest: /etc/{{ user.name }}_user_cloud_init_conf - src: users.j2 - owner: root - mode: 0640 - notify: User | update users for cloud-init - when: "{{ user.type|default('standard') == 'cloud-init' }}" - -- meta: flush_handlers - -- name: User | Hack groups for existing cloud-init users CoreOS - command: /usr/sbin/usermod -aG {{ item }} {{ user.name }} - with_items: "{{ user.groups.split(',') }}" - when: "{{ ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] and user.type|default('standard') == 'cloud-init' }}" - - name: User | Create User user: comment: "{{user.comment|default(omit)}}" - createhome: "{{user.createhome|default(omit)}}" + createhome: "{{user.create_home|default(omit)}}" group: "{{user.group|default(user.name)}}" - groups: "{{user.groups|default(omit)}}" home: "{{user.home|default(omit)}}" shell: "{{user.shell|default(omit)}}" name: "{{user.name}}" system: "{{user.system|default(omit)}}" - when: "{{ user.type|default('standard') != 'cloud-init' }}" diff --git a/roles/adduser/templates/users.j2 b/roles/adduser/templates/users.j2 deleted file mode 100644 index 345049dc7..000000000 --- a/roles/adduser/templates/users.j2 +++ /dev/null @@ -1,15 +0,0 @@ -#cloud-config -users: -- name: {{ user.name }} - gecos: {{ user.comment }} - system: {{ user.system|bool }} - no-log-init: {{ user.system|bool }} - primary-group: {{ user.group }} - no-create-home: {{ not user.createhome|bool }} - homedir: {{ user.home }} - shell: {{ user.shell }} - groups: | - {% for g in user.groups.split(',') %} - - {{ g }} - {% endfor %} - # diff --git a/roles/adduser/vars/coreos.yml b/roles/adduser/vars/coreos.yml new file mode 100644 index 000000000..9fa93e45b --- /dev/null +++ b/roles/adduser/vars/coreos.yml @@ -0,0 +1,8 @@ +--- +addusers: + - name: kube + comment: "Kubernetes user" + shell: /sbin/nologin + system: yes + group: "{{ kube_cert_group }}" + createhome: no diff --git a/roles/adduser/vars/debian.yml b/roles/adduser/vars/debian.yml new file mode 100644 index 000000000..16b39f656 --- /dev/null +++ b/roles/adduser/vars/debian.yml @@ -0,0 +1,15 @@ +--- +addusers: + - name: etcd + comment: "Etcd user" + createhome: yes + home: "/var/lib/etcd" + system: yes + shell: /bin/nologin + + - name: kube + comment: "Kubernetes user" + shell: /sbin/nologin + system: yes + group: "{{ kube_cert_group }}" + createhome: no diff --git a/roles/adduser/vars/redhat.yml b/roles/adduser/vars/redhat.yml new file mode 100644 index 000000000..16b39f656 --- /dev/null +++ b/roles/adduser/vars/redhat.yml @@ -0,0 +1,15 @@ +--- +addusers: + - name: etcd + comment: "Etcd user" + createhome: yes + home: "/var/lib/etcd" + system: yes + shell: /bin/nologin + + - name: kube + comment: "Kubernetes user" + shell: /sbin/nologin + system: yes + group: "{{ kube_cert_group }}" + createhome: no diff --git a/roles/dnsmasq/defaults/main.yml b/roles/dnsmasq/defaults/main.yml index 874e636c8..d8ac8b34b 100644 --- a/roles/dnsmasq/defaults/main.yml +++ b/roles/dnsmasq/defaults/main.yml @@ -26,16 +26,3 @@ dns_cpu_limit: 100m dns_memory_limit: 170Mi dns_cpu_requests: 70m dns_memory_requests: 70Mi - -# Linux capabilities to be dropped for dnsmasq k8s app ran container engines -dnsmasq_drop_cap: - - chown - - dac_override - - fowner - - fsetid - - kill - - setpcap - - sys_chroot - - mknod - - audit_write - - setfcap diff --git a/roles/dnsmasq/templates/dnsmasq-ds.yml b/roles/dnsmasq/templates/dnsmasq-ds.yml index 97d809832..adcbbeacb 100644 --- a/roles/dnsmasq/templates/dnsmasq-ds.yml +++ b/roles/dnsmasq/templates/dnsmasq-ds.yml @@ -26,10 +26,6 @@ spec: capabilities: add: - NET_ADMIN - drop: -{% for c in dnsmasq_drop_cap %} - - {{ c.upper() }} -{% endfor %} imagePullPolicy: IfNotPresent resources: limits: diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index 14b41edeb..e733fe56d 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -3,26 +3,10 @@ etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/ etcd_config_dir: /etc/ssl/etcd etcd_cert_dir: "{{ etcd_config_dir }}/ssl" +etcd_cert_group: root etcd_script_dir: "{{ bin_dir }}/etcd-scripts" -# Linux capabilities to be dropped for container engines -etcd_drop_cap: - - chown - - dac_override - - fowner - - fsetid - - kill - - setgid - - setuid - - setpcap - - net_bind_service - - net_raw - - sys_chroot - - mknod - - audit_write - - setfcap - # Limits etcd_memory_limit: 512M etcd_cpu_limit: 300m diff --git a/roles/etcd/files/make-ssl-etcd.sh b/roles/etcd/files/make-ssl-etcd.sh index da76e3f55..458b39d9b 100755 --- a/roles/etcd/files/make-ssl-etcd.sh +++ b/roles/etcd/files/make-ssl-etcd.sh @@ -94,8 +94,5 @@ if [ -n "$HOSTS" ]; then done fi -# Grant the group read access -chmod g+r *.pem - # Install certs mv *.pem ${SSLDIR}/ diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml index fbf654981..addd81053 100644 --- a/roles/etcd/meta/main.yml +++ b/roles/etcd/meta/main.yml @@ -2,7 +2,7 @@ dependencies: - role: adduser user: "{{ addusers.etcd }}" - tags: bootstrap-os + when: not ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] - role: download file: "{{ downloads.etcd }}" tags: download diff --git a/roles/etcd/tasks/gen_certs.yml b/roles/etcd/tasks/gen_certs.yml index 49a01ac34..a4fd3a9d7 100644 --- a/roles/etcd/tasks/gen_certs.yml +++ b/roles/etcd/tasks/gen_certs.yml @@ -4,15 +4,14 @@ path={{ etcd_cert_dir }} group={{ etcd_cert_group }} state=directory - mode=0750 - owner={{ etcd_user }} + owner=root recurse=yes - name: "Gen_certs | create etcd script dir (on {{groups['etcd'][0]}})" file: path: "{{ etcd_script_dir }}" state: directory - owner: "{{ etcd_user }}" + owner: root run_once: yes delegate_to: "{{groups['etcd'][0]}}" @@ -21,8 +20,7 @@ path={{ etcd_cert_dir }} group={{ etcd_cert_group }} state=directory - mode=0750 - owner={{ etcd_user }} + owner=root recurse=yes run_once: yes delegate_to: "{{groups['etcd'][0]}}" @@ -126,12 +124,12 @@ path={{ etcd_cert_dir }} group={{ etcd_cert_group }} state=directory - owner={{ etcd_user }} + owner=kube recurse=yes tags: facts -- name: Gen_certs | set shared group permissions on keys - shell: chmod 0640 {{ etcd_cert_dir}}/*.pem +- name: Gen_certs | set permissions on keys + shell: chmod 0600 {{ etcd_cert_dir}}/*key.pem when: inventory_hostname in groups['etcd'] changed_when: false diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index c9a662d6c..394e5de64 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -1,8 +1,6 @@ --- - include: pre_upgrade.yml tags: etcd-pre-upgrade -- include: set_facts.yml - tags: [bootstrap-os, facts] - include: check_certs.yml tags: [etcd-secrets, facts] - include: gen_certs.yml diff --git a/roles/etcd/tasks/pre_upgrade.yml b/roles/etcd/tasks/pre_upgrade.yml index 30f307a03..eb17e9871 100644 --- a/roles/etcd/tasks/pre_upgrade.yml +++ b/roles/etcd/tasks/pre_upgrade.yml @@ -1,4 +1,3 @@ ---- - name: "Pre-upgrade | check for etcd-proxy unit file" stat: path: /etc/systemd/system/etcd-proxy.service @@ -50,7 +49,3 @@ awk -F"[: =]" '{print "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses | regex_replace('https','http') }} member update "$1" https:"$7":"$8}' | bash run_once: true when: 'etcd_member_list.rc == 0 and "http://" in etcd_member_list.stdout' - -- name: "Pre-upgrade | share access to etcd certs for its users" - shell: chmod g+r {{ etcd_cert_dir }}/*.pem - failed_when: false diff --git a/roles/etcd/tasks/set_facts.yml b/roles/etcd/tasks/set_facts.yml deleted file mode 100644 index 1d5f20462..000000000 --- a/roles/etcd/tasks/set_facts.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: Etcd | get etcd user ID - shell: /usr/bin/id -u {{ etcd_user }} || echo 0 - register: etcd_uid - -- name: Etcd | get etcd group ID - shell: /usr/bin/getent group {{ etcd_group }} | cut -d':' -f3 || echo 0 - register: etcd_gid - -- name: Etcd | get etcd cert group ID - shell: /usr/bin/getent group {{ etcd_cert_group }} | cut -d':' -f3 || echo 0 - register: etcd_cert_gid - -- set_fact: - etcd_user_id: "{{ etcd_uid.stdout }}" - etcd_group_id: "{{ etcd_gid.stdout }}" - etcd_cert_group_id: "{{ etcd_cert_gid.stdout }}" diff --git a/roles/etcd/templates/etcd-docker.service.j2 b/roles/etcd/templates/etcd-docker.service.j2 index 05640b146..223d2d842 100644 --- a/roles/etcd/templates/etcd-docker.service.j2 +++ b/roles/etcd/templates/etcd-docker.service.j2 @@ -14,12 +14,8 @@ ExecStart={{ docker_bin_dir }}/docker run --restart=on-failure:5 \ -v /etc/ssl/certs:/etc/ssl/certs:ro \ -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \ -v /var/lib/etcd:/var/lib/etcd:rw \ -{% for c in etcd_drop_cap %} ---cap-drop={{ c }} \ -{% endfor %} --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \ --name={{ etcd_member_name | default("etcd") }} \ --u {{ etcd_user_id }}:{{ etcd_group_id }} --group-add {{ etcd_cert_group_id }} \ {{ etcd_image_repo }}:{{ etcd_image_tag }} \ {% if etcd_after_v3 %} {{ etcd_container_bin_dir }}etcd diff --git a/roles/etcd/templates/etcd-rkt.service.j2 b/roles/etcd/templates/etcd-rkt.service.j2 index 72e98d21e..eb26bc473 100644 --- a/roles/etcd/templates/etcd-rkt.service.j2 +++ b/roles/etcd/templates/etcd-rkt.service.j2 @@ -8,9 +8,6 @@ Restart=on-failure RestartSec=10s TimeoutStartSec=0 LimitNOFILE=40000 -User=root -Group={{ etcd_group_id }} -SupplementaryGroups={{ etcd_cert_group_id }} ExecStart=/usr/bin/rkt run \ --uuid-file-save=/var/run/etcd.uuid \ @@ -23,11 +20,6 @@ ExecStart=/usr/bin/rkt run \ --set-env-file=/etc/etcd.env \ --stage1-from-dir=stage1-fly.aci \ {{ etcd_image_repo }}:{{ etcd_image_tag }} \ -{% for c in etcd_drop_cap %} ---caps-remove=CAP_{{ c.upper() }} \ -{% endfor %} ---memory={{ etcd_memory_limit }} --cpu={{ etcd_cpu_limit }} \ ---user={{ etcd_user_id }} --group={{ etcd_group_id }} \ --name={{ etcd_member_name | default("etcd") }} ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/etcd.uuid diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 923e9dc4a..14deb333d 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -51,18 +51,3 @@ netchecker_kubectl_memory_requests: 64M etcd_cert_dir: "/etc/ssl/etcd/ssl" calico_cert_dir: "/etc/calico/certs" canal_cert_dir: "/etc/canal/certs" - -# Linux capabilities to be dropped for k8s apps ran by container engines -apps_drop_cap: - - chown - - dac_override - - fowner - - fsetid - - kill - - setgid - - setuid - - setpcap - - sys_chroot - - mknod - - audit_write - - setfcap diff --git a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 index 3734aea96..06bb78b7c 100644 --- a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 @@ -25,12 +25,6 @@ spec: - name: calico-policy-controller image: {{ calico_policy_image_repo }}:{{ calico_policy_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} - securityContext: - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} resources: limits: cpu: {{ calico_policy_controller_cpu_limit }} diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml index 7ea05c66b..41900ab33 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml @@ -23,12 +23,6 @@ spec: - name: REPORT_INTERVAL value: '{{ agent_report_interval }}' imagePullPolicy: {{ k8s_image_pull_policy }} - securityContext: - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} resources: limits: cpu: {{ netchecker_agent_cpu_limit }} diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml index 9e246c56d..5a6a63f36 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml @@ -24,12 +24,6 @@ spec: - name: REPORT_INTERVAL value: '{{ agent_report_interval }}' imagePullPolicy: {{ k8s_image_pull_policy }} - securityContext: - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} resources: limits: cpu: {{ netchecker_agent_cpu_limit }} diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml b/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml index a52c5d4a3..c1d8ddb9f 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml +++ b/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml @@ -33,9 +33,3 @@ spec: memory: {{ netchecker_kubectl_memory_requests }} args: - proxy - securityContext: - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 4e29a51fd..874925adf 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -13,21 +13,6 @@ kube_apiserver_node_port_range: "30000-32767" etcd_config_dir: /etc/ssl/etcd etcd_cert_dir: "{{ etcd_config_dir }}/ssl" -# Linux capabilities to be dropped for k8s apps ran by container engines -apps_drop_cap: - - chown - - dac_override - - fowner - - fsetid - - kill - - setgid - - setuid - - setpcap - - sys_chroot - - mknod - - audit_write - - setfcap - # Limits for kube components kube_controller_memory_limit: 512M kube_controller_cpu_limit: 250m diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index b0bab4cdc..a622594a1 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -2,9 +2,6 @@ - include: pre-upgrade.yml tags: k8s-pre-upgrade -- include: set_facts.yml - tags: facts - - name: Copy kubectl from hyperkube container command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp /hyperkube /systembindir/kubectl" register: kube_task_result diff --git a/roles/kubernetes/master/tasks/set_facts.yml b/roles/kubernetes/master/tasks/set_facts.yml deleted file mode 100644 index d5c3250b3..000000000 --- a/roles/kubernetes/master/tasks/set_facts.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- name: Master | get kube user ID - shell: /usr/bin/id -u {{ kubelet_user }} || echo 0 - register: kube_uid - -- name: Master | get kube group ID - shell: /usr/bin/getent group {{ kubelet_group }} | cut -d':' -f3 || echo 0 - register: kube_gid - -- name: Master | get kube cert group ID - shell: /usr/bin/getent group {{ kube_cert_group }} | cut -d':' -f3 || echo 0 - register: kube_cert_gid - -- name: Master | get etcd cert group ID - shell: /usr/bin/getent group {{ etcd_cert_group }} | cut -d':' -f3 || echo 0 - register: etcd_cert_gid - -- set_fact: - kubelet_user_id: "{{ kube_uid.stdout }}" - kubelet_group_id: "{{ kube_gid.stdout }}" - kube_cert_group_id: "{{ kube_cert_gid.stdout }}" - etcd_cert_group_id: "{{ etcd_cert_gid.stdout }}" diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index cab219f60..c05030697 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -12,14 +12,6 @@ spec: - name: kube-apiserver image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} - securityContext: - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} - add: - - DAC_OVERRIDE resources: limits: cpu: {{ kube_apiserver_cpu_limit }} diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 6db17f30a..49dd05ba8 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -11,17 +11,6 @@ spec: - name: kube-controller-manager image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} - securityContext: - runAsUser: {{ kubelet_user_id }} - fsGroup: {{ kubelet_group_id }} - supplementalGroups: - - {{ kube_cert_group_id }} - - {{ etcd_cert_group_id }} - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} resources: limits: cpu: {{ kube_controller_cpu_limit }} diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index ded01dc79..781e38d7b 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -11,17 +11,6 @@ spec: - name: kube-scheduler image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} - securityContext: - runAsUser: {{ kubelet_user_id }} - fsGroup: {{ kubelet_group_id }} - supplementalGroups: - - {{ kube_cert_group_id }} - - {{ etcd_cert_group_id }} - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} resources: limits: cpu: {{ kube_scheduler_cpu_limit }} diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 777a751fa..a74e52b77 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -29,18 +29,3 @@ nginx_image_repo: nginx nginx_image_tag: 1.11.4-alpine etcd_config_dir: /etc/ssl/etcd - -# Linux capabilities to be dropped for container engines -apps_drop_cap: - - chown - - dac_override - - fowner - - fsetid - - kill - - setgid - - setuid - - setpcap - - sys_chroot - - mknod - - audit_write - - setfcap diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index 52a32ccc3..bfe4a8cc8 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -26,6 +26,6 @@ notify: restart kubelet - name: install | Install kubelet launch script - template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner={{ kubelet_user }} mode=0755 backup=yes + template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes notify: restart kubelet when: kubelet_deployment_type == "docker" diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 29e9ef4e0..3e0c095e1 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -4,9 +4,6 @@ {%- if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] -%}true{%- else -%}false{%- endif -%} tags: facts -- include: pre-upgrade.yml - tags: k8s-pre-upgrade - - include: install.yml tags: kubelet diff --git a/roles/kubernetes/node/tasks/pre-upgrade.yml b/roles/kubernetes/node/tasks/pre-upgrade.yml deleted file mode 100644 index 9a61170d7..000000000 --- a/roles/kubernetes/node/tasks/pre-upgrade.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: "Pre-upgrade | share access to kube certs for its users" - shell: chmod g+r {{ kube_cert_dir }}/*.pem - failed_when: false diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 index 2537d9ffd..12ce01c75 100644 --- a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 @@ -29,7 +29,7 @@ ExecStart=/usr/bin/rkt run \ --volume run,kind=host,source=/run,readOnly=false \ --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ --volume var-lib-docker,kind=host,source={{ docker_daemon_graph }},readOnly=false \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false \ --volume var-log,kind=host,source=/var/log \ --mount volume=dns,target=/etc/resolv.conf \ --mount volume=etc-cni,target=/etc/cni \ @@ -44,7 +44,6 @@ ExecStart=/usr/bin/rkt run \ --mount volume=var-log,target=/var/log \ --stage1-from-dir=stage1-fly.aci \ {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} \ - --memory={{ kubelet_memory_limit }} --cpu={{ kubelet_cpu_limit }} \ --uuid-file-save=/var/run/kubelet.uuid \ --debug --exec=/kubelet -- \ $KUBE_LOGTOSTDERR \ diff --git a/roles/kubernetes/preinstall/meta/main.yml b/roles/kubernetes/preinstall/meta/main.yml index 34785ed88..cf440f5e2 100644 --- a/roles/kubernetes/preinstall/meta/main.yml +++ b/roles/kubernetes/preinstall/meta/main.yml @@ -2,7 +2,4 @@ dependencies: - role: adduser user: "{{ addusers.kube }}" - tags: [bootstrap-os, kubelet] - - role: adduser - user: "{{ addusers.netplug }}" - tags: [bootstrap-os, network] + tags: kubelet diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 7a8760f95..e7955ac76 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -23,12 +23,6 @@ - include: set_facts.yml tags: facts -- include: set_resolv_facts.yml - tags: [bootstrap-os, resolvconf, facts] - -- include: set_uid_facts.yml - tags: [bootstrap-os, facts] - - name: gather os specific variables include_vars: "{{ item }}" with_first_found: @@ -48,7 +42,7 @@ file: path: "{{ kube_config_dir }}" state: directory - owner: "{{ kubelet_user }}" + owner: kube when: "{{ inventory_hostname in groups['k8s-cluster'] }}" tags: [kubelet, k8s-secrets, kube-controller-manager, kube-apiserver, bootstrap-os, apps, network, master, node] @@ -56,7 +50,7 @@ file: path: "{{ kube_script_dir }}" state: directory - owner: "{{ kubelet_user }}" + owner: kube when: "{{ inventory_hostname in groups['k8s-cluster'] }}" tags: [k8s-secrets, bootstrap-os] @@ -64,7 +58,7 @@ file: path: "{{ kube_manifest_dir }}" state: directory - owner: "{{ kubelet_user }}" + owner: kube when: "{{ inventory_hostname in groups['k8s-cluster'] }}" tags: [kubelet, bootstrap-os, master, node] @@ -86,7 +80,7 @@ file: path: "{{ item }}" state: directory - owner: "{{ kubelet_user }}" + owner: kube with_items: - "/etc/cni/net.d" - "/opt/cni/bin" diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index d2fad6e3d..456467a97 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -51,3 +51,6 @@ etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}" - set_fact: peer_with_calico_rr: "{{ 'calico-rr' in groups and groups['calico-rr']|length > 0 }}" + +- include: set_resolv_facts.yml + tags: [bootstrap-os, resolvconf, facts] diff --git a/roles/kubernetes/preinstall/tasks/set_uid_facts.yml b/roles/kubernetes/preinstall/tasks/set_uid_facts.yml deleted file mode 100644 index 13a36b5db..000000000 --- a/roles/kubernetes/preinstall/tasks/set_uid_facts.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -- name: Preinstall | get kube user ID - shell: /usr/bin/id -u {{ kubelet_user }} || echo 0 - register: kube_uid - -- name: Preinstall | get kube group ID - shell: /usr/bin/id -g {{ kubelet_group }} || echo 0 - register: kube_gid - -- name: Preinstall | get kube cert group ID - shell: /usr/bin/id -g {{ kube_cert_group }} || echo 0 - register: kube_cert_gid - -- name: Preinstall | get etcd cert group ID - shell: /usr/bin/id -g {{ etcd_cert_group }} || echo 0 - register: etcd_cert_gid - -- name: Preinstall | get netplug user ID - shell: /usr/bin/id -u {{ netplug_user }} || echo 0 - register: netplug_uid - -- name: Preinstall | get netplug group ID - shell: /usr/bin/getent group {{ netplug_group }} | cut -d':' -f3 || echo 0 - register: netplug_gid - -- set_fact: - kubelet_user_id: "{{ kube_uid.stdout }}" - kubelet_group_id: "{{ kube_gid.stdout }}" - kube_cert_group_id: "{{ kube_cert_gid.stdout }}" - etcd_cert_group_id: "{{ etcd_cert_gid.stdout }}" - netplug_user_id: "{{ netplug_uid.stdout }}" - netplug_group_id: "{{ netplug_gid.stdout }}" diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh index cc0b49b95..dca9d9ab9 100755 --- a/roles/kubernetes/secrets/files/make-ssl.sh +++ b/roles/kubernetes/secrets/files/make-ssl.sh @@ -101,8 +101,5 @@ if [ -n "$HOSTS" ]; then done fi -# Grant the group read access -chmod g+r *.pem - # Install certs mv *.pem ${SSLDIR}/ diff --git a/roles/kubernetes/secrets/tasks/gen_certs.yml b/roles/kubernetes/secrets/tasks/gen_certs.yml index d56e487e3..484afff63 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs.yml @@ -140,11 +140,11 @@ file: path={{ kube_cert_dir }} group={{ kube_cert_group }} - owner={{ kubelet_user }} + owner=kube recurse=yes -- name: Gen_certs | set shared group permissions on keys - shell: chmod 0640 {{ kube_cert_dir}}/*.pem +- name: Gen_certs | set permissions on keys + shell: chmod 0600 {{ kube_cert_dir}}/*key.pem when: inventory_hostname in groups['kube-master'] changed_when: false diff --git a/roles/kubernetes/secrets/tasks/main.yml b/roles/kubernetes/secrets/tasks/main.yml index 81eb26743..4d25a94af 100644 --- a/roles/kubernetes/secrets/tasks/main.yml +++ b/roles/kubernetes/secrets/tasks/main.yml @@ -9,7 +9,6 @@ path={{ kube_cert_dir }} state=directory mode=o-rwx - owner={{ kubelet_user }} group={{ kube_cert_group }} - name: Make sure the tokens directory exits @@ -17,16 +16,14 @@ path={{ kube_token_dir }} state=directory mode=o-rwx - owner={{ kubelet_user }} - group={{ kubelet_group }} + group={{ kube_cert_group }} - name: Make sure the users directory exits file: path={{ kube_users_dir }} state=directory mode=o-rwx - owner={{ kubelet_user }} - group={{ kubelet_group }} + group={{ kube_cert_group }} - name: Populate users for basic auth in API lineinfile: diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml index 6cd5c2213..7681abc5c 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico/defaults/main.yml @@ -20,23 +20,6 @@ global_as_num: "64512" # defaults. The value should be a number, not a string. # calico_mtu: 1500 -# Linux capabilities to be dropped for container engines -calico_drop_cap: - - chown - - dac_override - - fowner - - fsetid - - kill - - setgid - - setuid - - setpcap - - net_bind_service - - net_raw - - sys_chroot - - mknod - - audit_write - - setfcap - # Limits for apps calico_node_memory_limit: 500M calico_node_cpu_limit: 300m diff --git a/roles/network_plugin/calico/rr/tasks/main.yml b/roles/network_plugin/calico/rr/tasks/main.yml index 1cbbb43f8..efe4616d2 100644 --- a/roles/network_plugin/calico/rr/tasks/main.yml +++ b/roles/network_plugin/calico/rr/tasks/main.yml @@ -12,8 +12,8 @@ dest: "{{ calico_cert_dir }}" state: directory mode: 0750 - owner: "{{ netplug_user }}" - group: "{{ netplug_group }}" + owner: root + group: root - name: Calico-rr | Link etcd certificates for calico-node file: @@ -31,8 +31,8 @@ path: /var/log/calico-rr state: directory mode: 0755 - owner: "{{ netplug_user }}" - group: "{{ netplug_group }}" + owner: root + group: root - name: Calico-rr | Write calico-rr.env for systemd init file template: src=calico-rr.env.j2 dest=/etc/calico/calico-rr.env diff --git a/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 b/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 index 6d4b344d3..f6da04a4d 100644 --- a/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 +++ b/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 @@ -6,7 +6,7 @@ Requires=docker.service [Service] EnvironmentFile=/etc/calico/calico-rr.env ExecStartPre=-{{ docker_bin_dir }}/docker rm -f calico-rr -ExecStart={{ docker_bin_dir }}/docker run --net=host \ +ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \ --name=calico-rr \ -e IP=${IP} \ -e IP6=${IP6} \ @@ -16,10 +16,6 @@ ExecStart={{ docker_bin_dir }}/docker run --net=host \ -e ETCD_KEY_FILE=${ETCD_KEY_FILE} \ -v /var/log/calico-rr:/var/log/calico \ -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \ -{% for c in calico_drop_cap %} - --cap-drop={{ c }} \ -{% endfor %} - -u {{ netplug_user_id }}:{{ netplug_group_id }} --group-add {{ etcd_cert_group }} \ --memory={{ calico_rr_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_rr_cpu_limit|regex_replace('m', '') }} \ {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }} diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index e88f7f552..dc9a96d19 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -9,16 +9,15 @@ template: src: "cni-calico.conf.j2" dest: "/etc/cni/net.d/10-calico.conf" - owner: "{{ kubelet_user }}" - group: "{{ kubelet_group }}" + owner: kube - name: Calico | Create calico certs directory file: dest: "{{ calico_cert_dir }}" state: directory mode: 0750 - owner: "{{ netplug_user }}" - group: "{{ netplug_group }}" + owner: root + group: root - name: Calico | Link etcd certificates for calico-node file: diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml index a6dedeec4..7ccbcdf2e 100644 --- a/roles/network_plugin/canal/tasks/main.yml +++ b/roles/network_plugin/canal/tasks/main.yml @@ -3,16 +3,15 @@ template: src: "cni-canal.conf.j2" dest: "/etc/cni/net.d/10-canal.conf" - owner: "{{ kubelet_user }}" - group: "{{ kubelet_group }}" + owner: kube - name: Canal | Create canal certs directory file: dest: "{{ canal_cert_dir }}" state: directory mode: 0750 - owner: "{{ netplug_user }}" - group: "{{ netplug_group }}" + owner: root + group: root - name: Canal | Link etcd certificates for canal-node file: