diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 000000000..fd45a579e --- /dev/null +++ b/docs/security.md @@ -0,0 +1,31 @@ +Users and groups +================ + +There are following users and groups defined by the addusers role: + +* Kube user, group from the ``kubelet_user`` and ``kubelet_group`` vars. +* Etcd user, group from the ``etcd_user`` and ``etcd_group`` vars. +* Network plugin user, group from the ``netplug_user`` and ``netplug_group`` vars. + +There are additional certificate access groups for kube and etcd users defined. +For example, kubelet and network plugins require read access to the +etcd certs and keys. This is defined via the corresponding ``etcd_cert_group`` +var. Members of that group (defaults to `kube` and `netplug` users) will read +etcd secret keys and certs. Same applies to the ``kube_cert_group`` +(defaults to `kube` user) members. You may want to share kube certs via that +group with bastion proxies or the like. + +Linux capabilites +================= + +Kargo allows to control dropped Linux capabilities for unprivileged docker +containers it configures for deployments. For examle, etcd or some networking +related systemd units or k8s workloads, like kubedns, dnsmasq or netchecker apps. + +Dropped capabilites are represented by the ``apps_drop_cap``, ``dnsmasq_drop_cap``, +``etcd_drop_cap``, ``calico_drop_cap`` vars. + +Be carefull changing defaults - different kube components and k8s apps might +expect specific capabilities to be present and can only run as root! Also note +that kublet, kube-proxy and network plugins require privileged mode and ignore +dropped capabilities. diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 0cee9c329..0403a4155 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -36,10 +36,19 @@ retry_stagger: 5 # Directory where python binary is installed # ansible_python_interpreter: "/opt/bin/python" -# This is the group that the cert creation scripts chgrp the -# cert files to. Not really changable... +# This is the users/groups to own files and groups that the cert creation +# scripts chgrp the cert files to +kubelet_user: kube +kubelet_group: kube kube_cert_group: kube-cert +netplug_user: netplug +netplug_group: netplug + +etcd_user: etcd +etcd_group: etcd +etcd_cert_group: etcd-cert + # Cluster Loglevel configuration kube_log_level: 2 diff --git a/roles/adduser/defaults/main.yml b/roles/adduser/defaults/main.yml index b3a69229c..3766b2c0d 100644 --- a/roles/adduser/defaults/main.yml +++ b/roles/adduser/defaults/main.yml @@ -1,24 +1,40 @@ --- addusers: etcd: - name: etcd + name: "{{ etcd_user }}" comment: "Etcd user" - createhome: yes + createhome: >- + {% if ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] %}no{% else %}yes{% endif %} home: "/var/lib/etcd" system: yes - shell: /bin/nologin + shell: /usr/sbin/nologin + group: "{{ etcd_group }}" + groups: "{{ etcd_cert_group }}" + type: >- + {% if ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] %}cloud-init{% endif %} kube: - name: kube + name: "{{ kubelet_user }}" comment: "Kubernetes user" - shell: /sbin/nologin + shell: /usr/sbin/nologin system: yes - group: "{{ kube_cert_group }}" + group: "{{ kubelet_group }}" + groups: "{{ etcd_cert_group }},{{ kube_cert_group }}" createhome: no + netplug: + name: "{{ netplug_user }}" + comment: "Network plugin user" + createhome: no + system: yes + shell: /usr/sbin/nologin + group: "{{ netplug_group }}" + groups: "{{ etcd_cert_group }}" adduser: name: "{{ user.name }}" group: "{{ user.name|default(None) }}" + groups: "{{ user.groups|default(None) }}" comment: "{{ user.comment|default(None) }}" shell: "{{ user.shell|default(None) }}" system: "{{ user.system|default(None) }}" createhome: "{{ user.createhome|default(None) }}" + type: "{{ user.type|default(None) }}" diff --git a/roles/adduser/handlers/main.yml b/roles/adduser/handlers/main.yml new file mode 100644 index 000000000..60d821f18 --- /dev/null +++ b/roles/adduser/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: User | update users for cloud-init + command: /usr/bin/coreos-cloudinit --from-file /etc/{{ user.name }}_user_cloud_init_conf + when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] diff --git a/roles/adduser/tasks/main.yml b/roles/adduser/tasks/main.yml index 394ff9294..8176a5c71 100644 --- a/roles/adduser/tasks/main.yml +++ b/roles/adduser/tasks/main.yml @@ -1,13 +1,35 @@ --- +- name: User | Create Certificate Access Groups + group: name={{ item }} system=yes + with_items: "{{ user.groups.split(',') }}" + - name: User | Create User Group group: name={{user.group|default(user.name)}} system={{user.system|default(omit)}} +- name: User | Create cloud-init user + template: + dest: /etc/{{ user.name }}_user_cloud_init_conf + src: users.j2 + owner: root + mode: 0640 + notify: User | update users for cloud-init + when: "{{ user.type|default('standard') == 'cloud-init' }}" + +- meta: flush_handlers + +- name: User | Hack groups for existing cloud-init users CoreOS + command: /usr/sbin/usermod -aG {{ item }} {{ user.name }} + with_items: "{{ user.groups.split(',') }}" + when: "{{ ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] and user.type|default('standard') == 'cloud-init' }}" + - name: User | Create User user: comment: "{{user.comment|default(omit)}}" - createhome: "{{user.create_home|default(omit)}}" + createhome: "{{user.createhome|default(omit)}}" group: "{{user.group|default(user.name)}}" + groups: "{{user.groups|default(omit)}}" home: "{{user.home|default(omit)}}" shell: "{{user.shell|default(omit)}}" name: "{{user.name}}" system: "{{user.system|default(omit)}}" + when: "{{ user.type|default('standard') != 'cloud-init' }}" diff --git a/roles/adduser/templates/users.j2 b/roles/adduser/templates/users.j2 new file mode 100644 index 000000000..345049dc7 --- /dev/null +++ b/roles/adduser/templates/users.j2 @@ -0,0 +1,15 @@ +#cloud-config +users: +- name: {{ user.name }} + gecos: {{ user.comment }} + system: {{ user.system|bool }} + no-log-init: {{ user.system|bool }} + primary-group: {{ user.group }} + no-create-home: {{ not user.createhome|bool }} + homedir: {{ user.home }} + shell: {{ user.shell }} + groups: | + {% for g in user.groups.split(',') %} + - {{ g }} + {% endfor %} + # diff --git a/roles/adduser/vars/coreos.yml b/roles/adduser/vars/coreos.yml deleted file mode 100644 index 9fa93e45b..000000000 --- a/roles/adduser/vars/coreos.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -addusers: - - name: kube - comment: "Kubernetes user" - shell: /sbin/nologin - system: yes - group: "{{ kube_cert_group }}" - createhome: no diff --git a/roles/adduser/vars/debian.yml b/roles/adduser/vars/debian.yml deleted file mode 100644 index 16b39f656..000000000 --- a/roles/adduser/vars/debian.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -addusers: - - name: etcd - comment: "Etcd user" - createhome: yes - home: "/var/lib/etcd" - system: yes - shell: /bin/nologin - - - name: kube - comment: "Kubernetes user" - shell: /sbin/nologin - system: yes - group: "{{ kube_cert_group }}" - createhome: no diff --git a/roles/adduser/vars/redhat.yml b/roles/adduser/vars/redhat.yml deleted file mode 100644 index 16b39f656..000000000 --- a/roles/adduser/vars/redhat.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -addusers: - - name: etcd - comment: "Etcd user" - createhome: yes - home: "/var/lib/etcd" - system: yes - shell: /bin/nologin - - - name: kube - comment: "Kubernetes user" - shell: /sbin/nologin - system: yes - group: "{{ kube_cert_group }}" - createhome: no diff --git a/roles/dnsmasq/defaults/main.yml b/roles/dnsmasq/defaults/main.yml index d8ac8b34b..874e636c8 100644 --- a/roles/dnsmasq/defaults/main.yml +++ b/roles/dnsmasq/defaults/main.yml @@ -26,3 +26,16 @@ dns_cpu_limit: 100m dns_memory_limit: 170Mi dns_cpu_requests: 70m dns_memory_requests: 70Mi + +# Linux capabilities to be dropped for dnsmasq k8s app ran container engines +dnsmasq_drop_cap: + - chown + - dac_override + - fowner + - fsetid + - kill + - setpcap + - sys_chroot + - mknod + - audit_write + - setfcap diff --git a/roles/dnsmasq/templates/dnsmasq-ds.yml b/roles/dnsmasq/templates/dnsmasq-ds.yml index 08ff70bff..1877c0001 100644 --- a/roles/dnsmasq/templates/dnsmasq-ds.yml +++ b/roles/dnsmasq/templates/dnsmasq-ds.yml @@ -26,6 +26,10 @@ spec: capabilities: add: - NET_ADMIN + drop: +{% for c in dnsmasq_drop_cap %} + - {{ c.upper() }} +{% endfor %} imagePullPolicy: IfNotPresent resources: limits: diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index e733fe56d..14b41edeb 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -3,10 +3,26 @@ etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/ etcd_config_dir: /etc/ssl/etcd etcd_cert_dir: "{{ etcd_config_dir }}/ssl" -etcd_cert_group: root etcd_script_dir: "{{ bin_dir }}/etcd-scripts" +# Linux capabilities to be dropped for container engines +etcd_drop_cap: + - chown + - dac_override + - fowner + - fsetid + - kill + - setgid + - setuid + - setpcap + - net_bind_service + - net_raw + - sys_chroot + - mknod + - audit_write + - setfcap + # Limits etcd_memory_limit: 512M etcd_cpu_limit: 300m diff --git a/roles/etcd/files/make-ssl-etcd.sh b/roles/etcd/files/make-ssl-etcd.sh index 458b39d9b..da76e3f55 100755 --- a/roles/etcd/files/make-ssl-etcd.sh +++ b/roles/etcd/files/make-ssl-etcd.sh @@ -94,5 +94,8 @@ if [ -n "$HOSTS" ]; then done fi +# Grant the group read access +chmod g+r *.pem + # Install certs mv *.pem ${SSLDIR}/ diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml index addd81053..fbf654981 100644 --- a/roles/etcd/meta/main.yml +++ b/roles/etcd/meta/main.yml @@ -2,7 +2,7 @@ dependencies: - role: adduser user: "{{ addusers.etcd }}" - when: not ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] + tags: bootstrap-os - role: download file: "{{ downloads.etcd }}" tags: download diff --git a/roles/etcd/tasks/gen_certs.yml b/roles/etcd/tasks/gen_certs.yml index a4fd3a9d7..49a01ac34 100644 --- a/roles/etcd/tasks/gen_certs.yml +++ b/roles/etcd/tasks/gen_certs.yml @@ -4,14 +4,15 @@ path={{ etcd_cert_dir }} group={{ etcd_cert_group }} state=directory - owner=root + mode=0750 + owner={{ etcd_user }} recurse=yes - name: "Gen_certs | create etcd script dir (on {{groups['etcd'][0]}})" file: path: "{{ etcd_script_dir }}" state: directory - owner: root + owner: "{{ etcd_user }}" run_once: yes delegate_to: "{{groups['etcd'][0]}}" @@ -20,7 +21,8 @@ path={{ etcd_cert_dir }} group={{ etcd_cert_group }} state=directory - owner=root + mode=0750 + owner={{ etcd_user }} recurse=yes run_once: yes delegate_to: "{{groups['etcd'][0]}}" @@ -124,12 +126,12 @@ path={{ etcd_cert_dir }} group={{ etcd_cert_group }} state=directory - owner=kube + owner={{ etcd_user }} recurse=yes tags: facts -- name: Gen_certs | set permissions on keys - shell: chmod 0600 {{ etcd_cert_dir}}/*key.pem +- name: Gen_certs | set shared group permissions on keys + shell: chmod 0640 {{ etcd_cert_dir}}/*.pem when: inventory_hostname in groups['etcd'] changed_when: false diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index 394e5de64..c9a662d6c 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -1,6 +1,8 @@ --- - include: pre_upgrade.yml tags: etcd-pre-upgrade +- include: set_facts.yml + tags: [bootstrap-os, facts] - include: check_certs.yml tags: [etcd-secrets, facts] - include: gen_certs.yml diff --git a/roles/etcd/tasks/pre_upgrade.yml b/roles/etcd/tasks/pre_upgrade.yml index eb17e9871..30f307a03 100644 --- a/roles/etcd/tasks/pre_upgrade.yml +++ b/roles/etcd/tasks/pre_upgrade.yml @@ -1,3 +1,4 @@ +--- - name: "Pre-upgrade | check for etcd-proxy unit file" stat: path: /etc/systemd/system/etcd-proxy.service @@ -49,3 +50,7 @@ awk -F"[: =]" '{print "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses | regex_replace('https','http') }} member update "$1" https:"$7":"$8}' | bash run_once: true when: 'etcd_member_list.rc == 0 and "http://" in etcd_member_list.stdout' + +- name: "Pre-upgrade | share access to etcd certs for its users" + shell: chmod g+r {{ etcd_cert_dir }}/*.pem + failed_when: false diff --git a/roles/etcd/tasks/set_facts.yml b/roles/etcd/tasks/set_facts.yml new file mode 100644 index 000000000..1d5f20462 --- /dev/null +++ b/roles/etcd/tasks/set_facts.yml @@ -0,0 +1,17 @@ +--- +- name: Etcd | get etcd user ID + shell: /usr/bin/id -u {{ etcd_user }} || echo 0 + register: etcd_uid + +- name: Etcd | get etcd group ID + shell: /usr/bin/getent group {{ etcd_group }} | cut -d':' -f3 || echo 0 + register: etcd_gid + +- name: Etcd | get etcd cert group ID + shell: /usr/bin/getent group {{ etcd_cert_group }} | cut -d':' -f3 || echo 0 + register: etcd_cert_gid + +- set_fact: + etcd_user_id: "{{ etcd_uid.stdout }}" + etcd_group_id: "{{ etcd_gid.stdout }}" + etcd_cert_group_id: "{{ etcd_cert_gid.stdout }}" diff --git a/roles/etcd/templates/etcd-docker.service.j2 b/roles/etcd/templates/etcd-docker.service.j2 index 223d2d842..05640b146 100644 --- a/roles/etcd/templates/etcd-docker.service.j2 +++ b/roles/etcd/templates/etcd-docker.service.j2 @@ -14,8 +14,12 @@ ExecStart={{ docker_bin_dir }}/docker run --restart=on-failure:5 \ -v /etc/ssl/certs:/etc/ssl/certs:ro \ -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \ -v /var/lib/etcd:/var/lib/etcd:rw \ +{% for c in etcd_drop_cap %} +--cap-drop={{ c }} \ +{% endfor %} --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \ --name={{ etcd_member_name | default("etcd") }} \ +-u {{ etcd_user_id }}:{{ etcd_group_id }} --group-add {{ etcd_cert_group_id }} \ {{ etcd_image_repo }}:{{ etcd_image_tag }} \ {% if etcd_after_v3 %} {{ etcd_container_bin_dir }}etcd diff --git a/roles/etcd/templates/etcd-rkt.service.j2 b/roles/etcd/templates/etcd-rkt.service.j2 index eb26bc473..72e98d21e 100644 --- a/roles/etcd/templates/etcd-rkt.service.j2 +++ b/roles/etcd/templates/etcd-rkt.service.j2 @@ -8,6 +8,9 @@ Restart=on-failure RestartSec=10s TimeoutStartSec=0 LimitNOFILE=40000 +User=root +Group={{ etcd_group_id }} +SupplementaryGroups={{ etcd_cert_group_id }} ExecStart=/usr/bin/rkt run \ --uuid-file-save=/var/run/etcd.uuid \ @@ -20,6 +23,11 @@ ExecStart=/usr/bin/rkt run \ --set-env-file=/etc/etcd.env \ --stage1-from-dir=stage1-fly.aci \ {{ etcd_image_repo }}:{{ etcd_image_tag }} \ +{% for c in etcd_drop_cap %} +--caps-remove=CAP_{{ c.upper() }} \ +{% endfor %} +--memory={{ etcd_memory_limit }} --cpu={{ etcd_cpu_limit }} \ +--user={{ etcd_user_id }} --group={{ etcd_group_id }} \ --name={{ etcd_member_name | default("etcd") }} ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/etcd.uuid diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 14deb333d..923e9dc4a 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -51,3 +51,18 @@ netchecker_kubectl_memory_requests: 64M etcd_cert_dir: "/etc/ssl/etcd/ssl" calico_cert_dir: "/etc/calico/certs" canal_cert_dir: "/etc/canal/certs" + +# Linux capabilities to be dropped for k8s apps ran by container engines +apps_drop_cap: + - chown + - dac_override + - fowner + - fsetid + - kill + - setgid + - setuid + - setpcap + - sys_chroot + - mknod + - audit_write + - setfcap diff --git a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 index 06bb78b7c..3734aea96 100644 --- a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 @@ -25,6 +25,12 @@ spec: - name: calico-policy-controller image: {{ calico_policy_image_repo }}:{{ calico_policy_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} + securityContext: + capabilities: + drop: +{% for c in apps_drop_cap %} + - {{ c.upper() }} +{% endfor %} resources: limits: cpu: {{ calico_policy_controller_cpu_limit }} diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml index 41900ab33..7ea05c66b 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml @@ -23,6 +23,12 @@ spec: - name: REPORT_INTERVAL value: '{{ agent_report_interval }}' imagePullPolicy: {{ k8s_image_pull_policy }} + securityContext: + capabilities: + drop: +{% for c in apps_drop_cap %} + - {{ c.upper() }} +{% endfor %} resources: limits: cpu: {{ netchecker_agent_cpu_limit }} diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml index 5a6a63f36..9e246c56d 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml @@ -24,6 +24,12 @@ spec: - name: REPORT_INTERVAL value: '{{ agent_report_interval }}' imagePullPolicy: {{ k8s_image_pull_policy }} + securityContext: + capabilities: + drop: +{% for c in apps_drop_cap %} + - {{ c.upper() }} +{% endfor %} resources: limits: cpu: {{ netchecker_agent_cpu_limit }} diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml b/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml index c1d8ddb9f..a52c5d4a3 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml +++ b/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml @@ -33,3 +33,9 @@ spec: memory: {{ netchecker_kubectl_memory_requests }} args: - proxy + securityContext: + capabilities: + drop: +{% for c in apps_drop_cap %} + - {{ c.upper() }} +{% endfor %} diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 874925adf..4e29a51fd 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -13,6 +13,21 @@ kube_apiserver_node_port_range: "30000-32767" etcd_config_dir: /etc/ssl/etcd etcd_cert_dir: "{{ etcd_config_dir }}/ssl" +# Linux capabilities to be dropped for k8s apps ran by container engines +apps_drop_cap: + - chown + - dac_override + - fowner + - fsetid + - kill + - setgid + - setuid + - setpcap + - sys_chroot + - mknod + - audit_write + - setfcap + # Limits for kube components kube_controller_memory_limit: 512M kube_controller_cpu_limit: 250m diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index a622594a1..b0bab4cdc 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -2,6 +2,9 @@ - include: pre-upgrade.yml tags: k8s-pre-upgrade +- include: set_facts.yml + tags: facts + - name: Copy kubectl from hyperkube container command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp /hyperkube /systembindir/kubectl" register: kube_task_result diff --git a/roles/kubernetes/master/tasks/set_facts.yml b/roles/kubernetes/master/tasks/set_facts.yml new file mode 100644 index 000000000..d5c3250b3 --- /dev/null +++ b/roles/kubernetes/master/tasks/set_facts.yml @@ -0,0 +1,22 @@ +--- +- name: Master | get kube user ID + shell: /usr/bin/id -u {{ kubelet_user }} || echo 0 + register: kube_uid + +- name: Master | get kube group ID + shell: /usr/bin/getent group {{ kubelet_group }} | cut -d':' -f3 || echo 0 + register: kube_gid + +- name: Master | get kube cert group ID + shell: /usr/bin/getent group {{ kube_cert_group }} | cut -d':' -f3 || echo 0 + register: kube_cert_gid + +- name: Master | get etcd cert group ID + shell: /usr/bin/getent group {{ etcd_cert_group }} | cut -d':' -f3 || echo 0 + register: etcd_cert_gid + +- set_fact: + kubelet_user_id: "{{ kube_uid.stdout }}" + kubelet_group_id: "{{ kube_gid.stdout }}" + kube_cert_group_id: "{{ kube_cert_gid.stdout }}" + etcd_cert_group_id: "{{ etcd_cert_gid.stdout }}" diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index c05030697..cab219f60 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -12,6 +12,14 @@ spec: - name: kube-apiserver image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} + securityContext: + capabilities: + drop: +{% for c in apps_drop_cap %} + - {{ c.upper() }} +{% endfor %} + add: + - DAC_OVERRIDE resources: limits: cpu: {{ kube_apiserver_cpu_limit }} diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 49dd05ba8..6db17f30a 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -11,6 +11,17 @@ spec: - name: kube-controller-manager image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} + securityContext: + runAsUser: {{ kubelet_user_id }} + fsGroup: {{ kubelet_group_id }} + supplementalGroups: + - {{ kube_cert_group_id }} + - {{ etcd_cert_group_id }} + capabilities: + drop: +{% for c in apps_drop_cap %} + - {{ c.upper() }} +{% endfor %} resources: limits: cpu: {{ kube_controller_cpu_limit }} diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index 781e38d7b..ded01dc79 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -11,6 +11,17 @@ spec: - name: kube-scheduler image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} + securityContext: + runAsUser: {{ kubelet_user_id }} + fsGroup: {{ kubelet_group_id }} + supplementalGroups: + - {{ kube_cert_group_id }} + - {{ etcd_cert_group_id }} + capabilities: + drop: +{% for c in apps_drop_cap %} + - {{ c.upper() }} +{% endfor %} resources: limits: cpu: {{ kube_scheduler_cpu_limit }} diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index a74e52b77..777a751fa 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -29,3 +29,18 @@ nginx_image_repo: nginx nginx_image_tag: 1.11.4-alpine etcd_config_dir: /etc/ssl/etcd + +# Linux capabilities to be dropped for container engines +apps_drop_cap: + - chown + - dac_override + - fowner + - fsetid + - kill + - setgid + - setuid + - setpcap + - sys_chroot + - mknod + - audit_write + - setfcap diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index bfe4a8cc8..52a32ccc3 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -26,6 +26,6 @@ notify: restart kubelet - name: install | Install kubelet launch script - template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes + template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner={{ kubelet_user }} mode=0755 backup=yes notify: restart kubelet when: kubelet_deployment_type == "docker" diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 3e0c095e1..29e9ef4e0 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -4,6 +4,9 @@ {%- if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] -%}true{%- else -%}false{%- endif -%} tags: facts +- include: pre-upgrade.yml + tags: k8s-pre-upgrade + - include: install.yml tags: kubelet diff --git a/roles/kubernetes/node/tasks/pre-upgrade.yml b/roles/kubernetes/node/tasks/pre-upgrade.yml new file mode 100644 index 000000000..9a61170d7 --- /dev/null +++ b/roles/kubernetes/node/tasks/pre-upgrade.yml @@ -0,0 +1,4 @@ +--- +- name: "Pre-upgrade | share access to kube certs for its users" + shell: chmod g+r {{ kube_cert_dir }}/*.pem + failed_when: false diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 index 12ce01c75..2537d9ffd 100644 --- a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 @@ -29,7 +29,7 @@ ExecStart=/usr/bin/rkt run \ --volume run,kind=host,source=/run,readOnly=false \ --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ --volume var-lib-docker,kind=host,source={{ docker_daemon_graph }},readOnly=false \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false \ --volume var-log,kind=host,source=/var/log \ --mount volume=dns,target=/etc/resolv.conf \ --mount volume=etc-cni,target=/etc/cni \ @@ -44,6 +44,7 @@ ExecStart=/usr/bin/rkt run \ --mount volume=var-log,target=/var/log \ --stage1-from-dir=stage1-fly.aci \ {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} \ + --memory={{ kubelet_memory_limit }} --cpu={{ kubelet_cpu_limit }} \ --uuid-file-save=/var/run/kubelet.uuid \ --debug --exec=/kubelet -- \ $KUBE_LOGTOSTDERR \ diff --git a/roles/kubernetes/preinstall/meta/main.yml b/roles/kubernetes/preinstall/meta/main.yml index cf440f5e2..34785ed88 100644 --- a/roles/kubernetes/preinstall/meta/main.yml +++ b/roles/kubernetes/preinstall/meta/main.yml @@ -2,4 +2,7 @@ dependencies: - role: adduser user: "{{ addusers.kube }}" - tags: kubelet + tags: [bootstrap-os, kubelet] + - role: adduser + user: "{{ addusers.netplug }}" + tags: [bootstrap-os, network] diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index e3ecf25aa..41d3f009e 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -23,6 +23,12 @@ - include: set_facts.yml tags: facts +- include: set_resolv_facts.yml + tags: [bootstrap-os, resolvconf, facts] + +- include: set_uid_facts.yml + tags: [bootstrap-os, facts] + - name: gather os specific variables include_vars: "{{ item }}" with_first_found: @@ -42,7 +48,7 @@ file: path: "{{ kube_config_dir }}" state: directory - owner: kube + owner: "{{ kubelet_user }}" when: "{{ inventory_hostname in groups['k8s-cluster'] }}" tags: [kubelet, k8s-secrets, kube-controller-manager, kube-apiserver, bootstrap-os, apps, network, master, node] @@ -50,7 +56,7 @@ file: path: "{{ kube_script_dir }}" state: directory - owner: kube + owner: "{{ kubelet_user }}" when: "{{ inventory_hostname in groups['k8s-cluster'] }}" tags: [k8s-secrets, bootstrap-os] @@ -58,7 +64,7 @@ file: path: "{{ kube_manifest_dir }}" state: directory - owner: kube + owner: "{{ kubelet_user }}" when: "{{ inventory_hostname in groups['k8s-cluster'] }}" tags: [kubelet, bootstrap-os, master, node] @@ -80,7 +86,7 @@ file: path: "{{ item }}" state: directory - owner: kube + owner: "{{ kubelet_user }}" with_items: - "/etc/cni/net.d" - "/opt/cni/bin" diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index 456467a97..d2fad6e3d 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -51,6 +51,3 @@ etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}" - set_fact: peer_with_calico_rr: "{{ 'calico-rr' in groups and groups['calico-rr']|length > 0 }}" - -- include: set_resolv_facts.yml - tags: [bootstrap-os, resolvconf, facts] diff --git a/roles/kubernetes/preinstall/tasks/set_uid_facts.yml b/roles/kubernetes/preinstall/tasks/set_uid_facts.yml new file mode 100644 index 000000000..13a36b5db --- /dev/null +++ b/roles/kubernetes/preinstall/tasks/set_uid_facts.yml @@ -0,0 +1,32 @@ +--- +- name: Preinstall | get kube user ID + shell: /usr/bin/id -u {{ kubelet_user }} || echo 0 + register: kube_uid + +- name: Preinstall | get kube group ID + shell: /usr/bin/id -g {{ kubelet_group }} || echo 0 + register: kube_gid + +- name: Preinstall | get kube cert group ID + shell: /usr/bin/id -g {{ kube_cert_group }} || echo 0 + register: kube_cert_gid + +- name: Preinstall | get etcd cert group ID + shell: /usr/bin/id -g {{ etcd_cert_group }} || echo 0 + register: etcd_cert_gid + +- name: Preinstall | get netplug user ID + shell: /usr/bin/id -u {{ netplug_user }} || echo 0 + register: netplug_uid + +- name: Preinstall | get netplug group ID + shell: /usr/bin/getent group {{ netplug_group }} | cut -d':' -f3 || echo 0 + register: netplug_gid + +- set_fact: + kubelet_user_id: "{{ kube_uid.stdout }}" + kubelet_group_id: "{{ kube_gid.stdout }}" + kube_cert_group_id: "{{ kube_cert_gid.stdout }}" + etcd_cert_group_id: "{{ etcd_cert_gid.stdout }}" + netplug_user_id: "{{ netplug_uid.stdout }}" + netplug_group_id: "{{ netplug_gid.stdout }}" diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh index dca9d9ab9..cc0b49b95 100755 --- a/roles/kubernetes/secrets/files/make-ssl.sh +++ b/roles/kubernetes/secrets/files/make-ssl.sh @@ -101,5 +101,8 @@ if [ -n "$HOSTS" ]; then done fi +# Grant the group read access +chmod g+r *.pem + # Install certs mv *.pem ${SSLDIR}/ diff --git a/roles/kubernetes/secrets/tasks/gen_certs.yml b/roles/kubernetes/secrets/tasks/gen_certs.yml index cf4614d74..f6c4d0079 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs.yml @@ -110,11 +110,11 @@ file: path={{ kube_cert_dir }} group={{ kube_cert_group }} - owner=kube + owner={{ kubelet_user }} recurse=yes -- name: Gen_certs | set permissions on keys - shell: chmod 0600 {{ kube_cert_dir}}/*key.pem +- name: Gen_certs | set shared group permissions on keys + shell: chmod 0640 {{ kube_cert_dir}}/*.pem when: inventory_hostname in groups['kube-master'] changed_when: false diff --git a/roles/kubernetes/secrets/tasks/main.yml b/roles/kubernetes/secrets/tasks/main.yml index 4d25a94af..81eb26743 100644 --- a/roles/kubernetes/secrets/tasks/main.yml +++ b/roles/kubernetes/secrets/tasks/main.yml @@ -9,6 +9,7 @@ path={{ kube_cert_dir }} state=directory mode=o-rwx + owner={{ kubelet_user }} group={{ kube_cert_group }} - name: Make sure the tokens directory exits @@ -16,14 +17,16 @@ path={{ kube_token_dir }} state=directory mode=o-rwx - group={{ kube_cert_group }} + owner={{ kubelet_user }} + group={{ kubelet_group }} - name: Make sure the users directory exits file: path={{ kube_users_dir }} state=directory mode=o-rwx - group={{ kube_cert_group }} + owner={{ kubelet_user }} + group={{ kubelet_group }} - name: Populate users for basic auth in API lineinfile: diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml index 7681abc5c..6cd5c2213 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico/defaults/main.yml @@ -20,6 +20,23 @@ global_as_num: "64512" # defaults. The value should be a number, not a string. # calico_mtu: 1500 +# Linux capabilities to be dropped for container engines +calico_drop_cap: + - chown + - dac_override + - fowner + - fsetid + - kill + - setgid + - setuid + - setpcap + - net_bind_service + - net_raw + - sys_chroot + - mknod + - audit_write + - setfcap + # Limits for apps calico_node_memory_limit: 500M calico_node_cpu_limit: 300m diff --git a/roles/network_plugin/calico/rr/tasks/main.yml b/roles/network_plugin/calico/rr/tasks/main.yml index efe4616d2..1cbbb43f8 100644 --- a/roles/network_plugin/calico/rr/tasks/main.yml +++ b/roles/network_plugin/calico/rr/tasks/main.yml @@ -12,8 +12,8 @@ dest: "{{ calico_cert_dir }}" state: directory mode: 0750 - owner: root - group: root + owner: "{{ netplug_user }}" + group: "{{ netplug_group }}" - name: Calico-rr | Link etcd certificates for calico-node file: @@ -31,8 +31,8 @@ path: /var/log/calico-rr state: directory mode: 0755 - owner: root - group: root + owner: "{{ netplug_user }}" + group: "{{ netplug_group }}" - name: Calico-rr | Write calico-rr.env for systemd init file template: src=calico-rr.env.j2 dest=/etc/calico/calico-rr.env diff --git a/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 b/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 index f6da04a4d..6d4b344d3 100644 --- a/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 +++ b/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 @@ -6,7 +6,7 @@ Requires=docker.service [Service] EnvironmentFile=/etc/calico/calico-rr.env ExecStartPre=-{{ docker_bin_dir }}/docker rm -f calico-rr -ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \ +ExecStart={{ docker_bin_dir }}/docker run --net=host \ --name=calico-rr \ -e IP=${IP} \ -e IP6=${IP6} \ @@ -16,6 +16,10 @@ ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \ -e ETCD_KEY_FILE=${ETCD_KEY_FILE} \ -v /var/log/calico-rr:/var/log/calico \ -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \ +{% for c in calico_drop_cap %} + --cap-drop={{ c }} \ +{% endfor %} + -u {{ netplug_user_id }}:{{ netplug_group_id }} --group-add {{ etcd_cert_group }} \ --memory={{ calico_rr_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_rr_cpu_limit|regex_replace('m', '') }} \ {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }} diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index 462fcec66..ab5657dfd 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -9,15 +9,16 @@ template: src: "cni-calico.conf.j2" dest: "/etc/cni/net.d/10-calico.conf" - owner: kube + owner: "{{ kubelet_user }}" + group: "{{ kubelet_group }}" - name: Calico | Create calico certs directory file: dest: "{{ calico_cert_dir }}" state: directory mode: 0750 - owner: root - group: root + owner: "{{ netplug_user }}" + group: "{{ netplug_group }}" - name: Calico | Link etcd certificates for calico-node file: diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml index 7ccbcdf2e..a6dedeec4 100644 --- a/roles/network_plugin/canal/tasks/main.yml +++ b/roles/network_plugin/canal/tasks/main.yml @@ -3,15 +3,16 @@ template: src: "cni-canal.conf.j2" dest: "/etc/cni/net.d/10-canal.conf" - owner: kube + owner: "{{ kubelet_user }}" + group: "{{ kubelet_group }}" - name: Canal | Create canal certs directory file: dest: "{{ canal_cert_dir }}" state: directory mode: 0750 - owner: root - group: root + owner: "{{ netplug_user }}" + group: "{{ netplug_group }}" - name: Canal | Link etcd certificates for canal-node file: