From 2ffc1afe400f80083c22cee76d138fecfd628c49 Mon Sep 17 00:00:00 2001 From: rongzhang Date: Wed, 15 Aug 2018 16:41:13 +0800 Subject: [PATCH] Support audit --- inventory/sample/group_vars/k8s-cluster.yml | 3 + roles/kubernetes/master/defaults/main.yml | 23 ++++ .../kubernetes/master/tasks/kubeadm-setup.yml | 10 ++ .../master/tasks/static-pod-setup.yml | 15 +++ .../templates/apiserver-audit-policy.yaml.j2 | 125 ++++++++++++++++++ .../templates/kubeadm-config.v1alpha2.yaml.j2 | 16 +++ .../manifests/kube-apiserver.manifest.j2 | 23 ++++ tests/files/gce_centos-weave-kubeadm.yml | 1 + tests/files/gce_centos7-flannel-addons.yml | 1 + 9 files changed, 217 insertions(+) create mode 100644 roles/kubernetes/master/templates/apiserver-audit-policy.yaml.j2 diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index 795d76eb3..3e43b0833 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -163,6 +163,9 @@ helm_deployment_type: host # K8s image pull policy (imagePullPolicy) k8s_image_pull_policy: IfNotPresent +# audit log for kubernetes +kubernetes_audit: false + # Kubernetes dashboard # RBAC required. see docs/getting-started.md for access details. dashboard_enabled: true diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 82669e8b3..68a09cef0 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -24,6 +24,29 @@ kube_apiserver_storage_backend: etcd3 # By default, force back to etcd2. Set to true to force etcd3 (experimental!) force_etcd3: false +# audit support +kubernetes_audit: false +audit_log_path: /var/log/audit/kube-apiserver-audit.log +# num days +audit_log_maxage: 30 +# the num of audit logs to retain +audit_log_maxbackups: 1 + # the max size in MB to retain +audit_log_maxsize: 100 +# policy file +audit_policy_file: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml" + +# audit log hostpath +audit_log_name: audit-logs +audit_log_hostpath: /var/log/kubernetes/audit +audit_log_mountpath: /var/log/audit +audit_log_writable: true + +# audit policy hostpath +audit_policy_name: audit-policy +audit_policy_hostpath: /etc/kubernetes/audit-policy +audit_policy_mountpath: "{{ audit_policy_hostpath }}" + # Limits for kube components kube_controller_memory_limit: 512M kube_controller_cpu_limit: 250m diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index 80e89acc9..2ba7485a1 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -65,6 +65,16 @@ command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd" changed_when: false +- name: Create audit-policy directory + file: path={{ kube_config_dir }}/audit-policy state=directory + when: kubernetes_audit|default(false) + +- name: Write api audit policy yaml + template: + src: apiserver-audit-policy.yaml.j2 + dest: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml" + when: kubernetes_audit|default(false) + - name: gets the kubeadm version command: "{{ bin_dir }}/kubeadm version -o short" register: kubeadm_output diff --git a/roles/kubernetes/master/tasks/static-pod-setup.yml b/roles/kubernetes/master/tasks/static-pod-setup.yml index ca00ca33c..b1fbdc095 100644 --- a/roles/kubernetes/master/tasks/static-pod-setup.yml +++ b/roles/kubernetes/master/tasks/static-pod-setup.yml @@ -1,4 +1,19 @@ --- +- name: Create audit-policy directory + file: path={{ kube_config_dir }}/audit-policy state=directory + tags: + - kube-apiserver + when: kubernetes_audit|default(false) + +- name: Write api audit policy yaml + template: + src: apiserver-audit-policy.yaml.j2 + dest: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml" + notify: Master | Restart apiserver + tags: + - kube-apiserver + when: kubernetes_audit|default(false) + - name: Write kube-apiserver manifest template: src: manifests/kube-apiserver.manifest.j2 diff --git a/roles/kubernetes/master/templates/apiserver-audit-policy.yaml.j2 b/roles/kubernetes/master/templates/apiserver-audit-policy.yaml.j2 new file mode 100644 index 000000000..40d6a8bb5 --- /dev/null +++ b/roles/kubernetes/master/templates/apiserver-audit-policy.yaml.j2 @@ -0,0 +1,125 @@ +apiVersion: audit.k8s.io/v1beta1 +kind: Policy +rules: + # The following requests were manually identified as high-volume and low-risk, + # so drop them. + - level: None + users: ["system:kube-proxy"] + verbs: ["watch"] + resources: + - group: "" # core + resources: ["endpoints", "services", "services/status"] + - level: None + # Ingress controller reads `configmaps/ingress-uid` through the unsecured port. + # TODO(#46983): Change this to the ingress controller service account. + users: ["system:unsecured"] + namespaces: ["kube-system"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["configmaps"] + - level: None + users: ["kubelet"] # legacy kubelet identity + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + userGroups: ["system:nodes"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + users: + - system:kube-controller-manager + - system:kube-scheduler + - system:serviceaccount:kube-system:endpoint-controller + verbs: ["get", "update"] + namespaces: ["kube-system"] + resources: + - group: "" # core + resources: ["endpoints"] + - level: None + users: ["system:apiserver"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["namespaces", "namespaces/status", "namespaces/finalize"] + # Don't log HPA fetching metrics. + - level: None + users: + - system:kube-controller-manager + verbs: ["get", "list"] + resources: + - group: "metrics.k8s.io" + # Don't log these read-only URLs. + - level: None + nonResourceURLs: + - /healthz* + - /version + - /swagger* + # Don't log events requests. + - level: None + resources: + - group: "" # core + resources: ["events"] + # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, + # so only log at the Metadata level. + - level: Metadata + resources: + - group: "" # core + resources: ["secrets", "configmaps"] + - group: authentication.k8s.io + resources: ["tokenreviews"] + omitStages: + - "RequestReceived" + # Get responses can be large; skip them. + - level: Request + verbs: ["get", "list", "watch"] + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for known APIs + - level: RequestResponse + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for all other requests. + - level: Metadata + omitStages: + - "RequestReceived" diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index 907ea55f2..29aac6f87 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -12,6 +12,12 @@ etcd: caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem +{% if kubernetes_audit %} +auditPolicy: + logDir: {{ audit_log_path }} + logMaxAge: {{ audit_log_maxage }} + path: {{ audit_policy_file }} +{% endif %} networking: dnsDomain: {{ dns_domain }} serviceSubnet: {{ kube_service_addresses }} @@ -82,6 +88,12 @@ controllerManagerExtraArgs: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} node-monitor-period: {{ kube_controller_node_monitor_period }} pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }} +{% if kubernetes_audit %} +apiServerExtraVolumes: +- name: {{ audit_policy_name }} + hostPath: {{ audit_policy_hostpath }} + mountPath: {{ audit_policy_mountpath }} +{% endif %} {% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %} controllerManagerExtraVolumes: - name: openstackcacert @@ -113,3 +125,7 @@ nodeRegistration: taints: - effect: NoSchedule key: node-role.kubernetes.io/master +{% if kubernetes_audit %} +featureGates: + Auditing: true +{% endif %} diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 82bd1db93..9cec5ded7 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -28,6 +28,13 @@ spec: command: - /hyperkube - apiserver +{% if kubernetes_audit %} + - --audit-log-path={{ audit_log_path }} + - --audit-log-maxage={{ audit_log_maxage }} + - --audit-log-maxbackup={{ audit_log_maxbackups }} + - --audit-log-maxsize={{ audit_log_maxsize }} + - --audit-policy-file={{ audit_policy_file }} +{% endif %} - --advertise-address={{ ip | default(ansible_default_ipv4.address) }} - --etcd-servers={{ etcd_access_addresses }} {% if etcd_events_cluster_enabled %} @@ -184,6 +191,14 @@ spec: - mountPath: /etc/ssl/certs/ca-bundle.crt name: rhel-ca-bundle readOnly: true +{% endif %} +{% if kubernetes_audit %} + - mountPath: {{ audit_log_mountpath }} + name: {{ audit_log_name }} + Writable: true + - mountPath: {{ audit_policy_mountpath }} + name: {{ audit_policy_name }} + Writable: true {% endif %} volumes: - hostPath: @@ -205,3 +220,11 @@ spec: path: /etc/ssl/certs/ca-bundle.crt name: rhel-ca-bundle {% endif %} +{% if kubernetes_audit %} + - hostPath: + path: {{ audit_log_hostpath }} + name: {{ audit_log_name }} + - hostPath: + path: {{ audit_policy_hostpath }} + name: {{ audit_policy_name }} +{% endif %} diff --git a/tests/files/gce_centos-weave-kubeadm.yml b/tests/files/gce_centos-weave-kubeadm.yml index a410be3f2..199fa437c 100644 --- a/tests/files/gce_centos-weave-kubeadm.yml +++ b/tests/files/gce_centos-weave-kubeadm.yml @@ -9,5 +9,6 @@ startup_script: "" kube_network_plugin: weave kubeadm_enabled: true deploy_netchecker: true +kubernetes_audit: true kubedns_min_replicas: 1 cloud_provider: gce diff --git a/tests/files/gce_centos7-flannel-addons.yml b/tests/files/gce_centos7-flannel-addons.yml index 1a03b0f9b..3dffa338f 100644 --- a/tests/files/gce_centos7-flannel-addons.yml +++ b/tests/files/gce_centos7-flannel-addons.yml @@ -8,6 +8,7 @@ mode: ha kube_network_plugin: flannel helm_enabled: true efk_enabled: true +kubernetes_audit: true etcd_events_cluster_setup: true local_volume_provisioner_enabled: true etcd_deployment_type: host