0c7e1889e4
This allows `kube_apiserver_insecure_port` to be set to 0 (disabled). It's working, but so far I have had to: 1. Make the `uri` module "Wait for apiserver up" checks use `kube_apiserver_port` (HTTPS) 2. Add apiserver client cert/key to the "Wait for apiserver up" checks 3. Update apiserver liveness probe to use HTTPS ports 4. Set `kube_api_anonymous_auth` to true to allow liveness probe to hit apiserver's /healthz over HTTPS (livenessProbes can't use client cert/key unfortunately) 5. RBAC has to be enabled. Anonymous requests are in the `system:unauthenticated` group which is granted access to /healthz by one of RBAC's default ClusterRoleBindings. An equivalent ABAC rule could allow this as well. Changes 1 and 2 should work for everyone, but 3, 4, and 5 require new coupling of currently independent configuration settings. So I also added a new settings check. Options: 1. The problem goes away if you have both anonymous-auth and RBAC enabled. This is how kubeadm does it. This may be the best way to go since RBAC is already on by default but anonymous auth is not. 2. Include conditional templates to set a different liveness probe for possible combinations of `kube_apiserver_insecure_port = 0`, RBAC, and `kube_api_anonymous_auth` (won't be possible to cover every case without a guaranteed authorizer for the secure port) 3. Use basic auth headers for the liveness probe (I really don't like this, it adds a new dependency on basic auth which I'd also like to leave independently configurable, and it requires encoded passwords in the apiserver manifest) Option 1 seems like the clear winner to me, but is there a reason we wouldn't want anonymous-auth on by default? The apiserver binary defaults anonymous-auth to true, but kubespray's default was false.
163 lines
5.8 KiB
Django/Jinja
163 lines
5.8 KiB
Django/Jinja
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: kube-apiserver
|
|
namespace: {{system_namespace}}
|
|
labels:
|
|
k8s-app: kube-apiserver
|
|
kubespray: v2
|
|
annotations:
|
|
kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
|
|
kubespray.apiserver-cert/serial: "{{ apiserver_cert_serial }}"
|
|
spec:
|
|
hostNetwork: true
|
|
{% if kube_version | version_compare('v1.6', '>=') %}
|
|
dnsPolicy: ClusterFirst
|
|
{% endif %}
|
|
containers:
|
|
- name: kube-apiserver
|
|
image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
|
|
imagePullPolicy: {{ k8s_image_pull_policy }}
|
|
resources:
|
|
limits:
|
|
cpu: {{ kube_apiserver_cpu_limit }}
|
|
memory: {{ kube_apiserver_memory_limit }}
|
|
requests:
|
|
cpu: {{ kube_apiserver_cpu_requests }}
|
|
memory: {{ kube_apiserver_memory_requests }}
|
|
command:
|
|
- /hyperkube
|
|
- apiserver
|
|
- --advertise-address={{ ip | default(ansible_default_ipv4.address) }}
|
|
- --etcd-servers={{ etcd_access_addresses }}
|
|
- --etcd-quorum-read=true
|
|
- --etcd-cafile={{ etcd_cert_dir }}/ca.pem
|
|
- --etcd-certfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
|
|
- --etcd-keyfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
|
|
- --insecure-bind-address={{ kube_apiserver_insecure_bind_address }}
|
|
- --apiserver-count={{ kube_apiserver_count }}
|
|
- --admission-control={{ kube_apiserver_admission_control | join(',') }}
|
|
- --service-cluster-ip-range={{ kube_service_addresses }}
|
|
- --service-node-port-range={{ kube_apiserver_node_port_range }}
|
|
- --client-ca-file={{ kube_cert_dir }}/ca.pem
|
|
- --profiling=false
|
|
- --repair-malformed-updates=false
|
|
- --kubelet-client-certificate={{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem
|
|
- --kubelet-client-key={{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem
|
|
- --service-account-lookup=true
|
|
{% if kube_basic_auth|default(true) %}
|
|
- --basic-auth-file={{ kube_users_dir }}/known_users.csv
|
|
{% endif %}
|
|
- --tls-cert-file={{ kube_cert_dir }}/apiserver.pem
|
|
- --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
|
|
- --proxy-client-cert-file={{ kube_cert_dir }}/apiserver.pem
|
|
- --proxy-client-key-file={{ kube_cert_dir }}/apiserver-key.pem
|
|
{% if kube_token_auth|default(true) %}
|
|
- --token-auth-file={{ kube_token_dir }}/known_tokens.csv
|
|
{% endif %}
|
|
- --service-account-key-file={{ kube_cert_dir }}/apiserver-key.pem
|
|
{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
|
|
- --oidc-issuer-url={{ kube_oidc_url }}
|
|
- --oidc-client-id={{ kube_oidc_client_id }}
|
|
{% if kube_oidc_ca_file is defined %}
|
|
- --oidc-ca-file={{ kube_oidc_ca_file }}
|
|
{% endif %}
|
|
{% if kube_oidc_username_claim is defined %}
|
|
- --oidc-username-claim={{ kube_oidc_username_claim }}
|
|
{% endif %}
|
|
{% if kube_oidc_groups_claim is defined %}
|
|
- --oidc-groups-claim={{ kube_oidc_groups_claim }}
|
|
{% endif %}
|
|
{% endif %}
|
|
- --secure-port={{ kube_apiserver_port }}
|
|
- --insecure-port={{ kube_apiserver_insecure_port }}
|
|
- --storage-backend={{ kube_apiserver_storage_backend }}
|
|
{% if kube_api_runtime_config is defined %}
|
|
{% for conf in kube_api_runtime_config %}
|
|
- --runtime-config={{ conf }}
|
|
{% endfor %}
|
|
{% endif %}
|
|
{% if enable_network_policy %}
|
|
{% if kube_version | version_compare('v1.8', '<') %}
|
|
- --runtime-config=extensions/v1beta1/networkpolicies=true
|
|
{% endif %}
|
|
{% endif %}
|
|
- --v={{ kube_log_level }}
|
|
- --allow-privileged=true
|
|
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere"] %}
|
|
- --cloud-provider={{ cloud_provider }}
|
|
- --cloud-config={{ kube_config_dir }}/cloud_config
|
|
{% elif cloud_provider is defined and cloud_provider == "aws" %}
|
|
- --cloud-provider={{ cloud_provider }}
|
|
{% endif %}
|
|
{% if kube_api_anonymous_auth is defined and kube_version | version_compare('v1.5', '>=') %}
|
|
- --anonymous-auth={{ kube_api_anonymous_auth }}
|
|
{% endif %}
|
|
{% if authorization_modes %}
|
|
- --authorization-mode={{ authorization_modes|join(',') }}
|
|
{% endif %}
|
|
{% if kube_feature_gates %}
|
|
- --feature-gates={{ kube_feature_gates|join(',') }}
|
|
{% endif %}
|
|
{% if apiserver_custom_flags is string %}
|
|
- {{ apiserver_custom_flags }}
|
|
{% else %}
|
|
{% for flag in apiserver_custom_flags %}
|
|
- {{ flag }}
|
|
{% endfor %}
|
|
{% endif %}
|
|
livenessProbe:
|
|
httpGet:
|
|
host: 127.0.0.1
|
|
path: /healthz
|
|
{% if kube_apiserver_insecure_port == 0 %}
|
|
port: {{ kube_apiserver_port }}
|
|
scheme: HTTPS
|
|
{% else %}
|
|
port: {{ kube_apiserver_insecure_port }}
|
|
{% endif %}
|
|
failureThreshold: 8
|
|
initialDelaySeconds: 15
|
|
periodSeconds: 10
|
|
successThreshold: 1
|
|
timeoutSeconds: 15
|
|
volumeMounts:
|
|
- mountPath: {{ kube_config_dir }}
|
|
name: kubernetes-config
|
|
readOnly: true
|
|
- mountPath: /etc/ssl
|
|
name: ssl-certs-host
|
|
readOnly: true
|
|
{% for dir in ssl_ca_dirs %}
|
|
- mountPath: {{ dir }}
|
|
name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
|
|
readOnly: true
|
|
{% endfor %}
|
|
- mountPath: {{ etcd_cert_dir }}
|
|
name: etcd-certs
|
|
readOnly: true
|
|
{% if cloud_provider is defined and cloud_provider == 'aws' and ansible_os_family == 'RedHat' %}
|
|
- mountPath: /etc/ssl/certs/ca-bundle.crt
|
|
name: rhel-ca-bundle
|
|
readOnly: true
|
|
{% endif %}
|
|
volumes:
|
|
- hostPath:
|
|
path: {{ kube_config_dir }}
|
|
name: kubernetes-config
|
|
- name: ssl-certs-host
|
|
hostPath:
|
|
path: /etc/ssl
|
|
{% for dir in ssl_ca_dirs %}
|
|
- name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
|
|
hostPath:
|
|
path: {{ dir }}
|
|
{% endfor %}
|
|
- hostPath:
|
|
path: {{ etcd_cert_dir }}
|
|
name: etcd-certs
|
|
{% if cloud_provider is defined and cloud_provider == 'aws' and ansible_os_family == 'RedHat' %}
|
|
- hostPath:
|
|
path: /etc/ssl/certs/ca-bundle.crt
|
|
name: rhel-ca-bundle
|
|
{% endif %}
|