Add HA/LB endpoints for kube-apiserver
* Add auto-evaluated internal endpoints and clarify the loadbalancer_apiserver vars and usecases. * Add loadbalancer_apiserver_localhost (default false). If enabled, override the external LB and expect localhost:443/8080 to be new internal only frontends. * Add kube_apiserver_multiaccess to ignore loadbalancers, and make clients to access the apiservers as a comma-separated list of access_ip/ip/ansible ip (a default mode). When disabled, allow clients to use the given loadbalancers. * Define connections security mode for kube controllers, schedulers, proxies. It is insecure be default, which is the current deployment choice. * Rework the groups['kube-master'][0] hardcode defining the apiserver endpoints. * Improve grouping of vars and add facts for kube_apiserver. * Define kube_apiserver_insecure_bind_address as a fact, add more facts for ease of use. Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
This commit is contained in:
parent
277c5d74cc
commit
a70c3b661e
8 changed files with 104 additions and 37 deletions
|
@ -53,7 +53,7 @@ cluster_name: cluster.local
|
||||||
|
|
||||||
# Service endpoints. May be a VIP or a load balanced frontend IP, like one
|
# Service endpoints. May be a VIP or a load balanced frontend IP, like one
|
||||||
# that a HAProxy or Nginx provides, or just a local service endpoint.
|
# that a HAProxy or Nginx provides, or just a local service endpoint.
|
||||||
#
|
|
||||||
# Etcd endpoints use a local etcd-proxies to reach the etcd cluster via
|
# Etcd endpoints use a local etcd-proxies to reach the etcd cluster via
|
||||||
# auto-evaluated endpoints. Those will reuse the access_ip for etcd cluster,
|
# auto-evaluated endpoints. Those will reuse the access_ip for etcd cluster,
|
||||||
# if specified, or defer to the localhost:2379 as well.
|
# if specified, or defer to the localhost:2379 as well.
|
||||||
|
@ -64,15 +64,64 @@ cluster_name: cluster.local
|
||||||
# This may be the case if clients support and loadbalance multiple etcd servers natively.
|
# This may be the case if clients support and loadbalance multiple etcd servers natively.
|
||||||
etcd_multiaccess: false
|
etcd_multiaccess: false
|
||||||
|
|
||||||
#
|
# The port the API Server will be listening on.
|
||||||
# TODO apiserver localhost:8080 and localhost:443 endpoints for kubelets and
|
kube_apiserver_port: 443 # (https)
|
||||||
# (hyper)kube-* and networking components.
|
kube_apiserver_insecure_port: 8080 # (http)
|
||||||
|
|
||||||
|
# For multi masters architecture:
|
||||||
|
# kube-proxy doesn't support multiple apiservers for the time being so you'll need to configure your own loadbalancer
|
||||||
|
# This domain name will be inserted into the /etc/hosts file of all servers
|
||||||
|
# configuration example with haproxy:
|
||||||
|
# listen kubernetes-apiserver-https
|
||||||
|
# bind 10.99.0.21:8383
|
||||||
|
# option ssl-hello-chk
|
||||||
|
# mode tcp
|
||||||
|
# timeout client 3h
|
||||||
|
# timeout server 3h
|
||||||
|
# server master1 10.99.0.26:443
|
||||||
|
# server master2 10.99.0.27:443
|
||||||
|
# balance roundrobin
|
||||||
|
|
||||||
|
# And the corresponding example config vars:
|
||||||
|
# apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local"
|
||||||
|
# loadbalancer_apiserver:
|
||||||
|
# adress: 10.99.0.21
|
||||||
|
# port: 8383
|
||||||
|
|
||||||
|
# For internal only multi-masters architecture:
|
||||||
|
# Assume there is no loadbalancers configured for internal access. Enable this flag,
|
||||||
|
# if a loadbalancer listens the localhost, which may be the case when there is neither
|
||||||
|
# loadbalancer_apiserver nor access_ip is configured, or you want to override both.
|
||||||
|
loadbalancer_apiserver_localhost: false
|
||||||
|
|
||||||
|
# The internal kube-apiserver endpoints:
|
||||||
|
# The secure endpoint is auto-evaluated. If loadbalancer_apiserver_localhost=true,
|
||||||
|
# it uses the localhost:kube_apiserver_port. Otherwise, it uses the external
|
||||||
|
# apiserver_loadbalancer_domain_name:loadbalancer_apiserver.port. Or defers to the
|
||||||
|
# access_ip:kube_apiserver_port, then ip, then default ansible ip.
|
||||||
|
|
||||||
|
# The insecure endpoint ignores the apiserver_loadbalancer_domain_name and access_ip.
|
||||||
|
# By default, it is a localhost:kube_apiserver_insecure_port.
|
||||||
|
# If loadbalancer_apiserver_localhost, it defers to ip, then default ansible ip.
|
||||||
|
|
||||||
|
# Apiserver access modes:
|
||||||
|
# Configure clients to access all of the apiservers directly as the
|
||||||
|
# "http(s)://hostX:port, http(s)://hostY:port, ..." and ignore the loadbalancers, if any.
|
||||||
|
# This may be the case, if clients support and loadbalance multiple apiservers or
|
||||||
|
# when there is no loadbalancers configured.
|
||||||
|
kube_apiserver_multiaccess: true
|
||||||
|
|
||||||
|
# Define connections security for kube controllers, schedulers and proxies
|
||||||
|
kube_proxy_insecure: true
|
||||||
|
kube_controller_insecure: true
|
||||||
|
kube_scheduler_insecure: true
|
||||||
|
|
||||||
# Choose network plugin (calico, weave or flannel)
|
# Choose network plugin (calico, weave or flannel)
|
||||||
kube_network_plugin: flannel
|
kube_network_plugin: flannel
|
||||||
|
|
||||||
# Kubernetes internal network for services, unused block of space.
|
# Kubernetes internal network for services, unused block of space.
|
||||||
kube_service_addresses: 10.233.0.0/18
|
kube_service_addresses: 10.233.0.0/18
|
||||||
|
kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}"
|
||||||
|
|
||||||
# internal network. When used, it will assign IP
|
# internal network. When used, it will assign IP
|
||||||
# addresses from this range to individual pods.
|
# addresses from this range to individual pods.
|
||||||
|
@ -93,11 +142,6 @@ peer_with_router: false
|
||||||
# Warning : enabling router peering will disable calico's default behavior ('node mesh').
|
# Warning : enabling router peering will disable calico's default behavior ('node mesh').
|
||||||
# The subnets of each nodes will be distributed by the datacenter router
|
# The subnets of each nodes will be distributed by the datacenter router
|
||||||
|
|
||||||
# The port the API Server will be listening on.
|
|
||||||
kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}"
|
|
||||||
kube_apiserver_port: 443 # (https)
|
|
||||||
kube_apiserver_insecure_port: 8080 # (http)
|
|
||||||
|
|
||||||
# Internal DNS configuration.
|
# Internal DNS configuration.
|
||||||
# Kubernetes can create and mainatain its own DNS server to resolve service names
|
# Kubernetes can create and mainatain its own DNS server to resolve service names
|
||||||
# into appropriate IP addresses. It's highly advisable to run such DNS server,
|
# into appropriate IP addresses. It's highly advisable to run such DNS server,
|
||||||
|
@ -126,21 +170,6 @@ dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address')
|
||||||
# like you would do when using nova-client before starting the playbook.
|
# like you would do when using nova-client before starting the playbook.
|
||||||
# cloud_provider:
|
# cloud_provider:
|
||||||
|
|
||||||
# For multi masters architecture:
|
|
||||||
# kube-proxy doesn't support multiple apiservers for the time being so you'll need to configure your own loadbalancer
|
|
||||||
# This domain name will be inserted into the /etc/hosts file of all servers
|
|
||||||
# configuration example with haproxy :
|
|
||||||
# listen kubernetes-apiserver-https
|
|
||||||
# bind 10.99.0.21:8383
|
|
||||||
# option ssl-hello-chk
|
|
||||||
# mode tcp
|
|
||||||
# timeout client 3h
|
|
||||||
# timeout server 3h
|
|
||||||
# server master1 10.99.0.26:443
|
|
||||||
# server master2 10.99.0.27:443
|
|
||||||
# balance roundrobin
|
|
||||||
# apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local"
|
|
||||||
|
|
||||||
## Set these proxy values in order to update docker daemon to use proxies
|
## Set these proxy values in order to update docker daemon to use proxies
|
||||||
# http_proxy: ""
|
# http_proxy: ""
|
||||||
# https_proxy: ""
|
# https_proxy: ""
|
||||||
|
|
|
@ -21,7 +21,7 @@ KUBE_ALLOW_PRIV="--allow_privileged=true"
|
||||||
KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{ kube_apiserver_port }}"
|
KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{ kube_apiserver_port }}"
|
||||||
|
|
||||||
# Insecure API address (default is localhost)
|
# Insecure API address (default is localhost)
|
||||||
KUBE_API_INSECURE_BIND="--insecure-bind-address={{ kube_apiserver_insecure_bind_address | default('127.0.0.1') }}"
|
KUBE_API_INSECURE_BIND="--insecure-bind-address={{ kube_apiserver_insecure_bind_address }}"
|
||||||
|
|
||||||
# Address range to use for services
|
# Address range to use for services
|
||||||
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}"
|
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}"
|
||||||
|
@ -42,7 +42,7 @@ KUBE_RUNTIME_CONFIG="{% if kube_api_runtime_config is defined %}{% for conf in k
|
||||||
KUBE_TLS_CONFIG="--tls_cert_file={{ kube_cert_dir }}/apiserver.pem --tls_private_key_file={{ kube_cert_dir }}/apiserver-key.pem --client_ca_file={{ kube_cert_dir }}/ca.pem"
|
KUBE_TLS_CONFIG="--tls_cert_file={{ kube_cert_dir }}/apiserver.pem --tls_private_key_file={{ kube_cert_dir }}/apiserver-key.pem --client_ca_file={{ kube_cert_dir }}/ca.pem"
|
||||||
|
|
||||||
# Add you own!
|
# Add you own!
|
||||||
KUBE_API_ARGS="--token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/apiserver-key.pem --advertise-address={{ ip | default(ansible_default_ipv4.address) }}"
|
KUBE_API_ARGS="--token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/apiserver-key.pem --advertise-address={{ kube_apiserver_address }}"
|
||||||
|
|
||||||
{% if cloud_provider is defined and cloud_provider == "openstack" %}
|
{% if cloud_provider is defined and cloud_provider == "openstack" %}
|
||||||
KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"
|
KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config"
|
||||||
|
|
|
@ -5,7 +5,7 @@ preferences: {}
|
||||||
clusters:
|
clusters:
|
||||||
- cluster:
|
- cluster:
|
||||||
certificate-authority-data: {{ kube_node_cert|b64encode }}
|
certificate-authority-data: {{ kube_node_cert|b64encode }}
|
||||||
server: https://{{ groups['kube-master'][0] }}:{{ kube_apiserver_port }}
|
server: {{ kube_apiserver_endpoint }}
|
||||||
name: {{ cluster_name }}
|
name: {{ cluster_name }}
|
||||||
contexts:
|
contexts:
|
||||||
- context:
|
- context:
|
||||||
|
|
|
@ -11,7 +11,11 @@ spec:
|
||||||
command:
|
command:
|
||||||
- /hyperkube
|
- /hyperkube
|
||||||
- controller-manager
|
- controller-manager
|
||||||
- --master=http://127.0.0.1:{{kube_apiserver_insecure_port}}
|
{% if kube_controller_insecure %}
|
||||||
|
- --master={{ kube_apiserver_insecure_endpoint }}
|
||||||
|
{% else %}
|
||||||
|
- --master={{ kube_apiserver_endpoint }}
|
||||||
|
{% endif %}
|
||||||
- --leader-elect=true
|
- --leader-elect=true
|
||||||
- --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
|
- --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
|
||||||
- --root-ca-file={{ kube_cert_dir }}/ca.pem
|
- --root-ca-file={{ kube_cert_dir }}/ca.pem
|
||||||
|
|
|
@ -12,7 +12,11 @@ spec:
|
||||||
- /hyperkube
|
- /hyperkube
|
||||||
- scheduler
|
- scheduler
|
||||||
- --leader-elect=true
|
- --leader-elect=true
|
||||||
- --master=http://127.0.0.1:{{kube_apiserver_insecure_port}}
|
{% if kube_scheduler_insecure %}
|
||||||
|
- --master={{ kube_apiserver_insecure_endpoint }}
|
||||||
|
{% else %}
|
||||||
|
- --master={{ kube_apiserver_endpoint }}
|
||||||
|
{% endif %}
|
||||||
- --v={{ kube_log_level | default('2') }}
|
- --v={{ kube_log_level | default('2') }}
|
||||||
livenessProbe:
|
livenessProbe:
|
||||||
httpGet:
|
httpGet:
|
||||||
|
|
|
@ -8,7 +8,7 @@ KUBE_LOGGING="--logtostderr=true"
|
||||||
KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}"
|
KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}"
|
||||||
KUBE_ALLOW_PRIV="--allow_privileged=true"
|
KUBE_ALLOW_PRIV="--allow_privileged=true"
|
||||||
{% if inventory_hostname in groups['kube-node'] %}
|
{% if inventory_hostname in groups['kube-node'] %}
|
||||||
KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}"
|
KUBELET_API_SERVER="--api_servers={{ kube_apiserver_access_endpoint }}"
|
||||||
{% endif %}
|
{% endif %}
|
||||||
# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
|
# The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
|
||||||
KUBELET_ADDRESS="--address={{ ip | default("0.0.0.0") }}"
|
KUBELET_ADDRESS="--address={{ ip | default("0.0.0.0") }}"
|
||||||
|
|
|
@ -12,14 +12,10 @@ spec:
|
||||||
- /hyperkube
|
- /hyperkube
|
||||||
- proxy
|
- proxy
|
||||||
- --v={{ kube_log_level | default('2') }}
|
- --v={{ kube_log_level | default('2') }}
|
||||||
{% if inventory_hostname in groups['kube-master'] %}
|
{% if kube_proxy_insecure %}
|
||||||
- --master=http://127.0.0.1:{{kube_apiserver_insecure_port}}
|
- --master={{ kube_apiserver_insecure_endpoint }}
|
||||||
{% else %}
|
{% else %}
|
||||||
{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
|
- --master={{ kube_apiserver_access_endpoint }}
|
||||||
- --master=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}
|
|
||||||
{% else %}
|
|
||||||
- --master=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}
|
|
||||||
{% endif%}
|
|
||||||
- --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml
|
- --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml
|
||||||
{% endif %}
|
{% endif %}
|
||||||
- --bind-address={{ ip | default(ansible_default_ipv4.address) }}
|
- --bind-address={{ ip | default(ansible_default_ipv4.address) }}
|
||||||
|
|
|
@ -1,4 +1,38 @@
|
||||||
---
|
---
|
||||||
|
- set_fact: kube_apiserver_count="{{ groups['kube-master'] | length }}"
|
||||||
|
- set_fact: kube_apiserver_address="{{ ip | default(ansible_default_ipv4['address']) }}"
|
||||||
|
- set_fact: kube_apiserver_access_address="{{ access_ip | default(kube_apiserver_address) }}"
|
||||||
|
- set_fact:
|
||||||
|
kube_apiserver_insecure_bind_address: |-
|
||||||
|
{% if loadbalancer_apiserver_localhost %}
|
||||||
|
{{ kube_apiserver_address }}{% else %}127.0.0.1{% endif %}
|
||||||
|
- set_fact:
|
||||||
|
apiserver_access_port: |-
|
||||||
|
{% if loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined %}
|
||||||
|
{{ loadbalancer_apiserver.port }}{% else %}{{ kube_apiserver_port }}{% endif %}
|
||||||
|
- set_fact:
|
||||||
|
kube_apiserver_endpoint: |-
|
||||||
|
{% if loadbalancer_apiserver_localhost %}
|
||||||
|
https://127.0.0.1:{{ apiserver_access_port }}
|
||||||
|
{% else -%}
|
||||||
|
https://{{ apiserver_loadbalancer_domain_name|default(kube_apiserver_access_address) }}:{{ apiserver_access_port }}
|
||||||
|
{%- endif %}
|
||||||
|
- set_fact: kube_apiserver_insecure_endpoint="http://{{ kube_apiserver_insecure_bind_address }}:{{ kube_apiserver_insecure_port }}"
|
||||||
|
- set_fact:
|
||||||
|
kube_apiserver_access_endpoint: |-
|
||||||
|
{% if kube_apiserver_multiaccess %}
|
||||||
|
{% for host in groups['kube-master'] -%}
|
||||||
|
https://{{ hostvars[host].kube_apiserver_access_address }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}
|
||||||
|
{%- endfor %}
|
||||||
|
{% else %}{{ kube_apiserver_endpoint }}{% endif %}
|
||||||
|
- set_fact:
|
||||||
|
kube_apiserver_insecure_access_endpoint: |-
|
||||||
|
{% if kube_apiserver_multiaccess %}
|
||||||
|
{% for host in groups['kube-master'] -%}
|
||||||
|
https://{{ hostvars[host].kube_apiserver_insecure_bind_address }}:{{ kube_apiserver_insecure_port }}{% if not loop.last %},{% endif %}
|
||||||
|
{%- endfor %}
|
||||||
|
{% else %}{{ kube_apiserver_insecure_endpoint }}{% endif %}
|
||||||
|
|
||||||
- set_fact: etcd_address="{{ ip | default(ansible_default_ipv4['address']) }}"
|
- set_fact: etcd_address="{{ ip | default(ansible_default_ipv4['address']) }}"
|
||||||
- set_fact: etcd_access_address="{{ access_ip | default(etcd_address) }}"
|
- set_fact: etcd_access_address="{{ access_ip | default(etcd_address) }}"
|
||||||
- set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380"
|
- set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380"
|
||||||
|
|
Loading…
Reference in a new issue