Merge pull request #109 from rackn/awsify

Add variables and defaults for multiple types of ip addresses.

inventory/group_vars/all.yml

@@ -27,6 +27,28 @@ cluster_name: cluster.local
 # set this variable to calico if needed. keep it empty if flannel is used
 kube_network_plugin: calico
 
+# For some environments, each node has a pubilcally accessible
+# address and an address it should bind services to.  These are
+# really inventory level variables, but described here for consistency.
+#
+# When advertising access, the access_ip will be used, but will defer to
+# ip and then the default ansible ip when unspecified.
+#
+# When binding to restrict access, the ip variable will be used, but will
+# defer to the default ansible ip when unspecified.
+#
+# The ip variable is used for specific address binding, e.g. listen address
+# for etcd.  This is use to help with environments like Vagrant or multi-nic
+# systems where one address should be preferred over another.
+# ip: 10.2.2.2
+#
+# The access_ip variable is used to define how other nodes should access
+# the node.  This is used in flannel to allow other flannel nodes to see
+# this node for example.  The access_ip is really useful AWS and Google
+# environments where the nodes are accessed remotely by the "public" ip,
+# but don't know about that address themselves.
+# access_ip: 1.1.1.1
+
 # Kubernetes internal network for services, unused block of space.
 kube_service_addresses: 10.233.0.0/18
 

roles/etcd/templates/etcd.j2

@@ -6,12 +6,12 @@ ETCD_DATA_DIR="/var/lib/etcd"
 {%             set _dummy = etcd.update({'name':"etcd"+loop.index|string}) %}
 {%         endif %}
 {%     endfor %}
-ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379"
-ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2380"
+ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)) }}:2379"
+ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address))  }}:2380"
 ETCD_INITIAL_CLUSTER_STATE="new"
 ETCD_INITIAL_CLUSTER_TOKEN="k8s_etcd"
 ETCD_LISTEN_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2380"
 ETCD_NAME="{{ etcd.name }}"
 {% endif %}
-ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}=http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
+ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}=http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
 ETCD_LISTEN_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address)  }}:2379,http://127.0.0.1:2379"

roles/kubernetes/master/templates/kube-apiserver.j2

@@ -24,7 +24,7 @@ KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{
 KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}"
 
 # Location of the etcd cluster
-KUBE_ETCD_SERVERS="--etcd_servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2379{% if not loop.last %},{% endif %}{% endfor %}"
+KUBE_ETCD_SERVERS="--etcd_servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %}"
 
 # default admission control policies
 KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -10,7 +10,7 @@ spec:
     command:
     - /hyperkube
     - apiserver
-    - --etcd-servers={% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %}
+    - --etcd-servers={% for srv in groups['etcd'] %}http://{{ hostvars[srv]['access_ip'] | default(hostvars[srv]['ip']|default(hostvars[srv]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %}
 
     - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
     - --service-cluster-ip-range={{ kube_service_addresses }}

roles/kubernetes/node/templates/kubelet.j2

@@ -7,7 +7,7 @@ KUBE_LOGGING="--logtostderr=true"
 {% endif %}
 KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}"
 KUBE_ALLOW_PRIV="--allow_privileged=true"
-KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}"
+KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}"
 # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
 KUBELET_ADDRESS="--address=0.0.0.0"
 # The port for the info server to serve on

roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2

@@ -18,7 +18,7 @@ spec:
 {%   if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
     - --master=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}
 {%   else %}
-    - --master=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{ kube_apiserver_port }}
+    - --master=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}
 {%   endif%}
     - --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml
 {% endif %}

roles/kubernetes/node/templates/openssl.conf.j2

@@ -14,7 +14,8 @@ DNS.3 = kubernetes.default.svc.{{ dns_domain }}
 DNS.4 = {{ apiserver_loadbalancer_domain_name }}
 {% endif %}
 {% for host in groups['kube-master'] %}
-IP.{{ loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
 {% endfor %}
-{% set idx =  groups['kube-master'] | length | int + 1 %}
+{% set idx =  groups['kube-master'] | length | int * 2 + 1 %}
 IP.{{ idx | string }} = {{ kube_apiserver_ip }}

roles/kubernetes/preinstall/tasks/etchosts.yml

@@ -2,8 +2,8 @@
 - name: Hosts | populate inventory into hosts file
   lineinfile:
     dest: /etc/hosts
-    regexp: "^{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address) }} {{ item }}$"
-    line: "{{ hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address) }} {{ item }}"
+    regexp: "^{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }}$"
+    line: "{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }}"
     state: present
     backup: yes
   when: hostvars[item].ansible_default_ipv4.address is defined

roles/network_plugin/defaults/main.yml

@@ -1,6 +1,9 @@
 ---
-## defines the IP used to talk to the node
-# flannel_public_ip:
+
+# Flannel public IP
+# The address that flannel should advertise as how to access the system
+flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}"
 
 ## interface that should be used for flannel operations
+## This is actually an inventory node-level item
 # flannel_interface:

roles/network_plugin/templates/calico/calico.conf.j2

@@ -9,7 +9,7 @@ ETCD_AUTHORITY=127.0.0.1:2379
 {% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
 KUBE_API_ROOT=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}/api/v1/
 {% else %}
-KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{kube_apiserver_port}}/api/v1/
+KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{kube_apiserver_port}}/api/v1/
 {% endif %}
 # Kubernetes authentication token
 {% if calico_token is defined | default('') %}