From a8de82e3c75446d7f03fb75485bc541c463ec0b4 Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 9 Dec 2016 13:38:38 +0400 Subject: [PATCH] Add inventory builder python script Includes tox support for running unit tests. Small note added to getting-started guide for using inventory_builder.py --- .gitignore | 5 + .travis.yml | 9 + contrib/inventory_builder/inventory.py | 239 ++++++++++++++++++ .../inventory_builder/tests/test_inventory.py | 212 ++++++++++++++++ docs/getting-started.md | 13 + requirements.txt | 1 + setup.cfg | 3 + setup.py | 31 +++ test-requirements.txt | 3 + tox.ini | 28 ++ 10 files changed, 544 insertions(+) create mode 100644 contrib/inventory_builder/inventory.py create mode 100644 contrib/inventory_builder/tests/test_inventory.py create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 test-requirements.txt create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 86dec6fb4..20412176d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,8 @@ inventory/vagrant_ansible_inventory temp .idea +.tox +.cache +*.egg-info +*.pyc +*.pyo diff --git a/.travis.yml b/.travis.yml index 058018ab5..19c73138f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,6 +81,13 @@ env: CLOUD_REGION=us-central1-f CLUSTER_MODE=separate BOOTSTRAP_OS=coreos + # Extra cases for separated roles + - >- + KUBE_NETWORK_PLUGIN=none + CLOUD_IMAGE=ubuntu-1604-xenial + CLOUD_REGION=europe-west1-a + CLUSTER_MODE=tox + matrix: allow_failures: @@ -115,6 +122,8 @@ before_script: # - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}' $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml" script: + - > + if [ ${CLUSTER_MODE} = "tox" ]; then pip install --user tox; tox -e py27,pep8; fi; exit $? - > $HOME/.local/bin/ansible-playbook tests/cloud_playbooks/create-gce.yml -i tests/local_inventory/hosts.cfg -c local $LOG_LEVEL -e mode=${CLUSTER_MODE} diff --git a/contrib/inventory_builder/inventory.py b/contrib/inventory_builder/inventory.py new file mode 100644 index 000000000..6ffe1a184 --- /dev/null +++ b/contrib/inventory_builder/inventory.py @@ -0,0 +1,239 @@ +#!/usr/bin/python3 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Usage: inventory.py ip1 [ip2 ...] +# Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5 +# +# Advanced usage: +# Add another host after initial creation: inventory.py 10.10.1.5 +# Delete a host: inventory.py -10.10.1.3 +# Delete a host by id: inventory.py -node1 + +from collections import OrderedDict +try: + import configparser +except ImportError: + import ConfigParser as configparser + +import os +import re +import sys + +ROLES = ['kube-master', 'all', 'k8s-cluster:children', 'kube-node', 'etcd'] +PROTECTED_NAMES = ROLES +AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips'] +_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True, + '0': False, 'no': False, 'false': False, 'off': False} + + +def get_var_as_bool(name, default): + value = os.environ.get(name, '') + return _boolean_states.get(value.lower(), default) + +CONFIG_FILE = os.environ.get("CONFIG_FILE", "./inventory.cfg") +DEBUG = get_var_as_bool("DEBUG", True) +HOST_PREFIX = os.environ.get("HOST_PREFIX", "node") + + +class KargoInventory(object): + + def __init__(self, changed_hosts=None, config_file=None): + self.config = configparser.ConfigParser(allow_no_value=True, + delimiters=('\t', ' ')) + if config_file: + self.config.read(config_file) + + if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS: + self.parse_command(changed_hosts[0], changed_hosts[1:]) + sys.exit(0) + + self.ensure_required_groups(ROLES) + + if changed_hosts: + self.hosts = self.build_hostnames(changed_hosts) + self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES) + self.set_kube_master(list(self.hosts.keys())[:2]) + self.set_all(self.hosts) + self.set_k8s_cluster() + self.set_kube_node(self.hosts.keys()) + self.set_etcd(list(self.hosts.keys())[:3]) + else: # Show help if no options + self.show_help() + sys.exit(0) + + if config_file: + with open(config_file, 'w') as f: + self.config.write(f) + + def debug(self, msg): + if DEBUG: + print("DEBUG: {0}".format(msg)) + + def get_ip_from_opts(self, optstring): + opts = optstring.split(' ') + for opt in opts: + if '=' not in opt: + continue + k, v = opt.split('=') + if k == "ip": + return v + raise ValueError("IP parameter not found in options") + + def ensure_required_groups(self, groups): + for group in groups: + try: + self.config.add_section(group) + except configparser.DuplicateSectionError: + pass + + def get_host_id(self, host): + '''Returns integer host ID (without padding) from a given hostname.''' + try: + short_hostname = host.split('.')[0] + return int(re.findall("\d+$", short_hostname)[-1]) + except IndexError: + raise ValueError("Host name must end in an integer") + + def build_hostnames(self, changed_hosts): + existing_hosts = OrderedDict() + highest_host_id = 0 + try: + for host, opts in self.config.items('all'): + existing_hosts[host] = opts + host_id = self.get_host_id(host) + if host_id > highest_host_id: + highest_host_id = host_id + except configparser.NoSectionError: + pass + + # FIXME(mattymo): Fix condition where delete then add reuses highest id + next_host_id = highest_host_id + 1 + + all_hosts = existing_hosts.copy() + for host in changed_hosts: + if host[0] == "-": + realhost = host[1:] + if self.exists_hostname(all_hosts, realhost): + self.debug("Marked {0} for deletion.".format(realhost)) + all_hosts.pop(realhost) + elif self.exists_ip(all_hosts, realhost): + self.debug("Marked {0} for deletion.".format(realhost)) + self.delete_host_by_ip(all_hosts, realhost) + elif host[0].isdigit(): + if self.exists_hostname(all_hosts, host): + self.debug("Skipping existing host {0}.".format(host)) + continue + elif self.exists_ip(all_hosts, host): + self.debug("Skipping existing host {0}.".format(host)) + continue + + next_host = "{0}{1}".format(HOST_PREFIX, next_host_id) + next_host_id += 1 + all_hosts[next_host] = "ansible_host={0} ip={1}".format( + host, host) + elif host[0].isalpha(): + raise Exception("Adding hosts by hostname is not supported.") + + return all_hosts + + def exists_hostname(self, existing_hosts, hostname): + return hostname in existing_hosts.keys() + + def exists_ip(self, existing_hosts, ip): + for host_opts in existing_hosts.values(): + if ip == self.get_ip_from_opts(host_opts): + return True + return False + + def delete_host_by_ip(self, existing_hosts, ip): + for hostname, host_opts in existing_hosts.items(): + if ip == self.get_ip_from_opts(host_opts): + del existing_hosts[hostname] + return + raise ValueError("Unable to find host by IP: {0}".format(ip)) + + def purge_invalid_hosts(self, hostnames, protected_names=[]): + for role in self.config.sections(): + for host, _ in self.config.items(role): + if host not in hostnames and host not in protected_names: + self.debug("Host {0} removed from role {1}".format(host, + role)) + self.config.remove_option(role, host) + + def add_host_to_group(self, group, host, opts=""): + self.debug("adding host {0} to group {1}".format(host, group)) + self.config.set(group, host, opts) + + def set_kube_master(self, hosts): + for host in hosts: + self.add_host_to_group('kube-master', host) + + def set_all(self, hosts): + for host, opts in hosts.items(): + self.add_host_to_group('all', host, opts) + + def set_k8s_cluster(self): + self.add_host_to_group('k8s-cluster:children', 'kube-node') + self.add_host_to_group('k8s-cluster:children', 'kube-master') + + def set_kube_node(self, hosts): + for host in hosts: + self.add_host_to_group('kube-node', host) + + def set_etcd(self, hosts): + for host in hosts: + self.add_host_to_group('etcd', host) + + def parse_command(self, command, args=None): + if command == 'help': + self.show_help() + elif command == 'print_cfg': + self.print_config() + elif command == 'print_ips': + self.print_ips() + else: + raise Exception("Invalid command specified.") + + def show_help(self): + help_text = '''Usage: inventory.py ip1 [ip2 ...] +Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5 + +Available commands: +help - Display this message +print_cfg - Write inventory file to stdout +print_ips - Write a space-delimited list of IPs from "all" group + +Advanced usage: +Add another host after initial creation: inventory.py 10.10.1.5 +Delete a host: inventory.py -10.10.1.3 +Delete a host by id: inventory.py -node1''' + print(help_text) + + def print_config(self): + self.config.write(sys.stdout) + + def print_ips(self): + ips = [] + for host, opts in self.config.items('all'): + ips.append(self.get_ip_from_opts(opts)) + print(' '.join(ips)) + + +def main(argv=None): + if not argv: + argv = sys.argv[1:] + KargoInventory(argv, CONFIG_FILE) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/contrib/inventory_builder/tests/test_inventory.py b/contrib/inventory_builder/tests/test_inventory.py new file mode 100644 index 000000000..681883772 --- /dev/null +++ b/contrib/inventory_builder/tests/test_inventory.py @@ -0,0 +1,212 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import unittest + +from collections import OrderedDict +import sys + +path = "./contrib/inventory_builder/" +if path not in sys.path: + sys.path.append(path) + +import inventory + + +class TestInventory(unittest.TestCase): + @mock.patch('inventory.sys') + def setUp(self, sys_mock): + sys_mock.exit = mock.Mock() + super(TestInventory, self).setUp() + self.data = ['10.90.3.2', '10.90.3.3', '10.90.3.4'] + self.inv = inventory.KargoInventory() + + def test_get_ip_from_opts(self): + optstring = "ansible_host=10.90.3.2 ip=10.90.3.2" + expected = "10.90.3.2" + result = self.inv.get_ip_from_opts(optstring) + self.assertEqual(expected, result) + + def test_get_ip_from_opts_invalid(self): + optstring = "notanaddr=value something random!chars:D" + self.assertRaisesRegexp(ValueError, "IP parameter not found", + self.inv.get_ip_from_opts, optstring) + + def test_ensure_required_groups(self): + groups = ['group1', 'group2'] + self.inv.ensure_required_groups(groups) + for group in groups: + self.assertTrue(group in self.inv.config.sections()) + + def test_get_host_id(self): + hostnames = ['node99', 'no99de01', '01node01', 'node1.domain', + 'node3.xyz123.aaa'] + expected = [99, 1, 1, 1, 3] + for hostname, expected in zip(hostnames, expected): + result = self.inv.get_host_id(hostname) + self.assertEqual(expected, result) + + def test_get_host_id_invalid(self): + bad_hostnames = ['node', 'no99de', '01node', 'node.111111'] + for hostname in bad_hostnames: + self.assertRaisesRegexp(ValueError, "Host name must end in an", + self.inv.get_host_id, hostname) + + def test_build_hostnames_add_one(self): + changed_hosts = ['10.90.0.2'] + expected = OrderedDict([('node1', + 'ansible_host=10.90.0.2 ip=10.90.0.2')]) + result = self.inv.build_hostnames(changed_hosts) + self.assertEqual(expected, result) + + def test_build_hostnames_add_duplicate(self): + changed_hosts = ['10.90.0.2'] + expected = OrderedDict([('node1', + 'ansible_host=10.90.0.2 ip=10.90.0.2')]) + self.inv.config['all'] = expected + result = self.inv.build_hostnames(changed_hosts) + self.assertEqual(expected, result) + + def test_build_hostnames_add_two(self): + changed_hosts = ['10.90.0.2', '10.90.0.3'] + expected = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + self.inv.config['all'] = OrderedDict() + result = self.inv.build_hostnames(changed_hosts) + self.assertEqual(expected, result) + + def test_build_hostnames_delete_first(self): + changed_hosts = ['-10.90.0.2'] + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + self.inv.config['all'] = existing_hosts + expected = OrderedDict([ + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + result = self.inv.build_hostnames(changed_hosts) + self.assertEqual(expected, result) + + def test_exists_hostname_positive(self): + hostname = 'node1' + expected = True + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + result = self.inv.exists_hostname(existing_hosts, hostname) + self.assertEqual(expected, result) + + def test_exists_hostname_negative(self): + hostname = 'node99' + expected = False + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + result = self.inv.exists_hostname(existing_hosts, hostname) + self.assertEqual(expected, result) + + def test_exists_ip_positive(self): + ip = '10.90.0.2' + expected = True + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + result = self.inv.exists_ip(existing_hosts, ip) + self.assertEqual(expected, result) + + def test_exists_ip_negative(self): + ip = '10.90.0.200' + expected = False + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + result = self.inv.exists_ip(existing_hosts, ip) + self.assertEqual(expected, result) + + def test_delete_host_by_ip_positive(self): + ip = '10.90.0.2' + expected = OrderedDict([ + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + self.inv.delete_host_by_ip(existing_hosts, ip) + self.assertEqual(expected, existing_hosts) + + def test_delete_host_by_ip_negative(self): + ip = '10.90.0.200' + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + self.assertRaisesRegexp(ValueError, "Unable to find host", + self.inv.delete_host_by_ip, existing_hosts, ip) + + def test_purge_invalid_hosts(self): + proper_hostnames = ['node1', 'node2'] + bad_host = 'doesnotbelong2' + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3'), + ('doesnotbelong2', 'whateveropts=ilike')]) + self.inv.config['all'] = existing_hosts + self.inv.purge_invalid_hosts(proper_hostnames) + self.assertTrue(bad_host not in self.inv.config['all'].keys()) + + def test_add_host_to_group(self): + group = 'etcd' + host = 'node1' + opts = 'ip=10.90.0.2' + + self.inv.add_host_to_group(group, host, opts) + self.assertEqual(self.inv.config[group].get(host), opts) + + def test_set_kube_master(self): + group = 'kube-master' + host = 'node1' + + self.inv.set_kube_master([host]) + self.assertTrue(host in self.inv.config[group]) + + def test_set_all(self): + group = 'all' + hosts = OrderedDict([ + ('node1', 'opt1'), + ('node2', 'opt2')]) + + self.inv.set_all(hosts) + for host, opt in hosts.items(): + self.assertEqual(self.inv.config[group].get(host), opt) + + def test_set_k8s_cluster(self): + group = 'k8s-cluster:children' + expected_hosts = ['kube-node', 'kube-master'] + + self.inv.set_k8s_cluster() + for host in expected_hosts: + self.assertTrue(host in self.inv.config[group]) + + def test_set_kube_node(self): + group = 'kube-node' + host = 'node1' + + self.inv.set_kube_node([host]) + self.assertTrue(host in self.inv.config[group]) + + def test_set_etcd(self): + group = 'etcd' + host = 'node1' + + self.inv.set_etcd([host]) + self.assertTrue(host in self.inv.config[group]) diff --git a/docs/getting-started.md b/docs/getting-started.md index 153c91a12..b912f0420 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -17,3 +17,16 @@ kargo aws --instances 3 ``` kargo deploy --aws -u centos -n calico ``` + +Building your own inventory +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Ansible inventory can be stored in 3 formats: YAML, JSON, or inifile. There is +an example inventory located +[here](https://github.com/kubernetes-incubator/kargo/blob/master/inventory/inventory.example). + +You can use an +[inventory generator](https://github.com/kubernetes-incubator/kargo/blob/master/contrib/inventory_generator/inventory_generator.py) +to create or modify an Ansible inventory. Currently, it is limited in +functionality and is only use for making a basic Kargo cluster, but it does +support creating large clusters. diff --git a/requirements.txt b/requirements.txt index 74b24ce8c..c80b40bee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ ansible netaddr +configparser>=3.3.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..aca7d2628 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[metadata] +name = kargo-notapackage +version = 0.1 diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..87c0a1060 --- /dev/null +++ b/setup.py @@ -0,0 +1,31 @@ +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT +import setuptools + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + +setuptools.setup( + setup_requires=[], + pbr=False) +# setup_requires=['pbr'], +# pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 000000000..4e334a094 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,3 @@ +hacking>=0.10.2 +pytest>=2.8.0 +mock>=1.3.0 diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..ed4125ab8 --- /dev/null +++ b/tox.ini @@ -0,0 +1,28 @@ +[tox] +minversion = 1.6 +skipsdist = True +envlist = pep8, py27 + +[testenv] +whitelist_externals = py.test +usedevelop = True +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +setenv = VIRTUAL_ENV={envdir} +passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY +commands = py.test -vv #{posargs:contrib/*/tests} + +[testenv:pep8] +usedevelop = False +whitelist_externals = bash +commands = + bash -c "find {toxinidir}/* -type f -name '*.py' -print0 | xargs -0 flake8" + +[testenv:venv] +commands = {posargs} + +[flake8] +show-source = true +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,roles/dnsmasq/library/kube.py,contrib/terraform/terraform.py,scripts/change_k8s_version.py,roles/kubernetes-apps/kpm/library/kpm.py,roles/kubernetes-apps/lib/library/kube.py,roles/bootstrap-os/files/get-pip.py