From 7b86b87dcac0c7ff459971edc43018b570c0625d Mon Sep 17 00:00:00 2001 From: Matthew Mosesohn Date: Fri, 9 Dec 2016 13:38:38 +0400 Subject: [PATCH] Add inventory builder python script Includes tox support for running unit tests. Small note added to getting-started guide for using inventory_builder.py Also adds manual-only unit test. --- .gitignore | 7 +- .gitlab-ci.yml | 9 + contrib/inventory_builder/inventory.py | 239 ++++++++++++++++++ contrib/inventory_builder/requirements.txt | 1 + .../inventory_builder/requirements.yml | 0 contrib/inventory_builder/setup.cfg | 3 + contrib/inventory_builder/setup.py | 29 +++ .../inventory_builder/test-requirements.txt | 3 + .../inventory_builder/tests/test_inventory.py | 212 ++++++++++++++++ contrib/inventory_builder/tox.ini | 28 ++ docs/getting-started.md | 13 + 11 files changed, 543 insertions(+), 1 deletion(-) create mode 100644 contrib/inventory_builder/inventory.py create mode 100644 contrib/inventory_builder/requirements.txt rename requirements.yml => contrib/inventory_builder/requirements.yml (100%) create mode 100644 contrib/inventory_builder/setup.cfg create mode 100644 contrib/inventory_builder/setup.py create mode 100644 contrib/inventory_builder/test-requirements.txt create mode 100644 contrib/inventory_builder/tests/test_inventory.py create mode 100644 contrib/inventory_builder/tox.ini diff --git a/.gitignore b/.gitignore index 8eae4884b..908b3c11e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,11 @@ inventory/vagrant_ansible_inventory temp .idea +.tox +.cache +*.egg-info +*.pyc +*.pyo *.tfstate *.tfstate.backup -/ssh-bastion.conf \ No newline at end of file +/ssh-bastion.conf diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 94d0ed2b9..ddef40d4d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -363,3 +363,12 @@ syntax-check: script: - ansible-playbook -i inventory/local-tests.cfg -u root -e ansible_ssh_user=root -b --become-user=root cluster.yml -vvv --syntax-check except: ['triggers'] + +tox-inventory-builder: + stage: unit-tests + <<: *job + script: + - pip install tox + - cd contrib/inventory_builder && tox + when: manual + except: ['triggers'] diff --git a/contrib/inventory_builder/inventory.py b/contrib/inventory_builder/inventory.py new file mode 100644 index 000000000..6ffe1a184 --- /dev/null +++ b/contrib/inventory_builder/inventory.py @@ -0,0 +1,239 @@ +#!/usr/bin/python3 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Usage: inventory.py ip1 [ip2 ...] +# Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5 +# +# Advanced usage: +# Add another host after initial creation: inventory.py 10.10.1.5 +# Delete a host: inventory.py -10.10.1.3 +# Delete a host by id: inventory.py -node1 + +from collections import OrderedDict +try: + import configparser +except ImportError: + import ConfigParser as configparser + +import os +import re +import sys + +ROLES = ['kube-master', 'all', 'k8s-cluster:children', 'kube-node', 'etcd'] +PROTECTED_NAMES = ROLES +AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips'] +_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True, + '0': False, 'no': False, 'false': False, 'off': False} + + +def get_var_as_bool(name, default): + value = os.environ.get(name, '') + return _boolean_states.get(value.lower(), default) + +CONFIG_FILE = os.environ.get("CONFIG_FILE", "./inventory.cfg") +DEBUG = get_var_as_bool("DEBUG", True) +HOST_PREFIX = os.environ.get("HOST_PREFIX", "node") + + +class KargoInventory(object): + + def __init__(self, changed_hosts=None, config_file=None): + self.config = configparser.ConfigParser(allow_no_value=True, + delimiters=('\t', ' ')) + if config_file: + self.config.read(config_file) + + if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS: + self.parse_command(changed_hosts[0], changed_hosts[1:]) + sys.exit(0) + + self.ensure_required_groups(ROLES) + + if changed_hosts: + self.hosts = self.build_hostnames(changed_hosts) + self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES) + self.set_kube_master(list(self.hosts.keys())[:2]) + self.set_all(self.hosts) + self.set_k8s_cluster() + self.set_kube_node(self.hosts.keys()) + self.set_etcd(list(self.hosts.keys())[:3]) + else: # Show help if no options + self.show_help() + sys.exit(0) + + if config_file: + with open(config_file, 'w') as f: + self.config.write(f) + + def debug(self, msg): + if DEBUG: + print("DEBUG: {0}".format(msg)) + + def get_ip_from_opts(self, optstring): + opts = optstring.split(' ') + for opt in opts: + if '=' not in opt: + continue + k, v = opt.split('=') + if k == "ip": + return v + raise ValueError("IP parameter not found in options") + + def ensure_required_groups(self, groups): + for group in groups: + try: + self.config.add_section(group) + except configparser.DuplicateSectionError: + pass + + def get_host_id(self, host): + '''Returns integer host ID (without padding) from a given hostname.''' + try: + short_hostname = host.split('.')[0] + return int(re.findall("\d+$", short_hostname)[-1]) + except IndexError: + raise ValueError("Host name must end in an integer") + + def build_hostnames(self, changed_hosts): + existing_hosts = OrderedDict() + highest_host_id = 0 + try: + for host, opts in self.config.items('all'): + existing_hosts[host] = opts + host_id = self.get_host_id(host) + if host_id > highest_host_id: + highest_host_id = host_id + except configparser.NoSectionError: + pass + + # FIXME(mattymo): Fix condition where delete then add reuses highest id + next_host_id = highest_host_id + 1 + + all_hosts = existing_hosts.copy() + for host in changed_hosts: + if host[0] == "-": + realhost = host[1:] + if self.exists_hostname(all_hosts, realhost): + self.debug("Marked {0} for deletion.".format(realhost)) + all_hosts.pop(realhost) + elif self.exists_ip(all_hosts, realhost): + self.debug("Marked {0} for deletion.".format(realhost)) + self.delete_host_by_ip(all_hosts, realhost) + elif host[0].isdigit(): + if self.exists_hostname(all_hosts, host): + self.debug("Skipping existing host {0}.".format(host)) + continue + elif self.exists_ip(all_hosts, host): + self.debug("Skipping existing host {0}.".format(host)) + continue + + next_host = "{0}{1}".format(HOST_PREFIX, next_host_id) + next_host_id += 1 + all_hosts[next_host] = "ansible_host={0} ip={1}".format( + host, host) + elif host[0].isalpha(): + raise Exception("Adding hosts by hostname is not supported.") + + return all_hosts + + def exists_hostname(self, existing_hosts, hostname): + return hostname in existing_hosts.keys() + + def exists_ip(self, existing_hosts, ip): + for host_opts in existing_hosts.values(): + if ip == self.get_ip_from_opts(host_opts): + return True + return False + + def delete_host_by_ip(self, existing_hosts, ip): + for hostname, host_opts in existing_hosts.items(): + if ip == self.get_ip_from_opts(host_opts): + del existing_hosts[hostname] + return + raise ValueError("Unable to find host by IP: {0}".format(ip)) + + def purge_invalid_hosts(self, hostnames, protected_names=[]): + for role in self.config.sections(): + for host, _ in self.config.items(role): + if host not in hostnames and host not in protected_names: + self.debug("Host {0} removed from role {1}".format(host, + role)) + self.config.remove_option(role, host) + + def add_host_to_group(self, group, host, opts=""): + self.debug("adding host {0} to group {1}".format(host, group)) + self.config.set(group, host, opts) + + def set_kube_master(self, hosts): + for host in hosts: + self.add_host_to_group('kube-master', host) + + def set_all(self, hosts): + for host, opts in hosts.items(): + self.add_host_to_group('all', host, opts) + + def set_k8s_cluster(self): + self.add_host_to_group('k8s-cluster:children', 'kube-node') + self.add_host_to_group('k8s-cluster:children', 'kube-master') + + def set_kube_node(self, hosts): + for host in hosts: + self.add_host_to_group('kube-node', host) + + def set_etcd(self, hosts): + for host in hosts: + self.add_host_to_group('etcd', host) + + def parse_command(self, command, args=None): + if command == 'help': + self.show_help() + elif command == 'print_cfg': + self.print_config() + elif command == 'print_ips': + self.print_ips() + else: + raise Exception("Invalid command specified.") + + def show_help(self): + help_text = '''Usage: inventory.py ip1 [ip2 ...] +Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5 + +Available commands: +help - Display this message +print_cfg - Write inventory file to stdout +print_ips - Write a space-delimited list of IPs from "all" group + +Advanced usage: +Add another host after initial creation: inventory.py 10.10.1.5 +Delete a host: inventory.py -10.10.1.3 +Delete a host by id: inventory.py -node1''' + print(help_text) + + def print_config(self): + self.config.write(sys.stdout) + + def print_ips(self): + ips = [] + for host, opts in self.config.items('all'): + ips.append(self.get_ip_from_opts(opts)) + print(' '.join(ips)) + + +def main(argv=None): + if not argv: + argv = sys.argv[1:] + KargoInventory(argv, CONFIG_FILE) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/contrib/inventory_builder/requirements.txt b/contrib/inventory_builder/requirements.txt new file mode 100644 index 000000000..fa76f1c94 --- /dev/null +++ b/contrib/inventory_builder/requirements.txt @@ -0,0 +1 @@ +configparser>=3.3.0 diff --git a/requirements.yml b/contrib/inventory_builder/requirements.yml similarity index 100% rename from requirements.yml rename to contrib/inventory_builder/requirements.yml diff --git a/contrib/inventory_builder/setup.cfg b/contrib/inventory_builder/setup.cfg new file mode 100644 index 000000000..a09927305 --- /dev/null +++ b/contrib/inventory_builder/setup.cfg @@ -0,0 +1,3 @@ +[metadata] +name = kargo-inventory-builder +version = 0.1 diff --git a/contrib/inventory_builder/setup.py b/contrib/inventory_builder/setup.py new file mode 100644 index 000000000..43c5ca1b4 --- /dev/null +++ b/contrib/inventory_builder/setup.py @@ -0,0 +1,29 @@ +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT +import setuptools + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + +setuptools.setup( + setup_requires=[], + pbr=False) diff --git a/contrib/inventory_builder/test-requirements.txt b/contrib/inventory_builder/test-requirements.txt new file mode 100644 index 000000000..4e334a094 --- /dev/null +++ b/contrib/inventory_builder/test-requirements.txt @@ -0,0 +1,3 @@ +hacking>=0.10.2 +pytest>=2.8.0 +mock>=1.3.0 diff --git a/contrib/inventory_builder/tests/test_inventory.py b/contrib/inventory_builder/tests/test_inventory.py new file mode 100644 index 000000000..681883772 --- /dev/null +++ b/contrib/inventory_builder/tests/test_inventory.py @@ -0,0 +1,212 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import unittest + +from collections import OrderedDict +import sys + +path = "./contrib/inventory_builder/" +if path not in sys.path: + sys.path.append(path) + +import inventory + + +class TestInventory(unittest.TestCase): + @mock.patch('inventory.sys') + def setUp(self, sys_mock): + sys_mock.exit = mock.Mock() + super(TestInventory, self).setUp() + self.data = ['10.90.3.2', '10.90.3.3', '10.90.3.4'] + self.inv = inventory.KargoInventory() + + def test_get_ip_from_opts(self): + optstring = "ansible_host=10.90.3.2 ip=10.90.3.2" + expected = "10.90.3.2" + result = self.inv.get_ip_from_opts(optstring) + self.assertEqual(expected, result) + + def test_get_ip_from_opts_invalid(self): + optstring = "notanaddr=value something random!chars:D" + self.assertRaisesRegexp(ValueError, "IP parameter not found", + self.inv.get_ip_from_opts, optstring) + + def test_ensure_required_groups(self): + groups = ['group1', 'group2'] + self.inv.ensure_required_groups(groups) + for group in groups: + self.assertTrue(group in self.inv.config.sections()) + + def test_get_host_id(self): + hostnames = ['node99', 'no99de01', '01node01', 'node1.domain', + 'node3.xyz123.aaa'] + expected = [99, 1, 1, 1, 3] + for hostname, expected in zip(hostnames, expected): + result = self.inv.get_host_id(hostname) + self.assertEqual(expected, result) + + def test_get_host_id_invalid(self): + bad_hostnames = ['node', 'no99de', '01node', 'node.111111'] + for hostname in bad_hostnames: + self.assertRaisesRegexp(ValueError, "Host name must end in an", + self.inv.get_host_id, hostname) + + def test_build_hostnames_add_one(self): + changed_hosts = ['10.90.0.2'] + expected = OrderedDict([('node1', + 'ansible_host=10.90.0.2 ip=10.90.0.2')]) + result = self.inv.build_hostnames(changed_hosts) + self.assertEqual(expected, result) + + def test_build_hostnames_add_duplicate(self): + changed_hosts = ['10.90.0.2'] + expected = OrderedDict([('node1', + 'ansible_host=10.90.0.2 ip=10.90.0.2')]) + self.inv.config['all'] = expected + result = self.inv.build_hostnames(changed_hosts) + self.assertEqual(expected, result) + + def test_build_hostnames_add_two(self): + changed_hosts = ['10.90.0.2', '10.90.0.3'] + expected = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + self.inv.config['all'] = OrderedDict() + result = self.inv.build_hostnames(changed_hosts) + self.assertEqual(expected, result) + + def test_build_hostnames_delete_first(self): + changed_hosts = ['-10.90.0.2'] + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + self.inv.config['all'] = existing_hosts + expected = OrderedDict([ + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + result = self.inv.build_hostnames(changed_hosts) + self.assertEqual(expected, result) + + def test_exists_hostname_positive(self): + hostname = 'node1' + expected = True + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + result = self.inv.exists_hostname(existing_hosts, hostname) + self.assertEqual(expected, result) + + def test_exists_hostname_negative(self): + hostname = 'node99' + expected = False + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + result = self.inv.exists_hostname(existing_hosts, hostname) + self.assertEqual(expected, result) + + def test_exists_ip_positive(self): + ip = '10.90.0.2' + expected = True + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + result = self.inv.exists_ip(existing_hosts, ip) + self.assertEqual(expected, result) + + def test_exists_ip_negative(self): + ip = '10.90.0.200' + expected = False + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + result = self.inv.exists_ip(existing_hosts, ip) + self.assertEqual(expected, result) + + def test_delete_host_by_ip_positive(self): + ip = '10.90.0.2' + expected = OrderedDict([ + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + self.inv.delete_host_by_ip(existing_hosts, ip) + self.assertEqual(expected, existing_hosts) + + def test_delete_host_by_ip_negative(self): + ip = '10.90.0.200' + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + self.assertRaisesRegexp(ValueError, "Unable to find host", + self.inv.delete_host_by_ip, existing_hosts, ip) + + def test_purge_invalid_hosts(self): + proper_hostnames = ['node1', 'node2'] + bad_host = 'doesnotbelong2' + existing_hosts = OrderedDict([ + ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), + ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3'), + ('doesnotbelong2', 'whateveropts=ilike')]) + self.inv.config['all'] = existing_hosts + self.inv.purge_invalid_hosts(proper_hostnames) + self.assertTrue(bad_host not in self.inv.config['all'].keys()) + + def test_add_host_to_group(self): + group = 'etcd' + host = 'node1' + opts = 'ip=10.90.0.2' + + self.inv.add_host_to_group(group, host, opts) + self.assertEqual(self.inv.config[group].get(host), opts) + + def test_set_kube_master(self): + group = 'kube-master' + host = 'node1' + + self.inv.set_kube_master([host]) + self.assertTrue(host in self.inv.config[group]) + + def test_set_all(self): + group = 'all' + hosts = OrderedDict([ + ('node1', 'opt1'), + ('node2', 'opt2')]) + + self.inv.set_all(hosts) + for host, opt in hosts.items(): + self.assertEqual(self.inv.config[group].get(host), opt) + + def test_set_k8s_cluster(self): + group = 'k8s-cluster:children' + expected_hosts = ['kube-node', 'kube-master'] + + self.inv.set_k8s_cluster() + for host in expected_hosts: + self.assertTrue(host in self.inv.config[group]) + + def test_set_kube_node(self): + group = 'kube-node' + host = 'node1' + + self.inv.set_kube_node([host]) + self.assertTrue(host in self.inv.config[group]) + + def test_set_etcd(self): + group = 'etcd' + host = 'node1' + + self.inv.set_etcd([host]) + self.assertTrue(host in self.inv.config[group]) diff --git a/contrib/inventory_builder/tox.ini b/contrib/inventory_builder/tox.ini new file mode 100644 index 000000000..8ca254295 --- /dev/null +++ b/contrib/inventory_builder/tox.ini @@ -0,0 +1,28 @@ +[tox] +minversion = 1.6 +skipsdist = True +envlist = pep8, py27 + +[testenv] +whitelist_externals = py.test +usedevelop = True +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +setenv = VIRTUAL_ENV={envdir} +passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY +commands = py.test -vv #{posargs:./tests} + +[testenv:pep8] +usedevelop = False +whitelist_externals = bash +commands = + bash -c "find {toxinidir}/* -type f -name '*.py' -print0 | xargs -0 flake8" + +[testenv:venv] +commands = {posargs} + +[flake8] +show-source = true +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg diff --git a/docs/getting-started.md b/docs/getting-started.md index 153c91a12..b912f0420 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -17,3 +17,16 @@ kargo aws --instances 3 ``` kargo deploy --aws -u centos -n calico ``` + +Building your own inventory +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Ansible inventory can be stored in 3 formats: YAML, JSON, or inifile. There is +an example inventory located +[here](https://github.com/kubernetes-incubator/kargo/blob/master/inventory/inventory.example). + +You can use an +[inventory generator](https://github.com/kubernetes-incubator/kargo/blob/master/contrib/inventory_generator/inventory_generator.py) +to create or modify an Ansible inventory. Currently, it is limited in +functionality and is only use for making a basic Kargo cluster, but it does +support creating large clusters.