From b73f009c072146797ab55250dcad5195d70f23fd Mon Sep 17 00:00:00 2001 From: tikitavi Date: Thu, 28 Feb 2019 17:28:27 +0300 Subject: [PATCH] rewrite inventory script to create inventory file in YAML format (#4303) * rewrite inventory script to create inventory file in YAML format * minor fixes to inventory script * change requirments for the inventory script --- contrib/inventory_builder/inventory.py | 135 ++++++++------- contrib/inventory_builder/requirements.txt | 1 + .../inventory_builder/tests/test_inventory.py | 154 ++++++++++++------ 3 files changed, 183 insertions(+), 107 deletions(-) diff --git a/contrib/inventory_builder/inventory.py b/contrib/inventory_builder/inventory.py index 0878849dc..6ab805948 100644 --- a/contrib/inventory_builder/inventory.py +++ b/contrib/inventory_builder/inventory.py @@ -31,21 +31,19 @@ # ip: X.X.X.X from collections import OrderedDict -try: - import configparser -except ImportError: - import ConfigParser as configparser +from ruamel.yaml import YAML import os import re import sys -ROLES = ['all', 'kube-master', 'kube-node', 'etcd', 'k8s-cluster:children', +ROLES = ['all', 'kube-master', 'kube-node', 'etcd', 'k8s-cluster', 'calico-rr'] PROTECTED_NAMES = ROLES AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'load'] _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True, '0': False, 'no': False, 'false': False, 'off': False} +yaml = YAML() def get_var_as_bool(name, default): @@ -54,7 +52,7 @@ def get_var_as_bool(name, default): # Configurable as shell vars start -CONFIG_FILE = os.environ.get("CONFIG_FILE", "./inventory/sample/hosts.ini") +CONFIG_FILE = os.environ.get("CONFIG_FILE", "./inventory/sample/hosts.yaml") # Reconfigures cluster distribution at scale SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 50)) MASSIVE_SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 200)) @@ -68,11 +66,14 @@ HOST_PREFIX = os.environ.get("HOST_PREFIX", "node") class KubesprayInventory(object): def __init__(self, changed_hosts=None, config_file=None): - self.config = configparser.ConfigParser(allow_no_value=True, - delimiters=('\t', ' ')) self.config_file = config_file + self.yaml_config = {} if self.config_file: - self.config.read(self.config_file) + try: + self.hosts_file = open(config_file, 'r') + self.yaml_config = yaml.load(self.hosts_file) + except FileNotFoundError: + pass if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS: self.parse_command(changed_hosts[0], changed_hosts[1:]) @@ -102,8 +103,9 @@ class KubesprayInventory(object): def write_config(self, config_file): if config_file: - with open(config_file, 'w') as f: - self.config.write(f) + with open(self.config_file, 'w') as f: + yaml.dump(self.yaml_config, f) + else: print("WARNING: Unable to save config. Make sure you set " "CONFIG_FILE env var.") @@ -113,22 +115,25 @@ class KubesprayInventory(object): print("DEBUG: {0}".format(msg)) def get_ip_from_opts(self, optstring): - opts = optstring.split(' ') - for opt in opts: - if '=' not in opt: - continue - k, v = opt.split('=') - if k == "ip": - return v - raise ValueError("IP parameter not found in options") + if 'ip' in optstring: + return optstring['ip'] + else: + raise ValueError("IP parameter not found in options") def ensure_required_groups(self, groups): for group in groups: - try: + if group == 'all': self.debug("Adding group {0}".format(group)) - self.config.add_section(group) - except configparser.DuplicateSectionError: - pass + if group not in self.yaml_config: + self.yaml_config = {'all': + {'hosts': {}, + 'vars': + {'ansible_user': 'centos'}, + 'children': {}}} + else: + self.debug("Adding group {0}".format(group)) + if group not in self.yaml_config['all']['children']: + self.yaml_config['all']['children'][group] = {'hosts': None} def get_host_id(self, host): '''Returns integer host ID (without padding) from a given hostname.''' @@ -142,12 +147,12 @@ class KubesprayInventory(object): existing_hosts = OrderedDict() highest_host_id = 0 try: - for host, opts in self.config.items('all'): - existing_hosts[host] = opts + for host in self.yaml_config['all']['hosts']: + existing_hosts[host] = self.yaml_config['all']['hosts'][host] host_id = self.get_host_id(host) if host_id > highest_host_id: highest_host_id = host_id - except configparser.NoSectionError: + except Exception: pass # FIXME(mattymo): Fix condition where delete then add reuses highest id @@ -173,8 +178,9 @@ class KubesprayInventory(object): next_host = "{0}{1}".format(HOST_PREFIX, next_host_id) next_host_id += 1 - all_hosts[next_host] = "ansible_host={0} ip={1}".format( - host, host) + all_hosts[next_host] = {'ansible_host': host, + 'ip': host, + 'access_ip': host} elif host[0].isalpha(): raise Exception("Adding hosts by hostname is not supported.") @@ -217,16 +223,32 @@ class KubesprayInventory(object): raise ValueError("Unable to find host by IP: {0}".format(ip)) def purge_invalid_hosts(self, hostnames, protected_names=[]): - for role in self.config.sections(): - for host, _ in self.config.items(role): + for role in self.yaml_config['all']['children']: + if role != 'k8s-cluster' and self.yaml_config['all']['children'][role]['hosts']: + all_hosts = self.yaml_config['all']['children'][role]['hosts'].copy() + for host in all_hosts.keys(): + if host not in hostnames and host not in protected_names: + self.debug("Host {0} removed from role {1}".format(host, role)) + del self.yaml_config['all']['children'][role]['hosts'][host] + # purge from all + if self.yaml_config['all']['hosts']: + all_hosts = self.yaml_config['all']['hosts'].copy() + for host in all_hosts.keys(): if host not in hostnames and host not in protected_names: - self.debug("Host {0} removed from role {1}".format(host, - role)) - self.config.remove_option(role, host) + self.debug("Host {0} removed from role all") + del self.yaml_config['all']['hosts'][host] def add_host_to_group(self, group, host, opts=""): self.debug("adding host {0} to group {1}".format(host, group)) - self.config.set(group, host, opts) + if group == 'all': + if self.yaml_config['all']['hosts'] is None: + self.yaml_config['all']['hosts'] = {host: None} + self.yaml_config['all']['hosts'][host] = opts + elif group != 'k8s-cluster:children': + if self.yaml_config['all']['children'][group]['hosts'] is None: + self.yaml_config['all']['children'][group]['hosts'] = {host: None} + else: + self.yaml_config['all']['children'][group]['hosts'][host] = None def set_kube_master(self, hosts): for host in hosts: @@ -237,16 +259,17 @@ class KubesprayInventory(object): self.add_host_to_group('all', host, opts) def set_k8s_cluster(self): - self.add_host_to_group('k8s-cluster:children', 'kube-node') - self.add_host_to_group('k8s-cluster:children', 'kube-master') + self.yaml_config['all']['children']['k8s-cluster'] = {'children': + {'kube-master': None, + 'kube-node': None}} def set_calico_rr(self, hosts): for host in hosts: - if host in self.config.items('kube-master'): + if host in self.yaml_config['all']['children']['kube-master']: self.debug("Not adding {0} to calico-rr group because it " "conflicts with kube-master group".format(host)) continue - if host in self.config.items('kube-node'): + if host in self.yaml_config['all']['children']['kube-node']: self.debug("Not adding {0} to calico-rr group because it " "conflicts with kube-node group".format(host)) continue @@ -254,14 +277,14 @@ class KubesprayInventory(object): def set_kube_node(self, hosts): for host in hosts: - if len(self.config['all']) >= SCALE_THRESHOLD: - if self.config.has_option('etcd', host): + if len(self.yaml_config['all']['hosts']) >= SCALE_THRESHOLD: + if host in self.yaml_config['all']['children']['etcd']['hosts']: self.debug("Not adding {0} to kube-node group because of " "scale deployment and host is in etcd " "group.".format(host)) continue - if len(self.config['all']) >= MASSIVE_SCALE_THRESHOLD: - if self.config.has_option('kube-master', host): + if len(self.yaml_config['all']['hosts']) >= MASSIVE_SCALE_THRESHOLD: + if host in self.yaml_config['all']['children']['kube-master']['hosts']: self.debug("Not adding {0} to kube-node group because of " "scale deployment and host is in kube-master " "group.".format(host)) @@ -273,39 +296,29 @@ class KubesprayInventory(object): self.add_host_to_group('etcd', host) def load_file(self, files=None): - '''Directly loads JSON, or YAML file to inventory.''' + '''Directly loads JSON to inventory.''' if not files: raise Exception("No input file specified.") import json - import yaml for filename in list(files): - # Try JSON, then YAML + # Try JSON try: with open(filename, 'r') as f: data = json.load(f) except ValueError: - try: - with open(filename, 'r') as f: - data = yaml.load(f) - print("yaml") - except ValueError: - raise Exception("Cannot read %s as JSON, YAML, or CSV", - filename) + raise Exception("Cannot read %s as JSON, or CSV", filename) self.ensure_required_groups(ROLES) self.set_k8s_cluster() for group, hosts in data.items(): self.ensure_required_groups([group]) for host, opts in hosts.items(): - optstring = "ansible_host={0} ip={0}".format(opts['ip']) - for key, val in opts.items(): - if key == "ip": - continue - optstring += " {0}={1}".format(key, val) - + optstring = {'ansible_host': opts['ip'], + 'ip': opts['ip'], + 'access_ip': opts['ip']} self.add_host_to_group('all', host, optstring) self.add_host_to_group(group, host) self.write_config(self.config_file) @@ -338,7 +351,7 @@ Delete a host by id: inventory.py -node1 Configurable env vars: DEBUG Enable debug printing. Default: True -CONFIG_FILE File to write config to Default: ./inventory/sample/hosts.ini +CONFIG_FILE File to write config to Default: ./inventory/sample/hosts.yaml HOST_PREFIX Host prefix for generated hosts. Default: node SCALE_THRESHOLD Separate ETCD role if # of nodes >= 50 MASSIVE_SCALE_THRESHOLD Separate K8s master and ETCD if # of nodes >= 200 @@ -346,11 +359,11 @@ MASSIVE_SCALE_THRESHOLD Separate K8s master and ETCD if # of nodes >= 200 print(help_text) def print_config(self): - self.config.write(sys.stdout) + yaml.dump(self.yaml_config, sys.stdout) def print_ips(self): ips = [] - for host, opts in self.config.items('all'): + for host, opts in self.yaml_config['all']['hosts'].items(): ips.append(self.get_ip_from_opts(opts)) print(' '.join(ips)) diff --git a/contrib/inventory_builder/requirements.txt b/contrib/inventory_builder/requirements.txt index fa76f1c94..1f22ba400 100644 --- a/contrib/inventory_builder/requirements.txt +++ b/contrib/inventory_builder/requirements.txt @@ -1 +1,2 @@ configparser>=3.3.0 +ruamel.yaml>=0.15.88 diff --git a/contrib/inventory_builder/tests/test_inventory.py b/contrib/inventory_builder/tests/test_inventory.py index cb108c821..6a5f1448a 100644 --- a/contrib/inventory_builder/tests/test_inventory.py +++ b/contrib/inventory_builder/tests/test_inventory.py @@ -34,7 +34,9 @@ class TestInventory(unittest.TestCase): self.inv = inventory.KubesprayInventory() def test_get_ip_from_opts(self): - optstring = "ansible_host=10.90.3.2 ip=10.90.3.2" + optstring = {'ansible_host': '10.90.3.2', + 'ip': '10.90.3.2', + 'access_ip': '10.90.3.2'} expected = "10.90.3.2" result = self.inv.get_ip_from_opts(optstring) self.assertEqual(expected, result) @@ -48,7 +50,7 @@ class TestInventory(unittest.TestCase): groups = ['group1', 'group2'] self.inv.ensure_required_groups(groups) for group in groups: - self.assertTrue(group in self.inv.config.sections()) + self.assertTrue(group in self.inv.yaml_config['all']['children']) def test_get_host_id(self): hostnames = ['node99', 'no99de01', '01node01', 'node1.domain', @@ -67,35 +69,49 @@ class TestInventory(unittest.TestCase): def test_build_hostnames_add_one(self): changed_hosts = ['10.90.0.2'] expected = OrderedDict([('node1', - 'ansible_host=10.90.0.2 ip=10.90.0.2')]) + {'ansible_host': '10.90.0.2', + 'ip': '10.90.0.2', + 'access_ip': '10.90.0.2'})]) result = self.inv.build_hostnames(changed_hosts) self.assertEqual(expected, result) def test_build_hostnames_add_duplicate(self): changed_hosts = ['10.90.0.2'] expected = OrderedDict([('node1', - 'ansible_host=10.90.0.2 ip=10.90.0.2')]) - self.inv.config['all'] = expected + {'ansible_host': '10.90.0.2', + 'ip': '10.90.0.2', + 'access_ip': '10.90.0.2'})]) + self.inv.yaml_config['all']['hosts'] = expected result = self.inv.build_hostnames(changed_hosts) self.assertEqual(expected, result) def test_build_hostnames_add_two(self): changed_hosts = ['10.90.0.2', '10.90.0.3'] expected = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) - self.inv.config['all'] = OrderedDict() + ('node1', {'ansible_host': '10.90.0.2', + 'ip': '10.90.0.2', + 'access_ip': '10.90.0.2'}), + ('node2', {'ansible_host': '10.90.0.3', + 'ip': '10.90.0.3', + 'access_ip': '10.90.0.3'})]) + self.inv.yaml_config['all']['hosts'] = OrderedDict() result = self.inv.build_hostnames(changed_hosts) self.assertEqual(expected, result) def test_build_hostnames_delete_first(self): changed_hosts = ['-10.90.0.2'] existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) - self.inv.config['all'] = existing_hosts + ('node1', {'ansible_host': '10.90.0.2', + 'ip': '10.90.0.2', + 'access_ip': '10.90.0.2'}), + ('node2', {'ansible_host': '10.90.0.3', + 'ip': '10.90.0.3', + 'access_ip': '10.90.0.3'})]) + self.inv.yaml_config['all']['hosts'] = existing_hosts expected = OrderedDict([ - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + ('node2', {'ansible_host': '10.90.0.3', + 'ip': '10.90.0.3', + 'access_ip': '10.90.0.3'})]) result = self.inv.build_hostnames(changed_hosts) self.assertEqual(expected, result) @@ -103,8 +119,12 @@ class TestInventory(unittest.TestCase): hostname = 'node1' expected = True existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + ('node1', {'ansible_host': '10.90.0.2', + 'ip': '10.90.0.2', + 'access_ip': '10.90.0.2'}), + ('node2', {'ansible_host': '10.90.0.3', + 'ip': '10.90.0.3', + 'access_ip': '10.90.0.3'})]) result = self.inv.exists_hostname(existing_hosts, hostname) self.assertEqual(expected, result) @@ -112,8 +132,12 @@ class TestInventory(unittest.TestCase): hostname = 'node99' expected = False existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + ('node1', {'ansible_host': '10.90.0.2', + 'ip': '10.90.0.2', + 'access_ip': '10.90.0.2'}), + ('node2', {'ansible_host': '10.90.0.3', + 'ip': '10.90.0.3', + 'access_ip': '10.90.0.3'})]) result = self.inv.exists_hostname(existing_hosts, hostname) self.assertEqual(expected, result) @@ -121,8 +145,12 @@ class TestInventory(unittest.TestCase): ip = '10.90.0.2' expected = True existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + ('node1', {'ansible_host': '10.90.0.2', + 'ip': '10.90.0.2', + 'access_ip': '10.90.0.2'}), + ('node2', {'ansible_host': '10.90.0.3', + 'ip': '10.90.0.3', + 'access_ip': '10.90.0.3'})]) result = self.inv.exists_ip(existing_hosts, ip) self.assertEqual(expected, result) @@ -130,26 +158,40 @@ class TestInventory(unittest.TestCase): ip = '10.90.0.200' expected = False existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + ('node1', {'ansible_host': '10.90.0.2', + 'ip': '10.90.0.2', + 'access_ip': '10.90.0.2'}), + ('node2', {'ansible_host': '10.90.0.3', + 'ip': '10.90.0.3', + 'access_ip': '10.90.0.3'})]) result = self.inv.exists_ip(existing_hosts, ip) self.assertEqual(expected, result) def test_delete_host_by_ip_positive(self): ip = '10.90.0.2' expected = OrderedDict([ - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + ('node2', {'ansible_host': '10.90.0.3', + 'ip': '10.90.0.3', + 'access_ip': '10.90.0.3'})]) existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + ('node1', {'ansible_host': '10.90.0.2', + 'ip': '10.90.0.2', + 'access_ip': '10.90.0.2'}), + ('node2', {'ansible_host': '10.90.0.3', + 'ip': '10.90.0.3', + 'access_ip': '10.90.0.3'})]) self.inv.delete_host_by_ip(existing_hosts, ip) self.assertEqual(expected, existing_hosts) def test_delete_host_by_ip_negative(self): ip = '10.90.0.200' existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3')]) + ('node1', {'ansible_host': '10.90.0.2', + 'ip': '10.90.0.2', + 'access_ip': '10.90.0.2'}), + ('node2', {'ansible_host': '10.90.0.3', + 'ip': '10.90.0.3', + 'access_ip': '10.90.0.3'})]) self.assertRaisesRegexp(ValueError, "Unable to find host", self.inv.delete_host_by_ip, existing_hosts, ip) @@ -157,59 +199,71 @@ class TestInventory(unittest.TestCase): proper_hostnames = ['node1', 'node2'] bad_host = 'doesnotbelong2' existing_hosts = OrderedDict([ - ('node1', 'ansible_host=10.90.0.2 ip=10.90.0.2'), - ('node2', 'ansible_host=10.90.0.3 ip=10.90.0.3'), - ('doesnotbelong2', 'whateveropts=ilike')]) - self.inv.config['all'] = existing_hosts + ('node1', {'ansible_host': '10.90.0.2', + 'ip': '10.90.0.2', + 'access_ip': '10.90.0.2'}), + ('node2', {'ansible_host': '10.90.0.3', + 'ip': '10.90.0.3', + 'access_ip': '10.90.0.3'}), + ('doesnotbelong2', {'whateveropts=ilike'})]) + self.inv.yaml_config['all']['hosts'] = existing_hosts self.inv.purge_invalid_hosts(proper_hostnames) - self.assertTrue(bad_host not in self.inv.config['all'].keys()) + self.assertTrue( + bad_host not in self.inv.yaml_config['all']['hosts'].keys()) def test_add_host_to_group(self): group = 'etcd' host = 'node1' - opts = 'ip=10.90.0.2' + opts = {'ip': '10.90.0.2'} self.inv.add_host_to_group(group, host, opts) - self.assertEqual(self.inv.config[group].get(host), opts) + self.assertEqual( + self.inv.yaml_config['all']['children'][group]['hosts'].get(host), + None) def test_set_kube_master(self): group = 'kube-master' host = 'node1' self.inv.set_kube_master([host]) - self.assertTrue(host in self.inv.config[group]) + self.assertTrue( + host in self.inv.yaml_config['all']['children'][group]['hosts']) def test_set_all(self): - group = 'all' hosts = OrderedDict([ ('node1', 'opt1'), ('node2', 'opt2')]) self.inv.set_all(hosts) for host, opt in hosts.items(): - self.assertEqual(self.inv.config[group].get(host), opt) + self.assertEqual( + self.inv.yaml_config['all']['hosts'].get(host), opt) def test_set_k8s_cluster(self): - group = 'k8s-cluster:children' + group = 'k8s-cluster' expected_hosts = ['kube-node', 'kube-master'] self.inv.set_k8s_cluster() for host in expected_hosts: - self.assertTrue(host in self.inv.config[group]) + self.assertTrue( + host in + self.inv.yaml_config['all']['children'][group]['children']) def test_set_kube_node(self): group = 'kube-node' host = 'node1' self.inv.set_kube_node([host]) - self.assertTrue(host in self.inv.config[group]) + self.assertTrue( + host in self.inv.yaml_config['all']['children'][group]['hosts']) def test_set_etcd(self): group = 'etcd' host = 'node1' self.inv.set_etcd([host]) - self.assertTrue(host in self.inv.config[group]) + self.assertTrue( + host in self.inv.yaml_config['all']['children'][group]['hosts']) def test_scale_scenario_one(self): num_nodes = 50 @@ -219,11 +273,13 @@ class TestInventory(unittest.TestCase): hosts["node" + str(hostid)] = "" self.inv.set_all(hosts) - self.inv.set_etcd(hosts.keys()[0:3]) - self.inv.set_kube_master(hosts.keys()[0:2]) + self.inv.set_etcd(list(hosts.keys())[0:3]) + self.inv.set_kube_master(list(hosts.keys())[0:2]) self.inv.set_kube_node(hosts.keys()) for h in range(3): - self.assertFalse(hosts.keys()[h] in self.inv.config['kube-node']) + self.assertFalse( + list(hosts.keys())[h] in + self.inv.yaml_config['all']['children']['kube-node']['hosts']) def test_scale_scenario_two(self): num_nodes = 500 @@ -233,15 +289,21 @@ class TestInventory(unittest.TestCase): hosts["node" + str(hostid)] = "" self.inv.set_all(hosts) - self.inv.set_etcd(hosts.keys()[0:3]) - self.inv.set_kube_master(hosts.keys()[3:5]) + self.inv.set_etcd(list(hosts.keys())[0:3]) + self.inv.set_kube_master(list(hosts.keys())[3:5]) self.inv.set_kube_node(hosts.keys()) for h in range(5): - self.assertFalse(hosts.keys()[h] in self.inv.config['kube-node']) + self.assertFalse( + list(hosts.keys())[h] in + self.inv.yaml_config['all']['children']['kube-node']['hosts']) def test_range2ips_range(self): changed_hosts = ['10.90.0.2', '10.90.0.4-10.90.0.6', '10.90.0.8'] - expected = ['10.90.0.2', '10.90.0.4', '10.90.0.5', '10.90.0.6', '10.90.0.8'] + expected = ['10.90.0.2', + '10.90.0.4', + '10.90.0.5', + '10.90.0.6', + '10.90.0.8'] result = self.inv.range2ips(changed_hosts) self.assertEqual(expected, result)