From cf920b86b43ef59e6431ff5690b8885dd309826f Mon Sep 17 00:00:00 2001 From: David Shrewsbury Date: Tue, 8 May 2018 09:21:39 -0400 Subject: [PATCH] Clean up configuration file loading This extends the config.Config() class to contain methods to parse YAML top-level configuration sections. It makes the code that loads the config easier to digest, and makes portions of it reusable (e.g., setZooKeeperServers()). The 'db' config attribute is removed since it was unused. Change-Id: Ia6aa64f8ad378f022f0c422a06935610e8c50de4 --- nodepool/config.py | 167 ++++++++++++++++++++++++++++----------------- 1 file changed, 104 insertions(+), 63 deletions(-) diff --git a/nodepool/config.py b/nodepool/config.py index 84e86117f..5fbaa9ba4 100755 --- a/nodepool/config.py +++ b/nodepool/config.py @@ -26,7 +26,99 @@ from nodepool.driver import Drivers class Config(ConfigValue): - pass + ''' + Class representing the nodepool configuration. + + This class implements methods to read each of the top-level configuration + items found in the YAML config file, and set attributes accordingly. + ''' + def __init__(self): + self.diskimages = {} + self.labels = {} + self.providers = {} + self.provider_managers = {} + self.zookeeper_servers = {} + + def setElementsDir(self, value): + self.elementsdir = value + + def setImagesDir(self, value): + self.imagesdir = value + + def setBuildLog(self, directory, retention): + if retention is None: + retention = 7 + self.build_log_dir = directory + self.build_log_retention = retention + + def setMaxHoldAge(self, value): + if value is None or value <= 0: + value = math.inf + self.max_hold_age = value + + def setWebApp(self, webapp_cfg): + if webapp_cfg is None: + webapp_cfg = {} + self.webapp = { + 'port': webapp_cfg.get('port', 8005), + 'listen_address': webapp_cfg.get('listen_address', '0.0.0.0') + } + + def setZooKeeperServers(self, zk_cfg): + if not zk_cfg: + return + + for server in zk_cfg: + z = zk.ZooKeeperConnectionConfig(server['host'], + server.get('port', 2181), + server.get('chroot', None)) + name = z.host + '_' + str(z.port) + self.zookeeper_servers[name] = z + + def setDiskImages(self, diskimages_cfg): + if not diskimages_cfg: + return + + for diskimage in diskimages_cfg: + d = DiskImage() + d.name = diskimage['name'] + if 'elements' in diskimage: + d.elements = u' '.join(diskimage['elements']) + else: + d.elements = '' + # must be a string, as it's passed as env-var to + # d-i-b, but might be untyped in the yaml and + # interpreted as a number (e.g. "21" for fedora) + d.release = str(diskimage.get('release', '')) + d.rebuild_age = int(diskimage.get('rebuild-age', 86400)) + d.env_vars = diskimage.get('env-vars', {}) + if not isinstance(d.env_vars, dict): + d.env_vars = {} + d.image_types = set(diskimage.get('formats', [])) + d.pause = bool(diskimage.get('pause', False)) + d.username = diskimage.get('username', 'zuul') + self.diskimages[d.name] = d + + def setLabels(self, labels_cfg): + if not labels_cfg: + return + + for label in labels_cfg: + l = Label() + l.name = label['name'] + l.max_ready_age = label.get('max-ready-age', 0) + l.min_ready = label.get('min-ready', 0) + l.pools = [] + self.labels[l.name] = l + + def setProviders(self, providers_cfg): + if not providers_cfg: + return + + for provider in providers_cfg: + p = get_provider_config(provider) + p.load(self) + self.providers[p.name] = p class Label(ConfigValue): @@ -101,64 +193,18 @@ def loadConfig(config_path): driver["config"].reset() newconfig = Config() - newconfig.db = None - newconfig.webapp = { - 'port': config.get('webapp', {}).get('port', 8005), - 'listen_address': config.get('webapp', {}).get('listen_address', - '0.0.0.0') - } - newconfig.providers = {} - newconfig.labels = {} - newconfig.elementsdir = config.get('elements-dir') - newconfig.imagesdir = config.get('images-dir') - newconfig.build_log_dir = config.get('build-log-dir') - newconfig.build_log_retention = config.get('build-log-retention', 7) - newconfig.provider_managers = {} - newconfig.zookeeper_servers = {} - newconfig.diskimages = {} - newconfig.max_hold_age = config.get('max-hold-age', math.inf) - if newconfig.max_hold_age <= 0: - newconfig.max_hold_age = math.inf - for server in config.get('zookeeper-servers', []): - z = zk.ZooKeeperConnectionConfig(server['host'], - server.get('port', 2181), - server.get('chroot', None)) - name = z.host + '_' + str(z.port) - newconfig.zookeeper_servers[name] = z + newconfig.setElementsDir(config.get('elements-dir')) + newconfig.setImagesDir(config.get('images-dir')) + newconfig.setBuildLog(config.get('build-log-dir'), + config.get('build-log-retention')) + newconfig.setMaxHoldAge(config.get('max-hold-age')) + newconfig.setWebApp(config.get('webapp')) + newconfig.setZooKeeperServers(config.get('zookeeper-servers')) + newconfig.setDiskImages(config.get('diskimages')) + newconfig.setLabels(config.get('labels')) + newconfig.setProviders(config.get('providers')) - for diskimage in config.get('diskimages', []): - d = DiskImage() - d.name = diskimage['name'] - newconfig.diskimages[d.name] = d - if 'elements' in diskimage: - d.elements = u' '.join(diskimage['elements']) - else: - d.elements = '' - # must be a string, as it's passed as env-var to - # d-i-b, but might be untyped in the yaml and - # interpreted as a number (e.g. "21" for fedora) - d.release = str(diskimage.get('release', '')) - d.rebuild_age = int(diskimage.get('rebuild-age', 86400)) - d.env_vars = diskimage.get('env-vars', {}) - if not isinstance(d.env_vars, dict): - d.env_vars = {} - d.image_types = set(diskimage.get('formats', [])) - d.pause = bool(diskimage.get('pause', False)) - d.username = diskimage.get('username', 'zuul') - - for label in config.get('labels', []): - l = Label() - l.name = label['name'] - newconfig.labels[l.name] = l - l.max_ready_age = label.get('max-ready-age', 0) - l.min_ready = label.get('min-ready', 0) - l.pools = [] - - for provider in config.get('providers', []): - p = get_provider_config(provider) - p.load(newconfig) - newconfig.providers[p.name] = p return newconfig @@ -172,9 +218,4 @@ def loadSecureConfig(config, secure_config_path): config.zookeeper_servers = {} # TODO(Shrews): Support ZooKeeper auth - for server in secure.get('zookeeper-servers', []): - z = zk.ZooKeeperConnectionConfig(server['host'], - server.get('port', 2181), - server.get('chroot', None)) - name = z.host + '_' + str(z.port) - config.zookeeper_servers[name] = z + config.setZooKeeperServers(secure.get('zookeeper-servers'))