From 23456bbcab4f16e58727939fd3507699dd2c04b6 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Wed, 21 Jun 2017 10:38:16 -0500 Subject: [PATCH] Update oslo_config integration to support auto-generation of files --- drydock_provisioner/__init__.py | 5 - drydock_provisioner/config.py | 94 +++++++++++++++++-- drydock_provisioner/drivers/__init__.py | 8 +- drydock_provisioner/drivers/node/__init__.py | 8 +- .../drivers/node/maasdriver/driver.py | 41 ++++---- .../drivers/oob/pyghmi_driver/__init__.py | 3 +- drydock_provisioner/drydock.py | 23 ++--- setup.py | 20 +--- 8 files changed, 133 insertions(+), 69 deletions(-) diff --git a/drydock_provisioner/__init__.py b/drydock_provisioner/__init__.py index 6eda855f..f10bbbf6 100644 --- a/drydock_provisioner/__init__.py +++ b/drydock_provisioner/__init__.py @@ -11,8 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from .config import DrydockConfig - -config_mgr = DrydockConfig() -conf = config_mgr.conf \ No newline at end of file diff --git a/drydock_provisioner/config.py b/drydock_provisioner/config.py index b2505b17..98e853dc 100644 --- a/drydock_provisioner/config.py +++ b/drydock_provisioner/config.py @@ -12,19 +12,42 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +"""Single point of entry to generate the sample configuration file. + +This module collects all the necessary info from the other modules in this +package. It is assumed that: + +* Every other module in this package has a 'list_opts' function which + returns a dict where: + + * The keys are strings which are the group names. + + * The value of each key is a list of config options for that group. + +* The conf package doesn't have further packages with config options. + +* This module is only used in the context of sample file generation. + +""" +import collections +import importlib +import os +import pkgutil + from oslo_config import cfg class DrydockConfig(object): """ Initialize all the core options """ - # Core/General options + # Logging options logging_options = [ cfg.StrOpt('log_level', default='INFO', help='Global log level for Drydock'), cfg.StrOpt('global_logger_name', default='drydock', help='Logger name for the top-level logger'), - cfg.StrOpt('oobdriver_logger_name', default='${global_logger_name}.oobdriver'), - cfg.StrOpt('nodedriver_logger_name', default='${global_logger_name}.nodedriver'), - cfg.StrOpt('control_logger_name', default='${global_logger_name}.control'), + cfg.StrOpt('oobdriver_logger_name', default='${global_logger_name}.oobdriver', help='Logger name for OOB driver logging'), + cfg.StrOpt('nodedriver_logger_name', default='${global_logger_name}.nodedriver', help='Logger name for Node driver logging'), + cfg.StrOpt('control_logger_name', default='${global_logger_name}.control', help='Logger name for API server logging'), ] # API Authentication options @@ -50,19 +73,72 @@ class DrydockConfig(object): # Timeouts for various tasks specified in minutes timeout_options = [ - cfg.IntOpt('create_network_template',default=2,help='Timeout in minutes for creating site network templates'), - cfg.IntOpt('identify_node',default=10,help='Timeout in minutes for initial node identification'), - cfg.IntOpt('configure_hardware',default=30,help='Timeout in minutes for node commissioning and hardware configuration'), - cfg.IntOpt('apply_node_networking',default=5,help='Timeout in minutes for configuring node networking'), - cfg.IntOpt('deploy_node',default=45,help='Timeout in minutes for deploying a node'), + cfg.IntOpt('drydock_timeout', default=5, help='Fallback timeout when a specific one is not configured'), + cfg.IntOpt('create_network_template', default=2, help='Timeout in minutes for creating site network templates'), + cfg.IntOpt('identify_node', default=10, help='Timeout in minutes for initial node identification'), + cfg.IntOpt('configure_hardware', default=30, help='Timeout in minutes for node commissioning and hardware configuration'), + cfg.IntOpt('apply_node_networking', default=5, help='Timeout in minutes for configuring node networking'), + cfg.IntOpt('deploy_node', default=45, help='Timeout in minutes for deploying a node'), ] def __init__(self): self.conf = cfg.ConfigOpts() + def register_options(self): self.conf.register_opts(DrydockConfig.logging_options, group='logging') self.conf.register_opts(DrydockConfig.auth_options, group='authentication') self.conf.register_opts(DrydockConfig.plugin_options, group='plugins') self.conf.register_opts(DrydockConfig.timeout_options, group='timeouts') +config_mgr = DrydockConfig() +conf = config_mgr.conf +IGNORED_MODULES = ('drydock', 'config') + +def list_opts(): + opts = {'logging': DrydockConfig.logging_options, + 'authentication': DrydockConfig.auth_options, + 'plugins': DrydockConfig.plugin_options, + 'timeouts': DrydockConfig.timeout_options + } + + package_path = os.path.dirname(os.path.abspath(__file__)) + parent_module = ".".join(__name__.split('.')[:-1]) + module_names = _list_module_names(package_path, parent_module) + imported_modules = _import_modules(module_names) + _append_config_options(imported_modules, opts) + return _tupleize(opts) + +def _tupleize(d): + """Convert a dict of options to the 2-tuple format.""" + return [(key, value) for key, value in d.items()] + +def _list_module_names(pkg_path, parent_module): + module_names = [] + for _, module_name, ispkg in pkgutil.iter_modules(path=[pkg_path]): + if module_name in IGNORED_MODULES: + # Skip this module. + continue + elif ispkg: + module_names.extend(_list_module_names(pkg_path + "/" + module_name, parent_module + "." + module_name)) + else: + module_names.append(parent_module + "." + module_name) + return module_names + +def _import_modules(module_names): + imported_modules = [] + for module_name in module_names: + module = importlib.import_module(module_name) + if hasattr(module, 'list_opts'): + print("Pulling options from module %s" % module.__name__) + imported_modules.append(module) + return imported_modules + +def _append_config_options(imported_modules, config_options): + for module in imported_modules: + configs = module.list_opts() + for key, val in configs.items(): + if key not in config_options: + config_options[key] = val + else: + config_options[key].extend(val) \ No newline at end of file diff --git a/drydock_provisioner/drivers/__init__.py b/drydock_provisioner/drivers/__init__.py index ec609412..169ad64b 100644 --- a/drydock_provisioner/drivers/__init__.py +++ b/drydock_provisioner/drivers/__init__.py @@ -25,6 +25,10 @@ import drydock_provisioner.error as errors # driver tasks and feed them via queue class ProviderDriver(object): + driver_name = "generic" + driver_key = "generic" + driver_desc = "Generic Provider Driver" + def __init__(self, orchestrator=None, state_manager=None, **kwargs): if orchestrator is None: raise ValueError("ProviderDriver requires valid orchestrator") @@ -39,9 +43,7 @@ class ProviderDriver(object): # These are the actions that this driver supports self.supported_actions = [hd_fields.OrchestratorAction.Noop] - self.driver_name = "generic" - self.driver_key = "generic" - self.driver_desc = "Generic Provider Driver" + def execute_task(self, task_id): task = self.state_manager.get_task(task_id) diff --git a/drydock_provisioner/drivers/node/__init__.py b/drydock_provisioner/drivers/node/__init__.py index a28c7cec..08939cb3 100644 --- a/drydock_provisioner/drivers/node/__init__.py +++ b/drydock_provisioner/drivers/node/__init__.py @@ -20,6 +20,10 @@ from drydock_provisioner.drivers import ProviderDriver class NodeDriver(ProviderDriver): + driver_name = "node_generic" + driver_key = "node_generic" + driver_desc = "Generic Node Driver" + def __init__(self, **kwargs): super(NodeDriver, self).__init__(**kwargs) @@ -37,10 +41,6 @@ class NodeDriver(ProviderDriver): hd_fields.OrchestratorAction.DeployNode, hd_fields.OrchestratorAction.DestroyNode] - self.driver_name = "node_generic" - self.driver_key = "node_generic" - self.driver_desc = "Generic Node Driver" - def execute_task(self, task_id): task = self.state_manager.get_task(task_id) task_action = task.action diff --git a/drydock_provisioner/drivers/node/maasdriver/driver.py b/drydock_provisioner/drivers/node/maasdriver/driver.py index f402c0ab..3e7d96a2 100644 --- a/drydock_provisioner/drivers/node/maasdriver/driver.py +++ b/drydock_provisioner/drivers/node/maasdriver/driver.py @@ -18,7 +18,7 @@ import sys from oslo_config import cfg -import drydock_provisioner +import drydock_provisioner.config as config import drydock_provisioner.error as errors import drydock_provisioner.drivers as drivers import drydock_provisioner.objects.fields as hd_fields @@ -33,26 +33,22 @@ import drydock_provisioner.drivers.node.maasdriver.models.subnet as maas_subnet import drydock_provisioner.drivers.node.maasdriver.models.machine as maas_machine class MaasNodeDriver(NodeDriver): - maasdriver_options = [ cfg.StrOpt('maas_api_key', help='The API key for accessing MaaS', secret=True), cfg.StrOpt('maas_api_url', help='The URL for accessing MaaS API'), ] + driver_name = 'maasdriver' + driver_key = 'maasdriver' + driver_desc = 'MaaS Node Provisioning Driver' + def __init__(self, **kwargs): super(MaasNodeDriver, self).__init__(**kwargs) - self.driver_name = "maasdriver" - self.driver_key = "maasdriver" - self.driver_desc = "MaaS Node Provisioning Driver" - - self.setup_config_options(drydock_provisioner.conf) + config.conf.register_opts(maasdriver_options, group='maasdriver') self.logger = logging.getLogger("%s.%s" % - (drydock_provisioner.conf.logging.nodedriver_logger_name, self.driver_key)) - - def setup_config_options(self, conf): - conf.register_opts(MaasNodeDriver.maasdriver_options, group=self.driver_key) + (config.conf.logging.nodedriver_logger_name, self.driver_key)) def execute_task(self, task_id): task = self.state_manager.get_task(task_id) @@ -67,7 +63,7 @@ class MaasNodeDriver(NodeDriver): if task.action == hd_fields.OrchestratorAction.ValidateNodeServices: self.orchestrator.task_field_update(task.get_id(), status=hd_fields.TaskStatus.Running) - maas_client = MaasRequestFactory(drydock_provisioner.conf.maasdriver.maas_api_url, drydock_provisioner.conf.maasdriver.maas_api_key) + maas_client = MaasRequestFactory(config.conf.maasdriver.maas_api_url, config.conf.maasdriver.maas_api_key) try: if maas_client.test_connectivity(): @@ -138,7 +134,7 @@ class MaasNodeDriver(NodeDriver): runner.start() - runner.join(timeout=drydock_provisioner.conf.timeouts.create_network_template * 60) + runner.join(timeout=config.conf.timeouts.create_network_template * 60) if runner.is_alive(): result = { @@ -189,7 +185,7 @@ class MaasNodeDriver(NodeDriver): attempts = 0 worked = failed = False - while running_subtasks > 0 and attempts < drydock_provisioner.conf.timeouts.identify_node: + while running_subtasks > 0 and attempts < config.conf.timeouts.identify_node: for t in subtasks: subtask = self.state_manager.get_task(t) @@ -259,7 +255,7 @@ class MaasNodeDriver(NodeDriver): worked = failed = False #TODO Add timeout to config - while running_subtasks > 0 and attempts < drydock_provisioner.conf.timeouts.configure_hardware: + while running_subtasks > 0 and attempts < config.conf.timeouts.configure_hardware: for t in subtasks: subtask = self.state_manager.get_task(t) @@ -328,7 +324,7 @@ class MaasNodeDriver(NodeDriver): attempts = 0 worked = failed = False - while running_subtasks > 0 and attempts < drydock_provisioner.conf.timeouts.apply_node_networking: + while running_subtasks > 0 and attempts < config.conf.timeouts.apply_node_networking: for t in subtasks: subtask = self.state_manager.get_task(t) @@ -397,7 +393,7 @@ class MaasNodeDriver(NodeDriver): attempts = 0 worked = failed = False - while running_subtasks > 0 and attempts < drydock_provisioner.conf.timeouts.deploy_node: + while running_subtasks > 0 and attempts < config.conf.timeouts.deploy_node: for t in subtasks: subtask = self.state_manager.get_task(t) @@ -449,8 +445,8 @@ class MaasTaskRunner(drivers.DriverTaskRunner): status=hd_fields.TaskStatus.Running, result=hd_fields.ActionResult.Incomplete) - self.maas_client = MaasRequestFactory(drydock_provisioner.conf.maasdriver.maas_api_url, - drydock_provisioner.conf.maasdriver.maas_api_key) + self.maas_client = MaasRequestFactory(config.conf.maasdriver.maas_api_url, + config.conf.maasdriver.maas_api_key) site_design = self.orchestrator.get_effective_site(self.task.design_id) @@ -739,7 +735,7 @@ class MaasTaskRunner(drivers.DriverTaskRunner): # Poll machine status attempts = 0 - while attempts < drydock_provisioner.conf.timeouts.configure_hardware and machine.status_name != 'Ready': + while attempts < config.conf.timeouts.configure_hardware and machine.status_name != 'Ready': attempts = attempts + 1 time.sleep(1 * 60) try: @@ -971,7 +967,7 @@ class MaasTaskRunner(drivers.DriverTaskRunner): continue attempts = 0 - while attempts < drydock_provisioner.conf.timeouts.deploy_node and not machine.status_name.startswith('Deployed'): + while attempts < config.conf.timeouts.deploy_node and not machine.status_name.startswith('Deployed'): attempts = attempts + 1 time.sleep(1 * 60) try: @@ -1000,3 +996,6 @@ class MaasTaskRunner(drivers.DriverTaskRunner): status=hd_fields.TaskStatus.Complete, result=final_result, result_detail=result_detail) + +def list_opts(): + return {MaasNodeDriver.driver_key: MaasNodeDriver.maasdriver_options} diff --git a/drydock_provisioner/drivers/oob/pyghmi_driver/__init__.py b/drydock_provisioner/drivers/oob/pyghmi_driver/__init__.py index c0fb9867..efdfa1c8 100644 --- a/drydock_provisioner/drivers/oob/pyghmi_driver/__init__.py +++ b/drydock_provisioner/drivers/oob/pyghmi_driver/__init__.py @@ -100,7 +100,8 @@ class PyghmiDriver(oob.OobDriver): runner.start() attempts = 0 - while len(incomplete_subtasks) > 0 and attempts <= getattr(drydock_provisioner.conf.timeouts, task.action, 5): + while (len(incomplete_subtasks) > 0 and + attempts <= getattr(drydock_provisioner.conf.timeouts, task.action, drydock_provisioner.conf.timeouts.drydock_timeout)): for n in incomplete_subtasks: t = self.state_manager.get_task(n) if t.get_status() in [hd_fields.TaskStatus.Terminated, diff --git a/drydock_provisioner/drydock.py b/drydock_provisioner/drydock.py index de1e1b80..da4e9ce4 100644 --- a/drydock_provisioner/drydock.py +++ b/drydock_provisioner/drydock.py @@ -15,7 +15,7 @@ import logging from oslo_config import cfg import sys -import drydock_provisioner +import drydock_provisioner.config as config import drydock_provisioner.objects as objects import drydock_provisioner.ingester as ingester import drydock_provisioner.statemgmt as statemgmt @@ -30,23 +30,24 @@ def start_drydock(): cfg.BoolOpt('debug', short='d', default=False, help='Enable debug logging'), ] - drydock_provisioner.conf.register_cli_opts(cli_options) - drydock_provisioner.conf(sys.argv[1:]) + config.conf.register_cli_opts(cli_options) + config.config_mgr.register_options() + config.conf(sys.argv[1:]) - if drydock_provisioner.conf.debug: - drydock_provisioner.conf.logging.log_level = 'DEBUG' + if config.conf.debug: + config.conf.logging.log_level = 'DEBUG' # Setup root logger - logger = logging.getLogger(drydock_provisioner.conf.logging.global_logger_name) + logger = logging.getLogger(config.conf.logging.global_logger_name) - logger.setLevel(drydock_provisioner.conf.logging.log_level) + logger.setLevel(config.conf.logging.log_level) ch = logging.StreamHandler() formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s:%(funcName)s - %(message)s') ch.setFormatter(formatter) logger.addHandler(ch) # Specalized format for API logging - logger = logging.getLogger(drydock_provisioner.conf.logging.control_logger_name) + logger = logging.getLogger(config.conf.logging.control_logger_name) logger.propagate = False formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(user)s - %(req_id)s - %(external_ctx)s - %(message)s') @@ -57,13 +58,13 @@ def start_drydock(): state = statemgmt.DesignState() - orchestrator = orch.Orchestrator(drydock_provisioner.conf.plugins, + orchestrator = orch.Orchestrator(config.conf.plugins, state_manager=state) input_ingester = ingester.Ingester() - input_ingester.enable_plugins(drydock_provisioner.conf.plugins.ingester) + input_ingester.enable_plugins(config.conf.plugins.ingester) # Now that loggers are configured, log the effective config - drydock_provisioner.conf.log_opt_values(logging.getLogger(drydock_provisioner.conf.logging.global_logger_name), logging.DEBUG) + config.conf.log_opt_values(logging.getLogger(config.conf.logging.global_logger_name), logging.DEBUG) return api.start_api(state_manager=state, ingester=input_ingester, orchestrator=orchestrator) diff --git a/setup.py b/setup.py index bca84cf4..88bef9fc 100644 --- a/setup.py +++ b/setup.py @@ -15,20 +15,7 @@ # drydock_provisioner - A tool to consume a host topology and orchestrate # and monitor the provisioning of those hosts and execution of bootstrap # scripts -# -# Modular services: -# smelter - A service to consume the host topology, will support multiple -# input formats. Initially supports a YAML schema as demonstrated -# in the examples folder -# tarot - A service for persisting the host topology and orchestration state -# and making the data available via API -# cockpit - The entrypoint API for users to control helm-drydock and query -# current state -# alchemist - The core orchestrator -# drivers - A tree with all of the plugins that alchemist uses to execute -# orchestrated tasks -# jabberwocky - An introspection API that newly provisioned nodes can use to -# ingest self-data and bootstrap their application deployment process + from setuptools import setup @@ -65,6 +52,9 @@ setup(name='drydock_provisioner', 'uwsgi>1.4', 'bson===0.4.7', 'oslo.config', - ] + ], + entry_points={ + 'oslo.config.opts': 'drydock_provisioner = drydock_provisioner.config:list_opts', + } )