diff --git a/README.md b/README.md index 3918f79f..8e2f0347 100644 --- a/README.md +++ b/README.md @@ -56,4 +56,4 @@ Pluggable provisioner for server bootstrapping. Initial implementation is MaaS c aka introspection API for bootstrapping nodes to load self data. Possibly pluggable as this is basically an -authenticated bridge to the Design State API +authenticated bridge to the Design State API \ No newline at end of file diff --git a/drydock_provisioner/config.py b/drydock_provisioner/config.py index d035d29e..f64e91e9 100644 --- a/drydock_provisioner/config.py +++ b/drydock_provisioner/config.py @@ -27,8 +27,8 @@ class DrydockConfig(object): node_driver = { 'maasdriver': { - 'api_key': 'KTMHgA42cNSMnfmJ82:cdg4yQUhp542aHsCTV:7Dc2KB9hQpWq3LfQAAAKAj6wdg22yWxZ', - 'api_url': 'http://localhost:5240/MAAS/api/2.0/', + 'api_key': 'UTBfxGL69XWjaffQek:NuKZSYGuBs6ZpYC6B9:byvXBgY8CsW5VQKxGdQjvJXtjXwr5G4U', + 'api_url': 'http://10.23.19.16:30773/MAAS/api/2.0/', }, } @@ -41,4 +41,4 @@ class DrydockConfig(object): 'oob': 'drydock_provisioner.drivers.oob.pyghmi_driver.PyghmiDriver', 'node': 'drydock_provisioner.drivers.node.maasdriver.driver.MaasNodeDriver', } - } + } \ No newline at end of file diff --git a/drydock_provisioner/drivers/node/maasdriver/models/__init__.py b/drydock_provisioner/drivers/node/maasdriver/models/__init__.py index f10bbbf6..2a385a45 100644 --- a/drydock_provisioner/drivers/node/maasdriver/models/__init__.py +++ b/drydock_provisioner/drivers/node/maasdriver/models/__init__.py @@ -10,4 +10,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. +# limitations under the License. \ No newline at end of file diff --git a/drydock_provisioner/drivers/oob/pyghmi_driver/__init__.py b/drydock_provisioner/drivers/oob/pyghmi_driver/__init__.py index 2a218659..73c2bd5c 100644 --- a/drydock_provisioner/drivers/oob/pyghmi_driver/__init__.py +++ b/drydock_provisioner/drivers/oob/pyghmi_driver/__init__.py @@ -300,4 +300,4 @@ class PyghmiTaskRunner(drivers.DriverTaskRunner): result=hd_fields.ActionResult.Success, status=hd_fields.TaskStatus.Complete, result_detail=mci_id) - return + return \ No newline at end of file diff --git a/helm_drydock/control/api.py b/helm_drydock/control/api.py deleted file mode 100644 index a7134693..00000000 --- a/helm_drydock/control/api.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import falcon - -from .designs import * -from .tasks import * - -from .base import DrydockRequest -from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware - -def start_api(state_manager=None, ingester=None, orchestrator=None): - """ - Start the Drydock API service - - :param state_manager: Instance of helm_drydock.statemgmt.manager.DesignState for accessing - state persistence - :param ingester: Instance of helm_drydock.ingester.ingester.Ingester for handling design - part input - """ - control_api = falcon.API(request_type=DrydockRequest, - middleware=[AuthMiddleware(), ContextMiddleware(), LoggingMiddleware()]) - - # v1.0 of Drydock API - v1_0_routes = [ - # API for managing orchestrator tasks - ('/tasks', TasksResource(state_manager=state_manager, orchestrator=orchestrator)), - ('/tasks/{task_id}', TaskResource(state_manager=state_manager)), - - # API for managing site design data - ('/designs', DesignsResource(state_manager=state_manager)), - ('/designs/{design_id}', DesignResource(state_manager=state_manager, orchestrator=orchestrator)), - ('/designs/{design_id}/parts', DesignsPartsResource(state_manager=state_manager, ingester=ingester)), - ('/designs/{design_id}/parts/{kind}', DesignsPartsKindsResource(state_manager=state_manager)), - ('/designs/{design_id}/parts/{kind}/{name}', DesignsPartResource(state_manager=state_manager, orchestrator=orchestrator)) - ] - - for path, res in v1_0_routes: - control_api.add_route('/api/v1.0' + path, res) - - return control_api diff --git a/helm_drydock/control/base.py b/helm_drydock/control/base.py deleted file mode 100644 index 2337ef42..00000000 --- a/helm_drydock/control/base.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import falcon.request as request -import uuid -import json -import logging - -import helm_drydock.error as errors - -class BaseResource(object): - - def __init__(self): - self.logger = logging.getLogger('control') - self.authorized_roles = [] - - def on_options(self, req, resp): - self_attrs = dir(self) - methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH'] - allowed_methods = [] - - for m in methods: - if 'on_' + m.lower() in self_attrs: - allowed_methods.append(m) - - resp.headers['Allow'] = ','.join(allowed_methods) - resp.status = falcon.HTTP_200 - - # For authorizing access at the Resource level. A Resource requiring - # finer grained authorization at the method or instance level must - # implement that in the request handlers - def authorize_roles(self, role_list): - authorized = set(self.authorized_roles) - applied = set(role_list) - - if authorized.isdisjoint(applied): - return False - else: - return True - - def req_json(self, req): - if req.content_length is None or req.content_length == 0: - return None - - if req.content_type is not None and req.content_type.lower() == 'application/json': - raw_body = req.stream.read(req.content_length or 0) - - if raw_body is None: - return None - - try: - json_body = json.loads(raw_body.decode('utf-8')) - return json_body - except json.JSONDecodeError as jex: - raise errors.InvalidFormat("%s: Invalid JSON in body: %s" % (req.path, jex)) - else: - raise errors.InvalidFormat("Requires application/json payload") - - def return_error(self, resp, status_code, message="", retry=False): - resp.body = json.dumps({'type': 'error', 'message': message, 'retry': retry}) - resp.status = status_code - - def log_error(self, ctx, level, msg): - extra = { - 'user': 'N/A', - 'req_id': 'N/A', - 'external_ctx': 'N/A' - } - - if ctx is not None: - extra = { - 'user': ctx.user, - 'req_id': ctx.request_id, - 'external_ctx': ctx.external_marker, - } - - self.logger.log(level, msg, extra=extra) - - def debug(self, ctx, msg): - self.log_error(ctx, logging.DEBUG, msg) - - def info(self, ctx, msg): - self.log_error(ctx, logging.INFO, msg) - - def warn(self, ctx, msg): - self.log_error(ctx, logging.WARN, msg) - - def error(self, ctx, msg): - self.log_error(ctx, logging.ERROR, msg) - - -class StatefulResource(BaseResource): - - def __init__(self, state_manager=None): - super(StatefulResource, self).__init__() - - if state_manager is None: - self.error(None, "StatefulResource:init - StatefulResources require a state manager be set") - raise ValueError("StatefulResources require a state manager be set") - - self.state_manager = state_manager - - -class DrydockRequestContext(object): - - def __init__(self): - self.log_level = 'ERROR' - self.user = None - self.roles = ['anyone'] - self.req_id = str(uuid.uuid4()) - self.external_marker = None - - def set_log_level(self, level): - if level in ['error', 'info', 'debug']: - self.log_level = level - - def set_user(self, user): - self.user = user - - def add_role(self, role): - self.roles.append(role) - - def add_roles(self, roles): - self.roles.extend(roles) - - def remove_role(self, role): - self.roles = [x for x in self.roles - if x != role] - - def set_external_marker(self, marker): - self.external_marker = str(marker)[:20] - -class DrydockRequest(request.Request): - context_type = DrydockRequestContext \ No newline at end of file diff --git a/helm_drydock/control/designs.py b/helm_drydock/control/designs.py deleted file mode 100644 index 402b23ad..00000000 --- a/helm_drydock/control/designs.py +++ /dev/null @@ -1,164 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import falcon -import json -import uuid -import logging - -import helm_drydock.objects as hd_objects -import helm_drydock.error as errors - -from .base import StatefulResource - -class DesignsResource(StatefulResource): - - def __init__(self, **kwargs): - super(DesignsResource, self).__init__(**kwargs) - self.authorized_roles = ['user'] - - def on_get(self, req, resp): - state = self.state_manager - - designs = list(state.designs.keys()) - - resp.body = json.dumps(designs) - resp.status = falcon.HTTP_200 - - def on_post(self, req, resp): - try: - json_data = self.req_json(req) - design = None - if json_data is not None: - base_design = json_data.get('base_design_id', None) - - if base_design is not None: - base_design = uuid.UUID(base_design) - design = hd_objects.SiteDesign(base_design_id=base_design_uuid) - else: - design = hd_objects.SiteDesign() - design.assign_id() - design.create(req.context, self.state_manager) - - resp.body = json.dumps(design.obj_to_simple()) - resp.status = falcon.HTTP_201 - except errors.StateError as stex: - self.error(req.context, "Error updating persistence") - self.return_error(resp, falcon.HTTP_500, message="Error updating persistence", retry=True) - except errors.InvalidFormat as fex: - self.error(req.context, str(fex)) - self.return_error(resp, falcon.HTTP_400, message=str(fex), retry=False) - - -class DesignResource(StatefulResource): - - def __init__(self, orchestrator=None, **kwargs): - super(DesignResource, self).__init__(**kwargs) - self.authorized_roles = ['user'] - self.orchestrator = orchestrator - - def on_get(self, req, resp, design_id): - source = req.params.get('source', 'designed') - - try: - design = None - if source == 'compiled': - design = self.orchestrator.get_effective_site(design_id) - elif source == 'designed': - design = self.orchestrator.get_described_site(design_id) - - resp.body = json.dumps(design.obj_to_simple()) - except errors.DesignError: - self.error(req.context, "Design %s not found" % design_id) - self.return_error(resp, falcon.HTTP_404, message="Design %s not found" % design_id, retry=False) - -class DesignsPartsResource(StatefulResource): - - def __init__(self, ingester=None, **kwargs): - super(DesignsPartsResource, self).__init__(**kwargs) - self.ingester = ingester - self.authorized_roles = ['user'] - - if ingester is None: - self.error(None, "DesignsPartsResource requires a configured Ingester instance") - raise ValueError("DesignsPartsResource requires a configured Ingester instance") - - def on_post(self, req, resp, design_id): - ingester_name = req.params.get('ingester', None) - - if ingester_name is None: - self.error(None, "DesignsPartsResource POST requires parameter 'ingester'") - self.return_error(resp, falcon.HTTP_400, message="POST requires parameter 'ingester'", retry=False) - else: - try: - raw_body = req.stream.read(req.content_length or 0) - if raw_body is not None and len(raw_body) > 0: - parsed_items = self.ingester.ingest_data(plugin_name=ingester_name, design_state=self.state_manager, - content=raw_body, design_id=design_id, context=req.context) - resp.status = falcon.HTTP_201 - resp.body = json.dumps([x.obj_to_simple() for x in parsed_items]) - else: - self.return_error(resp, falcon.HTTP_400, message="Empty body not supported", retry=False) - except ValueError: - self.return_error(resp, falcon.HTTP_500, message="Error processing input", retry=False) - except LookupError: - self.return_error(resp, falcon.HTTP_400, message="Ingester %s not registered" % ingester_name, retry=False) - - -class DesignsPartsKindsResource(StatefulResource): - def __init__(self, **kwargs): - super(DesignsPartsKindsResource, self).__init__(**kwargs) - self.authorized_roles = ['user'] - - def on_get(self, req, resp, design_id, kind): - pass - -class DesignsPartResource(StatefulResource): - - def __init__(self, orchestrator=None, **kwargs): - super(DesignsPartResource, self).__init__(**kwargs) - self.authorized_roles = ['user'] - self.orchestrator = orchestrator - - def on_get(self, req , resp, design_id, kind, name): - source = req.params.get('source', 'designed') - - try: - design = None - if source == 'compiled': - design = self.orchestrator.get_effective_site(design_id) - elif source == 'designed': - design = self.orchestrator.get_described_site(design_id) - - part = None - if kind == 'Site': - part = design.get_site() - elif kind == 'Network': - part = design.get_network(name) - elif kind == 'NetworkLink': - part = design.get_network_link(name) - elif kind == 'HardwareProfile': - part = design.get_hardware_profile(name) - elif kind == 'HostProfile': - part = design.get_host_profile(name) - elif kind == 'BaremetalNode': - part = design.get_baremetal_node(name) - else: - self.error(req.context, "Kind %s unknown" % kind) - self.return_error(resp, falcon.HTTP_404, message="Kind %s unknown" % kind, retry=False) - return - - resp.body = json.dumps(part.obj_to_simple()) - except errors.DesignError as dex: - self.error(req.context, str(dex)) - self.return_error(resp, falcon.HTTP_404, message=str(dex), retry=False) \ No newline at end of file diff --git a/helm_drydock/control/middleware.py b/helm_drydock/control/middleware.py deleted file mode 100644 index 157a677e..00000000 --- a/helm_drydock/control/middleware.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import falcon -import logging -import uuid - -import helm_drydock.config as config - -class AuthMiddleware(object): - - # Authentication - def process_request(self, req, resp): - ctx = req.context - token = req.get_header('X-Auth-Token') - - user = self.validate_token(token) - - if user is not None: - ctx.set_user(user) - user_roles = self.role_list(user) - ctx.add_roles(user_roles) - else: - ctx.add_role('anyone') - - # Authorization - def process_resource(self, req, resp, resource, params): - ctx = req.context - - if not resource.authorize_roles(ctx.roles): - raise falcon.HTTPUnauthorized('Authentication required', - ('This resource requires an authorized role.')) - - # Return the username associated with an authenticated token or None - def validate_token(self, token): - if token == '42': - return 'scott' - elif token == 'bigboss': - return 'admin' - else: - return None - - # Return the list of roles assigned to the username - # Roles need to be an enum - def role_list(self, username): - if username == 'scott': - return ['user'] - elif username == 'admin': - return ['user', 'admin'] - -class ContextMiddleware(object): - - def process_request(self, req, resp): - ctx = req.context - - requested_logging = req.get_header('X-Log-Level') - - if (config.DrydockConfig.global_config.get('log_level', '') == 'DEBUG' or - (requested_logging == 'DEBUG' and 'admin' in ctx.roles)): - ctx.set_log_level('DEBUG') - elif requested_logging == 'INFO': - ctx.set_log_level('INFO') - - ext_marker = req.get_header('X-Context-Marker') - - ctx.set_external_marker(ext_marker if ext_marker is not None else '') - -class LoggingMiddleware(object): - - def __init__(self): - self.logger = logging.getLogger('drydock.control') - - def process_response(self, req, resp, resource, req_succeeded): - ctx = req.context - extra = { - 'user': ctx.user, - 'req_id': ctx.req_id, - 'external_ctx': ctx.external_marker, - } - resp.append_header('X-Drydock-Req', ctx.req_id) - self.logger.info("%s - %s" % (req.uri, resp.status), extra=extra) diff --git a/helm_drydock/control/readme.md b/helm_drydock/control/readme.md deleted file mode 100644 index d09f2d0e..00000000 --- a/helm_drydock/control/readme.md +++ /dev/null @@ -1,30 +0,0 @@ -# Control # - -This is the external facing API service to control the rest -of Drydock and query Drydock-managed data. - -## v1.0 Endpoints ## - -### /api/v1.0/tasks ### - -POST - Create a new orchestration task and submit it for execution -GET - Get status of a task -DELETE - Cancel execution of a task if permitted - -### /api/v1.0/designs ### - -POST - Create a new site design so design parts can be added - -### /api/v1.0/designs/{id} - -GET - Get a current design if available. Param 'source=compiled' to calculate the inheritance chain and compile the effective design. - -### /api/v1.0/designs/{id}/parts - -POST - Submit a new design part to be ingested and added to this design -GET - View a currently defined design part -PUT - Replace an existing design part *Not Implemented* - -### /api/v1.0/designs/{id}/parts/{kind}/{name} - -GET - View a single design part. param 'source=compiled' to calculate the inheritance chain and compile the effective configuration for the design part. \ No newline at end of file diff --git a/helm_drydock/control/tasks.py b/helm_drydock/control/tasks.py deleted file mode 100644 index bf35861f..00000000 --- a/helm_drydock/control/tasks.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import falcon -import json -import threading -import traceback - -import helm_drydock.objects.task as obj_task -from .base import StatefulResource - -class TasksResource(StatefulResource): - - def __init__(self, orchestrator=None, **kwargs): - super(TasksResource, self).__init__(**kwargs) - self.authorized_roles = ['user'] - self.orchestrator = orchestrator - - def on_get(self, req, resp): - task_id_list = [str(x.get_id()) for x in self.state_manager.tasks] - resp.body = json.dumps(task_id_list) - - def on_post(self, req, resp): - try: - json_data = self.req_json(req) - - sitename = json_data.get('sitename', None) - design_id = json_data.get('design_id', None) - action = json_data.get('action', None) - - if sitename is None or design_id is None or action is None: - self.info(req.context, "Task creation requires fields sitename, design_id, action") - self.return_error(resp, falcon.HTTP_400, message="Task creation requires fields sitename, design_id, action", retry=False) - return - - task = self.orchestrator.create_task(obj_task.OrchestratorTask, site=sitename, - design_id=design_id, action=action) - - task_thread = threading.Thread(target=self.orchestrator.execute_task, args=[task.get_id()]) - task_thread.start() - - resp.body = json.dumps(task.to_dict()) - resp.status = falcon.HTTP_201 - except Exception as ex: - self.error(req.context, "Unknown error: %s\n%s" % (str(ex), traceback.format_exc())) - self.return_error(resp, falcon.HTTP_500, message="Unknown error", retry=False) - - -class TaskResource(StatefulResource): - - def __init__(self, orchestrator=None, **kwargs): - super(TaskResource, self).__init__(**kwargs) - self.authorized_roles = ['user'] - self.orchestrator = orchestrator - - def on_get(self, req, resp, task_id): - try: - task = self.state_manager.get_task(task_id) - - if task is None: - self.info(req.context, "Task %s does not exist" % task_id ) - self.return_error(resp, falcon.HTTP_404, message="Task %s does not exist" % task_id, retry=False) - return - - resp.body = json.dumps(task.to_dict()) - resp.status = falcon.HTTP_200 - except Exception as ex: - self.error(req.context, "Unknown error: %s" % (str(ex))) - self.return_error(resp, falcon.HTTP_500, message="Unknown error", retry=False) diff --git a/helm_drydock/drivers/node/maasdriver/driver.py b/helm_drydock/drivers/node/maasdriver/driver.py deleted file mode 100644 index 32216169..00000000 --- a/helm_drydock/drivers/node/maasdriver/driver.py +++ /dev/null @@ -1,317 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import helm_drydock.error as errors -import helm_drydock.config as config -import helm_drydock.drivers as drivers -import helm_drydock.objects.fields as hd_fields -import helm_drydock.objects.task as task_model - -from helm_drydock.drivers.node import NodeDriver -from .api_client import MaasRequestFactory -import helm_drydock.drivers.node.maasdriver.models.fabric as maas_fabric -import helm_drydock.drivers.node.maasdriver.models.vlan as maas_vlan -import helm_drydock.drivers.node.maasdriver.models.subnet as maas_subnet - -class MaasNodeDriver(NodeDriver): - - def __init__(self, **kwargs): - super(MaasNodeDriver, self).__init__(**kwargs) - - self.driver_name = "maasdriver" - self.driver_key = "maasdriver" - self.driver_desc = "MaaS Node Provisioning Driver" - - self.config = config.DrydockConfig.node_driver[self.driver_key] - - def execute_task(self, task_id): - task = self.state_manager.get_task(task_id) - - if task is None: - raise errors.DriverError("Invalid task %s" % (task_id)) - - if task.action not in self.supported_actions: - raise errors.DriverError("Driver %s doesn't support task action %s" - % (self.driver_desc, task.action)) - - if task.action == hd_fields.OrchestratorAction.ValidateNodeServices: - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Running) - maas_client = MaasRequestFactory(self.config['api_url'], self.config['api_key']) - - try: - if maas_client.test_connectivity(): - if maas_client.test_authentication(): - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=hd_fields.ActionResult.Success) - return - except errors.TransientDriverError(ex): - result = { - 'retry': True, - 'detail': str(ex), - } - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=hd_fields.ActionResult.Failure, - result_details=result) - return - except errors.PersistentDriverError(ex): - result = { - 'retry': False, - 'detail': str(ex), - } - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=hd_fields.ActionResult.Failure, - result_details=result) - return - except Exception(ex): - result = { - 'retry': False, - 'detail': str(ex), - } - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=hd_fields.ActionResult.Failure, - result_details=result) - return - - design_id = getattr(task, 'design_id', None) - - if design_id is None: - raise errors.DriverError("No design ID specified in task %s" % - (task_id)) - - - if task.site_name is None: - raise errors.DriverError("No site specified for task %s." % - (task_id)) - - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Running) - - site_design = self.orchestrator.get_effective_site(design_id) - - if task.action == hd_fields.OrchestratorAction.CreateNetworkTemplate: - subtask = self.orchestrator.create_task(task_model.DriverTask, - parent_task_id=task.get_id(), design_id=design_id, - action=task.action, site_name=task.site_name, - task_scope={'site': task.site_name}) - runner = MaasTaskRunner(state_manager=self.state_manager, - orchestrator=self.orchestrator, - task_id=subtask.get_id(),config=self.config) - runner.start() - - runner.join(timeout=120) - - if runner.is_alive(): - result = { - 'retry': False, - 'detail': 'MaaS Network creation timed-out' - } - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=hd_fields.ActionResult.Failure, - result_detail=result) - else: - subtask = self.state_manager.get_task(subtask.get_id()) - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=subtask.get_result()) - - return - -class MaasTaskRunner(drivers.DriverTaskRunner): - - def __init__(self, config=None, **kwargs): - super(MaasTaskRunner, self).__init__(**kwargs) - - self.driver_config = config - - def execute_task(self): - task_action = self.task.action - - self.orchestrator.task_field_update(self.task.get_id(), - status=hd_fields.TaskStatus.Running, - result=hd_fields.ActionResult.Incomplete) - - self.maas_client = MaasRequestFactory(self.driver_config['api_url'], - self.driver_config['api_key']) - - site_design = self.orchestrator.get_effective_site(self.task.design_id) - - if task_action == hd_fields.OrchestratorAction.CreateNetworkTemplate: - # Try to true up MaaS definitions of fabrics/vlans/subnets - # with the networks defined in Drydock - design_networks = site_design.networks - - subnets = maas_subnet.Subnets(self.maas_client) - subnets.refresh() - - result_detail = { - 'detail': [] - } - - for n in design_networks: - try: - subnet = subnets.singleton({'cidr': n.cidr}) - - if subnet is not None: - subnet.name = n.name - subnet.dns_servers = n.dns_servers - - vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=subnet.fabric) - vlan_list.refresh() - - vlan = vlan_list.select(subnet.vlan) - - if vlan is not None: - if ((n.vlan_id is None and vlan.vid != 0) or - (n.vlan_id is not None and vlan.vid != n.vlan_id)): - - # if the VLAN name matches, assume this is the correct resource - # and it needs to be updated - if vlan.name == n.name: - vlan.set_vid(n.vlan_id) - vlan.mtu = n.mtu - vlan.update() - result_detail['detail'].append("VLAN %s found for network %s, updated attributes" - % (vlan.resource_id, n.name)) - else: - # Found a VLAN with the correct VLAN tag, update subnet to use it - target_vlan = vlan_list.singleton({'vid': n.vlan_id if n.vlan_id is not None else 0}) - if target_vlan is not None: - subnet.vlan = target_vlan.resource_id - else: - # This is a flag that after creating a fabric and - # VLAN below, update the subnet - subnet.vlan = None - else: - subnet.vlan = None - - # Check if the routes have a default route - subnet.gateway_ip = n.get_default_gateway() - - - result_detail['detail'].append("Subnet %s found for network %s, updated attributes" - % (subnet.resource_id, n.name)) - - # Need to find or create a Fabric/Vlan for this subnet - if (subnet is None or (subnet is not None and subnet.vlan is None)): - fabric_list = maas_fabric.Fabrics(self.maas_client) - fabric_list.refresh() - fabric = fabric_list.singleton({'name': n.name}) - - vlan = None - - if fabric is not None: - vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=fabric.resource_id) - vlan_list.refresh() - - vlan = vlan_list.singleton({'vid': n.vlan_id if n.vlan_id is not None else 0}) - - if vlan is not None: - vlan = matching_vlans[0] - - vlan.name = n.name - if getattr(n, 'mtu', None) is not None: - vlan.mtu = n.mtu - - if subnet is not None: - subnet.vlan = vlan.resource_id - subnet.update() - - vlan.update() - result_detail['detail'].append("VLAN %s found for network %s, updated attributes" - % (vlan.resource_id, n.name)) - else: - # Create a new VLAN in this fabric and assign subnet to it - vlan = maas_vlan.Vlan(self.maas_client, name=n.name, vid=vlan_id, - mtu=getattr(n, 'mtu', None),fabric_id=fabric.resource_id) - vlan = vlan_list.add(vlan) - - result_detail['detail'].append("VLAN %s created for network %s" - % (vlan.resource_id, n.name)) - if subnet is not None: - subnet.vlan = vlan.resource_id - subnet.update() - - else: - # Create new fabric and VLAN - fabric = maas_fabric.Fabric(self.maas_client, name=n.name) - fabric = fabric_list.add(fabric) - fabric_list.refresh() - - result_detail['detail'].append("Fabric %s created for network %s" - % (fabric.resource_id, n.name)) - - vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=new_fabric.resource_id) - vlan_list.refresh() - - # A new fabric comes with a single default VLAN. Retrieve it and update attributes - vlan = vlan_list.single() - - vlan.name = n.name - vlan.vid = n.vlan_id if n.vlan_id is not None else 0 - if getattr(n, 'mtu', None) is not None: - vlan.mtu = n.mtu - - vlan.update() - result_detail['detail'].append("VLAN %s updated for network %s" - % (vlan.resource_id, n.name)) - if subnet is not None: - # If subnet was found above, but needed attached to a new fabric/vlan then - # attach it - subnet.vlan = vlan.resource_id - subnet.update() - - if subnet is None: - # If subnet did not exist, create it here and attach it to the fabric/VLAN - subnet = maas_subnet.Subnet(self.maas_client, name=n.name, cidr=n.cidr, fabric=fabric.resource_id, - vlan=vlan.resource_id, gateway_ip=n.get_default_gateway()) - - subnet_list = maas_subnet.Subnets(self.maas_client) - subnet = subnet_list.add(subnet) - except ValueError as vex: - raise errors.DriverError("Inconsistent data from MaaS") - - subnet_list = maas_subnet.Subnets(self.maas_client) - subnet_list.refresh() - - action_result = hd_fields.ActionResult.Incomplete - - success_rate = 0 - - for n in design_networks: - exists = subnet_list.query({'cidr': n.cidr}) - if len(exists) > 0: - subnet = exists[0] - if subnet.name == n.name: - success_rate = success_rate + 1 - else: - success_rate = success_rate + 1 - else: - success_rate = success_rate + 1 - - if success_rate == len(design_networks): - action_result = hd_fields.ActionResult.Success - elif success_rate == - (len(design_networks)): - action_result = hd_fields.ActionResult.Failure - else: - action_result = hd_fields.ActionResult.PartialSuccess - - self.orchestrator.task_field_update(self.task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=action_result, - result_detail=result_detail) \ No newline at end of file diff --git a/helm_drydock/drydock.py b/helm_drydock/drydock.py deleted file mode 100644 index 4be2340e..00000000 --- a/helm_drydock/drydock.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging - -import helm_drydock.config as config -import helm_drydock.objects as objects -import helm_drydock.ingester as ingester -import helm_drydock.statemgmt as statemgmt -import helm_drydock.orchestrator as orch -import helm_drydock.control.api as api - -def start_drydock(): - objects.register_all() - - # Setup root logger - logger = logging.getLogger('drydock') - - logger.setLevel(config.DrydockConfig.global_config.get('log_level')) - ch = logging.StreamHandler() - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - ch.setFormatter(formatter) - logger.addHandler(ch) - - # Specalized format for API logging - logger = logging.getLogger('drydock.control') - logger.propagate = False - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(user)s - %(req_id)s - %(external_ctx)s - %(message)s') - - ch = logging.StreamHandler() - ch.setFormatter(formatter) - logger.addHandler(ch) - - state = statemgmt.DesignState() - - orchestrator = orch.Orchestrator(config.DrydockConfig.orchestrator_config.get('drivers', {}), - state_manager=state) - input_ingester = ingester.Ingester() - input_ingester.enable_plugins(config.DrydockConfig.ingester_config.get('plugins', [])) - - return api.start_api(state_manager=state, ingester=input_ingester, - orchestrator=orchestrator) - -drydock = start_drydock() - diff --git a/helm_drydock/ingester/__init__.py b/helm_drydock/ingester/__init__.py deleted file mode 100644 index 07862605..00000000 --- a/helm_drydock/ingester/__init__.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ingester - Ingest host topologies to define site design and -# persist design to helm-drydock's statemgmt service - -import logging -import yaml -import uuid -import importlib - -import helm_drydock.objects as objects -import helm_drydock.objects.site as site -import helm_drydock.objects.network as network -import helm_drydock.objects.hwprofile as hwprofile -import helm_drydock.objects.node as node -import helm_drydock.objects.hostprofile as hostprofile - -from helm_drydock.statemgmt import DesignState - -class Ingester(object): - - def __init__(self): - self.logger = logging.getLogger("drydock.ingester") - self.registered_plugins = {} - - def enable_plugins(self, plugins=[]): - """ - enable_plugins - - :params plugins: - A list of strings naming class objects denoting the ingester plugins to be enabled - - Enable plugins that can be used for ingest_data calls. Each plugin should use - helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one - enabled plugin successfully initializes, the call is considered successful. Otherwise - it will throw an exception - """ - if len(plugins) == 0: - self.log.error("Cannot have an empty plugin list.") - - for plugin in plugins: - try: - (module, x, classname) = plugin.rpartition('.') - - if module == '': - raise Exception() - mod = importlib.import_module(module) - klass = getattr(mod, classname) - new_plugin = klass() - plugin_name = new_plugin.get_name() - self.registered_plugins[plugin_name] = new_plugin - except Exception as ex: - self.logger.error("Could not enable plugin %s - %s" % (plugin, str(ex))) - - if len(self.registered_plugins) == 0: - self.logger.error("Could not enable at least one plugin") - raise Exception("Could not enable at least one plugin") - - - def ingest_data(self, plugin_name='', design_state=None, design_id=None, context=None, **kwargs): - if design_state is None: - self.logger.error("Ingester:ingest_data called without valid DesignState handler") - raise ValueError("Invalid design_state handler") - - # If no design_id is specified, instantiate a new one - if 'design_id' is None: - self.logger.error("Ingester:ingest_data required kwarg 'design_id' missing") - raise ValueError("Ingester:ingest_data required kwarg 'design_id' missing") - - design_data = design_state.get_design(design_id) - - self.logger.debug("Ingester:ingest_data ingesting design parts for design %s" % design_id) - - if plugin_name in self.registered_plugins: - design_items = self.registered_plugins[plugin_name].ingest_data(**kwargs) - self.logger.debug("Ingester:ingest_data parsed %s design parts" % str(len(design_items))) - for m in design_items: - if context is not None: - m.set_create_fields(context) - if type(m) is site.Site: - design_data.set_site(m) - elif type(m) is network.Network: - design_data.add_network(m) - elif type(m) is network.NetworkLink: - design_data.add_network_link(m) - elif type(m) is hostprofile.HostProfile: - design_data.add_host_profile(m) - elif type(m) is hwprofile.HardwareProfile: - design_data.add_hardware_profile(m) - elif type(m) is node.BaremetalNode: - design_data.add_baremetal_node(m) - design_state.put_design(design_data) - return design_items - else: - self.logger.error("Could not find plugin %s to ingest data." % (plugin_name)) - raise LookupError("Could not find plugin %s" % plugin_name) - """ - ingest_data - - params: plugin_name - Which plugin should be used for ingestion - params: params - A map of parameters that will be passed to the plugin's ingest_data method - - Execute a data ingestion using the named plugin (assuming it is enabled) - """ -