From cff99e4d1c4a7153e02923df2734bc956c1c528f Mon Sep 17 00:00:00 2001 From: Aaron Sheffield Date: Mon, 12 Mar 2018 12:21:01 -0500 Subject: [PATCH] Enhanced Drydock Health Check - Added a database connection check to the health check. - Added a MaaS connection check to the health check. Change-Id: I6c771a9ed6a278eb75dbc0f2e503088ff9046149 --- drydock_provisioner/control/api.py | 3 +- drydock_provisioner/control/health.py | 48 ++++++++++++++++++- .../drivers/node/maasdriver/actions/node.py | 2 +- drydock_provisioner/statemgmt/state.py | 21 ++++++++ .../postgres}/test_api_health.py | 4 +- 5 files changed, 72 insertions(+), 6 deletions(-) rename tests/{unit => integration/postgres}/test_api_health.py (85%) diff --git a/drydock_provisioner/control/api.py b/drydock_provisioner/control/api.py index bed5b45f..f410fd58 100644 --- a/drydock_provisioner/control/api.py +++ b/drydock_provisioner/control/api.py @@ -55,7 +55,8 @@ def start_api(state_manager=None, ingester=None, orchestrator=None): # v1.0 of Drydock API v1_0_routes = [ # API for managing orchestrator tasks - ('/health', HealthResource()), + ('/health', HealthResource(state_manager=state_manager, + orchestrator=orchestrator)), ('/tasks', TasksResource(state_manager=state_manager, orchestrator=orchestrator)), diff --git a/drydock_provisioner/control/health.py b/drydock_provisioner/control/health.py index 0c2915a4..d81715c7 100644 --- a/drydock_provisioner/control/health.py +++ b/drydock_provisioner/control/health.py @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. import falcon +import json from drydock_provisioner.control.base import BaseResource +from drydock_provisioner.drivers.node.maasdriver.actions.node import ValidateNodeServices +from drydock_provisioner.objects.fields import ActionResult +import drydock_provisioner.objects.fields as hd_fields class HealthResource(BaseResource): @@ -21,9 +25,49 @@ class HealthResource(BaseResource): Return empty response/body to show that Drydock is healthy """ + def __init__(self, state_manager=None, orchestrator=None, **kwargs): + """Object initializer. + + :param state_manager: instance of Drydock state_manager + """ + super().__init__(**kwargs) + self.state_manager = state_manager + self.orchestrator = orchestrator def on_get(self, req, resp): """ - It really does nothing right now. It may do more later + Returns 204 on success, otherwise 500 with a response body. """ - resp.status = falcon.HTTP_204 + healthy = True + # Test database connection + try: + now = self.state_manager.get_now() + if now is None: + raise Exception('None received from database for now()') + except Exception as ex: + healthy = False + resp.body = json.dumps({ + 'type': 'error', + 'message': 'Database error', + 'retry': True + }) + resp.status = falcon.HTTP_500 + + # Test MaaS connection + try: + task = self.orchestrator.create_task(action=hd_fields.OrchestratorAction.Noop) + maas_validation = ValidateNodeServices(task, self.orchestrator, self.state_manager) + maas_validation.start() + if maas_validation.task.get_status() == ActionResult.Failure: + raise Exception('MaaS task failure') + except Exception as ex: + healthy = False + resp.body = json.dumps({ + 'type': 'error', + 'message': 'MaaS error', + 'retry': True + }) + resp.status = falcon.HTTP_500 + + if healthy: + resp.status = falcon.HTTP_204 diff --git a/drydock_provisioner/drivers/node/maasdriver/actions/node.py b/drydock_provisioner/drivers/node/maasdriver/actions/node.py index 9434cd62..bd4beb0c 100644 --- a/drydock_provisioner/drivers/node/maasdriver/actions/node.py +++ b/drydock_provisioner/drivers/node/maasdriver/actions/node.py @@ -66,7 +66,7 @@ class ValidateNodeServices(BaseMaasAction): ctx_type='NA') self.task.success() if self.maas_client.test_authentication(): - self.logger.info("Able to authenitcate with MaaS API.") + self.logger.info("Able to authenticate with MaaS API.") self.task.add_status_msg( msg='Able to authenticate with MaaS API.', error=False, diff --git a/drydock_provisioner/statemgmt/state.py b/drydock_provisioner/statemgmt/state.py index ec9ca0bf..c025ef45 100644 --- a/drydock_provisioner/statemgmt/state.py +++ b/drydock_provisioner/statemgmt/state.py @@ -667,3 +667,24 @@ class DrydockState(object): except Exception as ex: self.logger.error("Error selecting build data.", exc_info=ex) raise errors.BuildDataError("Error selecting build data.") + + def get_now(self): + """Query the database for now() from dual. + """ + try: + with self.db_engine.connect() as conn: + query = sql.text("SELECT now()") + rs = conn.execute(query) + + r = rs.first() + + if r is not None and r.now: + return r.now + else: + return None + except Exception as ex: + self.logger.error(str(ex)) + self.logger.error( + "Error querying for now()", + exc_info=True) + return None diff --git a/tests/unit/test_api_health.py b/tests/integration/postgres/test_api_health.py similarity index 85% rename from tests/unit/test_api_health.py rename to tests/integration/postgres/test_api_health.py index ddbd1c1b..c0a5d786 100644 --- a/tests/unit/test_api_health.py +++ b/tests/integration/postgres/test_api_health.py @@ -18,8 +18,8 @@ from drydock_provisioner.control.health import HealthResource import falcon -def test_get_health(mocker): - api = HealthResource() +def test_get_health(mocker, deckhand_orchestrator, drydock_state): + api = HealthResource(state_manager=drydock_state, orchestrator=deckhand_orchestrator) # Configure mocked request and response req = mocker.MagicMock(spec=falcon.Request)