diff --git a/etc/valet/api/config.py b/etc/valet/api/config.py index dc08178..53288d9 100644 --- a/etc/valet/api/config.py +++ b/etc/valet/api/config.py @@ -47,45 +47,6 @@ app = { ], } -logging = { - 'root': {'level': 'INFO', 'handlers': ['console']}, - 'loggers': { - 'api': { - 'level': 'DEBUG', 'handlers': ['console'], 'propagate': False - }, - 'api.models': { - 'level': 'INFO', 'handlers': ['console'], 'propagate': False - }, - 'api.common': { - 'level': 'INFO', 'handlers': ['console'], 'propagate': False - }, - 'pecan': { - 'level': 'DEBUG', 'handlers': ['console'], 'propagate': False - }, - 'py.warnings': {'handlers': ['console']}, - '__force_dict__': True - }, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'color' - } - }, - 'formatters': { - 'simple': { - 'format': ('%(asctime)s %(levelname)-5.5s [%(name)s]' - '[%(threadName)s] %(message)s') - }, - 'color': { - '()': 'pecan.log.ColorFormatter', - 'format': ('%(asctime)s [%(padded_color_levelname)s] [%(name)s]' - '[%(threadName)s] %(message)s'), - '__force_dict__': True - } - } -} - ostro = { 'tries': CONF.music.tries, 'interval': CONF.music.interval, diff --git a/etc/valet/valet.conf b/etc/valet/valet.conf index 57b9db2..ed3c4f2 100644 --- a/etc/valet/valet.conf +++ b/etc/valet/valet.conf @@ -1,3 +1,9 @@ +[DEFAULT] +default_log_levels="api=DEBUG,valet=DEBUG,ostro_daemon=DEBUG,ostro_listener=INFO,music=INFO,requests=ERROR,pika=ERROR,pecan=ERROR,urllib3=ERROR" +logging_default_format_string='%(asctime)s.%(msecs)03d [%(levelname)-5.5s] [%(name)s] - %(message)s' +use_stderr=False +log_dir=/var/log/valet + # __ # /_\ |__| | # / \ | | diff --git a/valet/api/__init__.py b/valet/api/__init__.py index e69de29..85192b5 100644 --- a/valet/api/__init__.py +++ b/valet/api/__init__.py @@ -0,0 +1 @@ +LOG = None diff --git a/valet/api/app.py b/valet/api/app.py index 8e90e6b..2992b16 100644 --- a/valet/api/app.py +++ b/valet/api/app.py @@ -17,9 +17,11 @@ from pecan.deploy import deploy from pecan import make_app +from valet import api from valet.api.common import identity, messaging -from valet.api.conf import register_conf, set_domain from valet.api.db import models +from valet.common.conf import get_logger +from valet.common.conf import init_conf def setup_app(config): @@ -37,6 +39,7 @@ def setup_app(config): # entry point for apache2 def load_app(config_file): """App Load.""" - register_conf() - set_domain(project='valet') + init_conf("api.log") + api.LOG = get_logger("api") + return deploy(config_file) diff --git a/valet/api/common/hooks.py b/valet/api/common/hooks.py index f0e2bd2..7f16de9 100644 --- a/valet/api/common/hooks.py +++ b/valet/api/common/hooks.py @@ -16,27 +16,23 @@ """Hooks.""" import json -import logging -from valet.api.common.i18n import _ -from valet.api.common import terminate_thread -from valet.api.v1.controllers import error - from pecan import conf from pecan.hooks import PecanHook import threading +from valet import api +from valet.api.common.i18n import _ +from valet.api.common import terminate_thread +from valet.api.v1.controllers import error import webob -LOG = logging.getLogger(__name__) - - class MessageNotificationHook(PecanHook): """Send API request/responses out as Oslo msg notifications.""" def after(self, state): """Function sends valet notification.""" self.dummy = True - LOG.info('sending notification') + api.LOG.info('sending notification') notifier = conf.messaging.notifier status_code = state.response.status_code status = webob.exc.status_map.get(status_code) @@ -95,11 +91,11 @@ class MessageNotificationHook(PecanHook): notifier_thread = threading.Thread(target=notifier_fn, args=(ctxt, event_type, payload)) notifier_thread.start() # launch a timer to verify no hung threads are left behind - # (when timeout expired kill the notifier thread if it still alive) + # (when timeout expires kill the notifier thread if it still alive) watcher = threading.Timer(conf.messaging.timeout, terminate_thread, args=[notifier_thread]) watcher.start() - LOG.info('valet notification hook - end') + api.LOG.info('notification sent.') class NotFoundHook(PecanHook): diff --git a/valet/api/common/identity.py b/valet/api/common/identity.py index a062d22..9cc213d 100644 --- a/valet/api/common/identity.py +++ b/valet/api/common/identity.py @@ -22,11 +22,9 @@ import iso8601 from keystoneauth1.identity import v2 from keystoneauth1 import session from keystoneclient.v2_0 import client -import logging from pecan import conf import pytz - -LOG = logging.getLogger(__name__) +from valet.api import LOG def utcnow(): diff --git a/valet/api/common/messaging.py b/valet/api/common/messaging.py index ca66fcd..af2f743 100644 --- a/valet/api/common/messaging.py +++ b/valet/api/common/messaging.py @@ -18,7 +18,6 @@ from oslo_config import cfg import oslo_messaging as messaging from pecan import conf -from valet.api.conf import set_domain, DOMAIN def _messaging_notifier_from_config(config): @@ -33,7 +32,6 @@ def _messaging_notifier_from_config(config): def init_messaging(): """Initialize the messaging engine and place in the config.""" - set_domain(DOMAIN) config = conf.messaging.config notifier = _messaging_notifier_from_config(config) conf.messaging.notifier = notifier diff --git a/valet/api/common/ostro_helper.py b/valet/api/common/ostro_helper.py index 5c00507..5d6f6d1 100644 --- a/valet/api/common/ostro_helper.py +++ b/valet/api/common/ostro_helper.py @@ -16,7 +16,6 @@ """Ostro helper library.""" import json -import logging from pecan import conf import time @@ -27,8 +26,7 @@ from valet.api.db.models import Group from valet.api.db.models import PlacementRequest from valet.api.db.models import PlacementResult from valet.api.db.models import Query - -LOG = logging.getLogger(__name__) +from valet.api import LOG SERVICEABLE_RESOURCES = [ 'OS::Nova::Server' @@ -192,8 +190,8 @@ class Ostro(object): GROUP_TYPE, EXCLUSIVITY) - group = Group.query.filter_by( # pylint: disable=E1101 - name=group_name).first() + group = Group.query.filter_by(name=group_name).first() # pylint: disable=E1101 + if not group: self.error_uri = '/errors/not_found' return_message = "%s '%s' not found" % (GROUP_NAME, group_name) diff --git a/valet/api/db/models/music/__init__.py b/valet/api/db/models/music/__init__.py index 0efab51..82aff4a 100644 --- a/valet/api/db/models/music/__init__.py +++ b/valet/api/db/models/music/__init__.py @@ -21,7 +21,7 @@ from pecan import conf import six import uuid from valet.api.common.i18n import _ -from valet.api.db.models.music.music import Music +from valet.common.music import Music def get_class(kls): @@ -211,13 +211,18 @@ class Query(object): def __rows_to_objects(self, rows): """Convert query response rows to objects""" - results = [] - pk_name = self.model.pk_name() # pylint: disable=E1101 - for __, row in rows.iteritems(): # pylint: disable=W0612 - the_id = row.pop(pk_name) - result = self.model(_insert=False, **row) - setattr(result, pk_name, the_id) - results.append(result) + try: + results = [] + pk_name = self.model.pk_name() # pylint: disable=E1101 + for __, row in rows.iteritems(): # pylint: disable=W0612 + the_id = row.pop(pk_name) + result = self.model(_insert=False, **row) + setattr(result, pk_name, the_id) + results.append(result) + except Exception: + import traceback + print(traceback.format_exc()) + return Results(results) def all(self): diff --git a/valet/api/v1/commands/populate.py b/valet/api/v1/commands/populate.py index ec97188..b0fe9b2 100644 --- a/valet/api/v1/commands/populate.py +++ b/valet/api/v1/commands/populate.py @@ -16,9 +16,8 @@ """Populate command.""" from pecan.commands.base import BaseCommand - +from valet import api from valet.api.common.i18n import _ -from valet.api.conf import register_conf, set_domain from valet.api.db import models from valet.api.db.models import Event from valet.api.db.models import Group @@ -26,11 +25,7 @@ from valet.api.db.models import Placement from valet.api.db.models import PlacementRequest from valet.api.db.models import PlacementResult from valet.api.db.models import Plan - - -def out(string): - """Output helper.""" - print("==> %s" % string) +from valet.common.conf import init_conf, get_logger class PopulateCommand(BaseCommand): @@ -39,13 +34,15 @@ class PopulateCommand(BaseCommand): def run(self, args): """Function creates and initializes database and environment.""" super(PopulateCommand, self).run(args) - out(_("Loading environment")) - register_conf() - set_domain() - self.load_app() - out(_("Building schema")) try: - out(_("Starting a transaction...")) + init_conf("populate.log") +# cfg.CONF.log_file = "populate.log" +# cfg.CONF.use_stderr = True + LOG = api.LOG = get_logger("populate") + LOG.info(_("Loading environment")) + self.load_app() + LOG.info(_("Building schema")) + LOG.info(_("Starting a transaction...")) models.start() # FIXME: There's no create_all equivalent for Music. @@ -59,10 +56,10 @@ class PopulateCommand(BaseCommand): Event.create_table() PlacementRequest.create_table() PlacementResult.create_table() - except Exception: + except Exception as ex: models.rollback() - out(_("Rolling back...")) + LOG.error("Rolling back... %s" % ex) raise else: - out(_("Committing.")) + LOG.info(_("Committing.")) models.commit() diff --git a/valet/api/v1/controllers/__init__.py b/valet/api/v1/controllers/__init__.py index 9cd9f40..39082f8 100644 --- a/valet/api/v1/controllers/__init__.py +++ b/valet/api/v1/controllers/__init__.py @@ -15,28 +15,26 @@ """Controllers Package.""" -import logging from notario.decorators import instance_of from notario import ensure from os import path from pecan import redirect, request import string +from valet import api from valet.api.common.i18n import _ from valet.api.db.models import Placement -LOG = logging.getLogger(__name__) - - # # Notario Helpers # + def valid_group_name(value): """Validator for group name type.""" if not value or not set(value) <= set(string.letters + string.digits + "-._~"): - LOG.error("group name is not valid") - LOG.error("group name must contain only uppercase and lowercase letters, decimal digits, \ + api.LOG.error("group name is not valid") + api.LOG.error("group name must contain only uppercase and lowercase letters, decimal digits, \ hyphens, periods, underscores, and tildes [RFC 3986, Section 2.3]") @@ -73,16 +71,13 @@ def reserve_placement(placement, resource_id=None, reserve=True, update=True): the data store (if the update will be made later). """ if placement: - LOG.info(_('%(rsrv)s placement of %(orch_id)s in %(loc)s.'), - {'rsrv': _("Reserving") if reserve else _("Unreserving"), - 'orch_id': placement.orchestration_id, - 'loc': placement.location}) + api.LOG.info(_('%(rsrv)s placement of %(orch_id)s in %(loc)s.'), + {'rsrv': _("Reserving") if reserve else _("Unreserving"), + 'orch_id': placement.orchestration_id, 'loc': placement.location}) placement.reserved = reserve if resource_id: - LOG.info(_('Associating resource id %(res_id)s with ' - 'orchestration id %(orch_id)s.'), - {'res_id': resource_id, - 'orch_id': placement.orchestration_id}) + api.LOG.info(_('Associating resource id %(res_id)s with orchestration id %(orch_id)s.'), + {'res_id': resource_id, 'orch_id': placement.orchestration_id}) placement.resource_id = resource_id if update: placement.update() @@ -97,11 +92,9 @@ def update_placements(placements, reserve_id=None, unlock_all=False): properties = placements[uuid]['properties'] location = properties['host'] if placement.location != location: - LOG.info(_('Changing placement of %(orch_id)s ' - 'from %(old_loc)s to %(new_loc)s.'), - {'orch_id': placement.orchestration_id, - 'old_loc': placement.location, - 'new_loc': location}) + api.LOG.info(_('Changing placement of %(orch_id)s from %(old_loc)s to %(new_loc)s.'), + {'orch_id': placement.orchestration_id, 'old_loc': placement.location, + 'new_loc': location}) placement.location = location if unlock_all: reserve_placement(placement, reserve=False, update=False) diff --git a/valet/api/v1/controllers/errors.py b/valet/api/v1/controllers/errors.py index e0e3da0..47211b0 100644 --- a/valet/api/v1/controllers/errors.py +++ b/valet/api/v1/controllers/errors.py @@ -15,15 +15,11 @@ """Errors.""" -import logging from pecan import expose, request, response from valet.api.common.i18n import _ +from valet.api import LOG from webob.exc import status_map -LOG = logging.getLogger(__name__) - -# pylint: disable=R0201 - def error_wrapper(func): """Error decorator.""" diff --git a/valet/api/v1/controllers/groups.py b/valet/api/v1/controllers/groups.py index 3e05165..0d4912f 100644 --- a/valet/api/v1/controllers/groups.py +++ b/valet/api/v1/controllers/groups.py @@ -15,8 +15,6 @@ """Groups.""" -import logging - from notario import decorators from notario.validators import types from pecan import conf, expose, request, response @@ -27,8 +25,7 @@ from valet.api.common.i18n import _ from valet.api.common.ostro_helper import Ostro from valet.api.db.models import Group from valet.api.v1.controllers import error, valid_group_name - -LOG = logging.getLogger(__name__) +from valet import api GROUPS_SCHEMA = ( (decorators.optional('description'), types.string), @@ -82,8 +79,8 @@ def tenant_servers_in_group(tenant_id, group): if server.tenant_id == tenant_id: servers.append(server_id) except Exception as ex: # TODO(JD): update DB - LOG.error("Instance %s could not be found" % server_id) - LOG.error(ex) + api.LOG.error("Instance %s could not be found" % server_id) + api.LOG.error(ex) if len(servers) > 0: return servers @@ -292,9 +289,14 @@ class GroupsController(object): @index.when(method='GET', template='json') def index_get(self): """List groups.""" - groups_array = [] - for group in Group.query.all(): # pylint: disable=E1101 - groups_array.append(group) + try: + groups_array = [] + for group in Group.query.all(): # pylint: disable=E1101 + groups_array.append(group) + except Exception: + import traceback + api.LOG.error(traceback.format_exc()) + response.status = 500 return {'groups': groups_array} @index.when(method='POST', template='json') diff --git a/valet/api/v1/controllers/placements.py b/valet/api/v1/controllers/placements.py index 00b5cbe..497231b 100644 --- a/valet/api/v1/controllers/placements.py +++ b/valet/api/v1/controllers/placements.py @@ -15,18 +15,12 @@ """Placements.""" -import logging - from pecan import expose, request, response from valet.api.common.i18n import _ from valet.api.common.ostro_helper import Ostro from valet.api.db.models import Placement, Plan -from valet.api.v1.controllers import error -from valet.api.v1.controllers import reserve_placement -from valet.api.v1.controllers import update_placements - - -LOG = logging.getLogger(__name__) +from valet.api.v1.controllers import error, reserve_placement, update_placements +from valet import api # pylint: disable=R0201 @@ -80,12 +74,11 @@ class PlacementsItemController(object): Once reserved, the location effectively becomes immutable. """ res_id = kwargs.get('resource_id') - LOG.info(_('Placement reservation request for resource id ' - '%(res_id)s, orchestration id %(orch_id)s.'), - {'res_id': res_id, 'orch_id': self.placement.orchestration_id}) + api.LOG.info(_('Placement reservation request for resource id %(res_id)s, orchestration id %(orch_id)s.'), + {'res_id': res_id, 'orch_id': self.placement.orchestration_id}) locations = kwargs.get('locations', []) locations_str = ', '.join(locations) - LOG.info(_('Candidate locations: %s'), locations_str) + api.LOG.info(_('Candidate locations: %s'), locations_str) if self.placement.location in locations: # Ostro's placement is in the list of candidates. Good! # Reserve it. Remember the resource id too. @@ -95,12 +88,8 @@ class PlacementsItemController(object): else: # Ostro's placement is NOT in the list of candidates. # Time for Plan B. - LOG.info(_('Placement of resource id %(res_id)s, ' - 'orchestration id %(orch_id)s in %(loc)s ' - 'not allowed. Replanning.'), - {'res_id': res_id, - 'orch_id': self.placement.orchestration_id, - 'loc': self.placement.location}) + api.LOG.info(_('Placement of resource id %(res_id)s, orchestration id %(orch_id)s in %(loc)s not allowed. Replanning.'), + {'res_id': res_id, 'orch_id': self.placement.orchestration_id, 'loc': self.placement.location}) # Unreserve the placement. Remember the resource id too. kwargs = {'resource_id': res_id, 'reserve': False} @@ -117,9 +106,9 @@ class PlacementsItemController(object): exclusions = [x.orchestration_id for x in reserved] if exclusions: exclusions_str = ', '.join(exclusions) - LOG.info(_('Excluded orchestration IDs: %s'), exclusions_str) + api.LOG.info(_('Excluded orchestration IDs: %s'), exclusions_str) else: - LOG.info(_('No excluded orchestration IDs.')) + api.LOG.info(_('No excluded orchestration IDs.')) # Ask Ostro to try again with new constraints. # We may get one or more updated placements in return. @@ -158,7 +147,7 @@ class PlacementsItemController(object): """Delete a Placement.""" orch_id = self.placement.orchestration_id self.placement.delete() - LOG.info(_('Placement with orchestration id %s deleted.'), orch_id) + api.LOG.info(_('Placement with orchestration id %s deleted.'), orch_id) response.status = 204 diff --git a/valet/api/v1/controllers/plans.py b/valet/api/v1/controllers/plans.py index 0638dec..db8da78 100644 --- a/valet/api/v1/controllers/plans.py +++ b/valet/api/v1/controllers/plans.py @@ -15,22 +15,16 @@ """Plans.""" -import logging - from notario import decorators from notario.validators import types from pecan import expose, request, response from pecan_notario import validate - from valet.api.common.i18n import _ from valet.api.common.ostro_helper import Ostro from valet.api.db.models import Placement, Plan -from valet.api.v1.controllers import error -from valet.api.v1.controllers import set_placements -from valet.api.v1.controllers import update_placements -from valet.api.v1.controllers import valid_plan_update_action +from valet.api.v1.controllers import error, set_placements, update_placements, valid_plan_update_action +from valet.api import LOG -LOG = logging.getLogger(__name__) CREATE_SCHEMA = ( ('plan_name', types.string), diff --git a/valet/api/v1/controllers/root.py b/valet/api/v1/controllers/root.py index 1df6d2f..7211a04 100644 --- a/valet/api/v1/controllers/root.py +++ b/valet/api/v1/controllers/root.py @@ -15,8 +15,6 @@ """Root.""" -import logging - from pecan import expose, request, response from valet.api.common.i18n import _ from valet.api.v1.controllers import error @@ -25,8 +23,6 @@ from valet.api.v1.controllers.v1 import V1Controller from webob.exc import status_map -LOG = logging.getLogger(__name__) - # pylint: disable=R0201 diff --git a/valet/api/v1/controllers/status.py b/valet/api/v1/controllers/status.py index cdf2c28..39bc1cf 100644 --- a/valet/api/v1/controllers/status.py +++ b/valet/api/v1/controllers/status.py @@ -15,15 +15,11 @@ """Status.""" -import logging - from pecan import expose, request, response from valet.api.common.i18n import _ from valet.api.common.ostro_helper import Ostro from valet.api.v1.controllers import error -LOG = logging.getLogger(__name__) - # pylint: disable=R0201 diff --git a/valet/api/v1/controllers/v1.py b/valet/api/v1/controllers/v1.py index c88e095..36c4a47 100644 --- a/valet/api/v1/controllers/v1.py +++ b/valet/api/v1/controllers/v1.py @@ -15,11 +15,9 @@ """v1.""" -import logging - from pecan import conf, expose, request, response from pecan.secure import SecureController - +from valet import api from valet.api.common.i18n import _ from valet.api.v1.controllers import error from valet.api.v1.controllers.groups import GroupsController @@ -28,11 +26,6 @@ from valet.api.v1.controllers.plans import PlansController from valet.api.v1.controllers.status import StatusController -LOG = logging.getLogger(__name__) - -# pylint: disable=R0201 - - class V1Controller(SecureController): """v1 Controller /v1.""" @@ -58,11 +51,11 @@ class V1Controller(SecureController): token = conf.identity.engine.validate_token(auth_token) if token: - LOG.debug("Checking token permissions") + api.LOG.debug("Checking token permissions") msg = "Unauthorized - Permission was not granted" if V1Controller._permission_granted(request, token): tenant_id = conf.identity.engine.tenant_from_token(token) - LOG.info("tenant_id - " + str(tenant_id)) + api.LOG.info("tenant_id - " + str(tenant_id)) if tenant_id: request.context['tenant_id'] = tenant_id user_id = conf.identity.engine.user_from_token(token) diff --git a/valet/api/wsgi.py b/valet/api/wsgi.py index fe89b94..7b264f7 100644 --- a/valet/api/wsgi.py +++ b/valet/api/wsgi.py @@ -18,6 +18,8 @@ from common.i18n import _ import os from pecan.deploy import deploy +from valet.common.conf import init_conf, get_logger +from valet import api def config_file(file_name=None): @@ -39,15 +41,20 @@ def application(environ, start_response): # TODO(JD): Integrate this with a python entry point # This way we can run valet-api from the command line in a pinch. if __name__ == '__main__': - from wsgiref.simple_server import make_server # disable=C0411,C0413 + try: + from wsgiref.simple_server import make_server # disable=C0411,C0413 - # TODO(JD): At some point, it would be nice to use pecan_mount - from valet.api.conf import register_conf, set_domain - register_conf() - set_domain() - HTTPD = make_server('', 8090, - deploy(config_file('/var/www/valet/config.py'))) - print(_("Serving HTTP on port 8090...")) + # TODO(JD): At some point, it would be nice to use pecan_mount + # import pecan_mount + # HTTPD = make_server('', 8090, pecan_mount.tree) +# from valet.api.conf import register_conf, set_domain + init_conf("wsgi.log") + api.LOG = get_logger("wsgi") + HTTPD = make_server('', 8090, deploy(config_file('/var/www/valet/config.py'))) + print(_("Serving HTTP on port 8090...")) - # Respond to requests until process is killed - HTTPD.serve_forever() + # Respond to requests until process is killed + HTTPD.serve_forever() + except Exception: + import traceback + print(traceback.format_exc()) diff --git a/valet/cli/groupcli.py b/valet/cli/groupcli.py index 7d2b77b..3bcf6d6 100644 --- a/valet/cli/groupcli.py +++ b/valet/cli/groupcli.py @@ -19,7 +19,7 @@ import argparse import json from oslo_config import cfg import requests -from valet.api.conf import register_conf, set_domain +from valet.common import conf as common CONF = cfg.CONF @@ -211,8 +211,7 @@ def populate_args_request_body(args): def run(args): """Run.""" - register_conf() - set_domain(project='valet') + common.init_conf("cli.log") args.host = args.host or CONF.server.host args.port = args.port or CONF.server.port args.timeout = args.timeout or 10 diff --git a/valet/cli/valetcli.py b/valet/cli/valetcli.py index fab5e2b..3e8215c 100755 --- a/valet/cli/valetcli.py +++ b/valet/cli/valetcli.py @@ -18,7 +18,6 @@ import argparse import sys import valet.cli.groupcli as groupcli -# import logging class Cli(object): diff --git a/valet/common/__init__.py b/valet/common/__init__.py new file mode 100644 index 0000000..3befeaf --- /dev/null +++ b/valet/common/__init__.py @@ -0,0 +1,15 @@ + +from oslo_config import cfg + + +def logger_conf(logger_name): + return [ + cfg.StrOpt('output_format', default="%(asctime)s - %(levelname)s - %(message)s"), # dict + cfg.BoolOpt('store', default=True), + cfg.StrOpt('logging_level', default='debug'), + cfg.StrOpt('logging_dir', default='/var/log/valet/'), + cfg.StrOpt('logger_name', default=logger_name + ".log"), + cfg.IntOpt('max_main_log_size', default=5000000), + cfg.IntOpt('max_log_size', default=1000000), + cfg.IntOpt('max_num_of_logs', default=3), + ] diff --git a/valet/api/conf.py b/valet/common/conf.py similarity index 68% rename from valet/api/conf.py rename to valet/common/conf.py index 59552bc..96fbaef 100644 --- a/valet/api/conf.py +++ b/valet/common/conf.py @@ -16,12 +16,16 @@ """Conf.""" from oslo_config import cfg +from oslo_log import log as logging - +CONF = cfg.CONF DOMAIN = 'valet' -CONF = cfg.CONF +def get_logger(name): + return logging.getLogger(name) + +LOG = get_logger("engine") server_group = cfg.OptGroup(name='server', title='Valet API Server conf') server_opts = [ @@ -70,19 +74,34 @@ music_opts = [ # cfg.ListOpt('db_hosts', default='valet1,valet2,valet3') ] - -def set_domain(project=DOMAIN): - """Set Domain.""" - CONF([], project) +def load_conf(args=None, project=DOMAIN, default_files=None): + CONF(default_config_files=default_files) if default_files else CONF(args or [], project=project) -def register_conf(): - """Register confs.""" - CONF.register_group(server_group) - CONF.register_opts(server_opts, server_group) - CONF.register_group(music_group) - CONF.register_opts(music_opts, music_group) - CONF.register_group(identity_group) - CONF.register_opts(identity_opts, identity_group) - CONF.register_group(messaging_group) - CONF.register_opts(messaging_opts, messaging_group) +def init_conf(log_file="valet.log", args=None, grp2opt=None, cli_opts=None, default_config_files=None): + CONF.log_file = log_file + logging.register_options(CONF) + + # init conf + general_groups = {server_group: server_opts, music_group: music_opts, + identity_group: identity_opts, messaging_group: messaging_opts} + + general_groups.update(grp2opt or {}) + + _register_conf(general_groups, cli_opts) + load_conf(args=args, default_files=default_config_files) + + # set logger + _set_logger() + + +def _set_logger(): + logging.setup(CONF, DOMAIN) + +def _register_conf(grp2opt, cli_opts): + for grp in grp2opt or {}: + CONF.register_group(grp) + CONF.register_opts(grp2opt[grp], grp) + + for opt in cli_opts or []: + CONF.register_cli_opts(opt) diff --git a/valet/api/db/models/music/music.py b/valet/common/music.py similarity index 96% rename from valet/api/db/models/music/music.py rename to valet/common/music.py index d56e189..2c05dd5 100644 --- a/valet/api/db/models/music/music.py +++ b/valet/common/music.py @@ -16,14 +16,12 @@ """Music Data Store API.""" import json -import logging -import time - -from valet.api.common.i18n import _ - import requests +import time +from valet.api.common.i18n import _ +from valet.common.conf import get_logger -LOG = logging.getLogger(__name__) +LOG = get_logger("music") class REST(object): @@ -47,7 +45,8 @@ class REST(object): def urls(self): """Return list of URLs using each host, plus the port/path.""" if not self._urls: - urls = [] + # make localhost as first option + urls = ['http://localhost:%s%s' % (self.port, self.path)] for host in self.hosts: # Must end without a slash urls.append('http://%(host)s:%(port)s%(path)s' % { @@ -81,12 +80,10 @@ class REST(object): full_url = url + path try: data_json = json.dumps(data) if data else None - LOG.debug("Music Request: %s %s%s", method.upper(), full_url, - data_json if data else '') - response = method_fn(full_url, data=data_json, - headers=self.__headers(content_type), - timeout=self.timeout) + LOG.debug("Music Request: %s %s%s", method.upper(), full_url, data_json if data else '') + response = method_fn(full_url, data=data_json, headers=self.__headers(content_type), timeout=self.timeout) response.raise_for_status() + return response except requests.exceptions.Timeout as err: response = requests.Response() @@ -151,6 +148,7 @@ class Music(object): path = '/keyspaces/%s' % keyspace response = self.rest.request(method='post', path=path, data=data) + return response.ok def create_table(self, keyspace, table, schema): diff --git a/valet/engine/conf.py b/valet/engine/conf.py index fd6a948..6c9a38a 100644 --- a/valet/engine/conf.py +++ b/valet/engine/conf.py @@ -16,7 +16,8 @@ """Conf.""" from oslo_config import cfg -from valet.api import conf as api +import sys +from valet.common import logger_conf, conf as common CONF = cfg.CONF @@ -27,178 +28,47 @@ ostro_cli_opts = [ help='engine command.'), ] - engine_group = cfg.OptGroup(name='engine', title='Valet Engine conf') engine_opts = [ - cfg.StrOpt( - 'pid', - default='/var/run/valet/ostro-daemon.pid' - ), - cfg.StrOpt( - 'mode', - default='live', - help=""" -Sim will let Ostro simulate datacenter, while live will -let it handle a real datacenter. -"""), - cfg.StrOpt( - 'sim_cfg_loc', - default='/etc/valet/engine/ostro_sim.cfg'), - cfg.BoolOpt( - 'network_control', - default=False, - help=""" -Whether network controller (i.e., Tegu) has been deployed -"""), - cfg.StrOpt( - 'network_control_url', - default='http://network_control:29444/tegu/api'), - cfg.StrOpt( - 'ip', - default='localhost'), - cfg.IntOpt( - 'priority', - default=1, - help=""" -This instance priority (master=1) -"""), - cfg.StrOpt( - 'rpc_server_ip', - default='localhost', - help=""" -Set RPC server ip and port if used. Otherwise, ignore these parameters -"""), - cfg.StrOpt( - 'rpc_server_port', - default='8002' - ), - cfg.StrOpt( - 'logger_name', - default='engine.log' - ), - cfg.StrOpt( - 'logging_level', - default='debug' - ), - cfg.StrOpt( - 'logging_dir', - default='/var/log/valet/' - ), - cfg.StrOpt( - 'max_main_log_size', - default=5000000 - ), - cfg.IntOpt( - 'max_log_size', - default=1000000 - ), - cfg.IntOpt( - 'max_num_of_logs', - default=20 - ), - cfg.StrOpt( - 'datacenter_name', - default='bigsite', - help=""" -Inform the name of datacenter (region name), where Valet/Ostro is deployed. -"""), - cfg.IntOpt( - 'num_of_region_chars', - default='3', - help=""" -Number of chars that indicates the region code -"""), - cfg.StrOpt( - 'rack_code_list', - default='r', - help=""" -Rack indicator. -"""), - cfg.ListOpt( - 'node_code_list', - default='a,c,u,f,o,p,s', - help=""" -Indicates the node type. - -Values: - -* a: network -* c KVM compute -* u: ESXi compute -* f: ? -* o: operation -* p: power -* s: storage. -"""), - cfg.StrOpt( - 'compute_trigger_time', - default='1:00', - help=""" -Trigger time or frequency for checking compute hosting server status -(i.e., call Nova) -"""), - cfg.IntOpt( - 'compute_trigger_frequency', - default=3600, - help=""" -Trigger time or frequency for checking compute hosting server status -(i.e., call Nova). -"""), - cfg.StrOpt( - 'topology_trigger_time', - default='2:00', - help=""" -Set trigger time or frequency for checking datacenter topology. -"""), - cfg.IntOpt( - 'topology_trigger_frequency', - default=3600, - help=""" -Set trigger time or frequency for checking datacenter topology. -"""), - cfg.IntOpt( - 'default_cpu_allocation_ratio', - default=16, - help=""" -Set default overbooking ratios. -Note that each compute node can have its own ratios. -"""), - cfg.IntOpt( - 'default_ram_allocation_ratio', - default=1.5, - help=""" -Set default overbooking ratios. -Note that each compute node can have its own ratios. -"""), - cfg.IntOpt( - 'default_disk_allocation_ratio', - default=1, - help=""" -Set default overbooking ratios. -Note that each compute node can have its own ratios. -"""), - cfg.IntOpt( - 'static_cpu_standby_ratio', - default=20, - help=""" -Unused percentages of resources (i.e. standby) that are set -aside for applications workload spikes. -"""), - cfg.IntOpt( - 'static_mem_standby_ratio', - default=20, - help=""" -Unused percentages of resources (i.e. standby) that are set -aside for applications workload spikes. -"""), - cfg.IntOpt( - 'static_local_disk_standby_ratio', - default=20, - help=""" -Unused percentages of resources (i.e. standby) that are set -aside for applications workload spikes. -"""), -] + cfg.StrOpt('pid', default='/var/run/valet/ostro-daemon.pid'), + cfg.StrOpt('mode', default='live', help='sim will let Ostro simulate datacenter, while live will let it handle a real datacenter'), + cfg.StrOpt('sim_cfg_loc', default='/etc/valet/engine/ostro_sim.cfg'), + cfg.BoolOpt('network_control', default=False, help='whether network controller (i.e., Tegu) has been deployed'), + cfg.StrOpt('network_control_url', default='http://network_control:29444/tegu/api'), + cfg.StrOpt('ip', default='localhost'), + cfg.IntOpt('health_timeout', default=6, help='health check grace period (seconds, default=5)'), + cfg.IntOpt('priority', default=1, help='this instance priority (master=1)'), + cfg.StrOpt('rpc_server_ip', default='localhost', + help='Set RPC server ip and port if used. Otherwise, ignore these parameters'), + cfg.StrOpt('rpc_server_port', default='8002'), + cfg.StrOpt('datacenter_name', default='bigsite', + help='Inform the name of datacenter (region name), where Valet/Ostro is deployed.'), + cfg.IntOpt('num_of_region_chars', default='3', help='number of chars that indicates the region code'), + cfg.StrOpt('rack_code_list', default='r', help='rack indicator.'), + cfg.ListOpt('node_code_list', default='a,c,u,f,o,p,s', + help='indicates the node type. a: network, c KVM compute, u: ESXi compute, f: ?, o: operation, ' + 'p: power, s: storage.'), + cfg.StrOpt('compute_trigger_time', default='1:00', + help='trigger time or frequency for checking compute hosting server status (i.e., call Nova)'), + cfg.IntOpt('compute_trigger_frequency', default=3600, + help='trigger time or frequency for checking compute hosting server status (i.e., call Nova)'), + cfg.StrOpt('topology_trigger_time', default='2:00', + help='Set trigger time or frequency for checking datacenter topology (i.e., call AIC Formation)'), + cfg.IntOpt('topology_trigger_frequency', default=3600, + help='Set trigger time or frequency for checking datacenter topology (i.e., call AIC Formation)'), + cfg.IntOpt('default_cpu_allocation_ratio', default=16, help='Set default overbooking ratios. ' + 'Note that each compute node can have its own ratios'), + cfg.IntOpt('default_ram_allocation_ratio', default=1.5, help='Set default overbooking ratios. ' + 'Note that each compute node can have its own ratios'), + cfg.IntOpt('default_disk_allocation_ratio', default=1, help='Set default overbooking ratios. ' + 'Note that each compute node can have its own ratios'), + cfg.IntOpt('static_cpu_standby_ratio', default=20, help='unused percentages of resources (i.e. standby) ' + 'that are set aside for applications workload spikes.'), + cfg.IntOpt('static_mem_standby_ratio', default=20, help='unused percentages of resources (i.e. standby) ' + 'that are set aside for applications workload spikes.'), + cfg.IntOpt('static_local_disk_standby_ratio', default=20, help='unused percentages of resources (i.e. standby) ' + 'that are set aside for applications workload spikes.'), +] + logger_conf("engine") listener_group = cfg.OptGroup(name='events_listener', title='Valet Engine listener') @@ -206,20 +76,14 @@ listener_opts = [ cfg.StrOpt('exchange', default='nova'), cfg.StrOpt('exchange_type', default='topic'), cfg.BoolOpt('auto_delete', default=False), - cfg.StrOpt('output_format', default='dict'), cfg.BoolOpt('store', default=True), - cfg.StrOpt('logging_level', default='debug'), - cfg.StrOpt('logging_loc', default='/var/log/valet/'), - cfg.StrOpt('logger_name', default='ostro_listener.log'), - cfg.IntOpt('max_main_log_size', default=5000000), -] +] + logger_conf("ostro_listener") -def register_conf(): - """Function calls api and registers configs opts.""" - api.register_conf() - CONF.register_group(engine_group) - CONF.register_opts(engine_opts, engine_group) - CONF.register_group(listener_group) - CONF.register_opts(listener_opts, listener_group) - CONF.register_cli_opts(ostro_cli_opts) +def init_engine(default_config_files=None): + """ register the engine and the listener groups """ + common.init_conf("engine.log", args=sys.argv[1:], + grp2opt={engine_group: engine_opts, + listener_group: listener_opts}, + cli_opts=[ostro_cli_opts], + default_config_files=default_config_files) diff --git a/valet/engine/listener/listener_manager.py b/valet/engine/listener/listener_manager.py index 3e1944f..64f6e64 100644 --- a/valet/engine/listener/listener_manager.py +++ b/valet/engine/listener/listener_manager.py @@ -21,9 +21,9 @@ import pika import pprint import threading import traceback -from valet.api.db.models.music import Music +from valet.common.conf import get_logger +from valet.common.music import Music from valet.engine.listener.oslo_messages import OsloMessage -from valet.engine.optimizer.util.util import init_logger import yaml @@ -36,7 +36,7 @@ class ListenerManager(threading.Thread): self.thread_id = _t_id self.thread_name = _t_name self.config = _config - self.listener_logger = init_logger(self.config.events_listener) + self.listener_logger = get_logger("ostro_listener") self.MUSIC = None def run(self): diff --git a/valet/engine/optimizer/app_manager/app_handler.py b/valet/engine/optimizer/app_manager/app_handler.py index 0231f07..e83c409 100755 --- a/valet/engine/optimizer/app_manager/app_handler.py +++ b/valet/engine/optimizer/app_manager/app_handler.py @@ -181,11 +181,9 @@ class AppHandler(object): return vm_info def update_vm_info(self, _s_uuid, _h_uuid): - """Update vm info (the ids) in the database.""" - s_uuid_exist = bool(_s_uuid is not None and _s_uuid != "none") - h_uuid_exist = bool(_h_uuid is not None and _h_uuid != "none") - if s_uuid_exist and h_uuid_exist: + if _h_uuid and _h_uuid != "none" and _s_uuid and _s_uuid != "none": return self.db.update_vm_info(_s_uuid, _h_uuid) + return True def _regenerate_app_topology(self, _stack_id, _app, _app_topology, _action): diff --git a/valet/engine/optimizer/db_connect/music_handler.py b/valet/engine/optimizer/db_connect/music_handler.py index 79df55b..0a1b9ca 100644 --- a/valet/engine/optimizer/db_connect/music_handler.py +++ b/valet/engine/optimizer/db_connect/music_handler.py @@ -17,7 +17,7 @@ import json import operator -from valet.api.db.models.music import Music +from valet.common.music import Music from valet.engine.optimizer.db_connect.event import Event diff --git a/valet/engine/optimizer/ostro/ostro.py b/valet/engine/optimizer/ostro/ostro.py index 6d563d8..7e8e5e8 100755 --- a/valet/engine/optimizer/ostro/ostro.py +++ b/valet/engine/optimizer/ostro/ostro.py @@ -135,26 +135,26 @@ class Ostro(object): resource_status = self.db.get_resource_status( self.resource.datacenter.name) if resource_status is None: + self.logger.error("Ostro.bootstrap: failed to read from table: " + self.config.db_resource_table) return False if len(resource_status) > 0: self.logger.info("Ostro.bootstrap: bootstrap from db") - if self.resource.bootstrap_from_db(resource_status) is False: - return False - else: - self.logger.info("bootstrap from OpenStack") + if not self.resource.bootstrap_from_db(resource_status): + self.logger.error("Ostro.bootstrap: failed to parse bootstrap data!") - if self._set_hosts() is False: - self.logger.error('_set_hosts is false') - return False + self.logger.info("read bootstrap data from OpenStack") + if not self._set_hosts(): + self.logger.error('_set_hosts is false') + return False - if self._set_flavors() is False: - self.logger.info("_set_flavors is false") - return False + if not self._set_flavors(): + self.logger.info("_set_flavors is false") + return False - if self._set_topology() is False: - self.logger.error("_set_topology is false") - return False + if not self._set_topology(): + self.logger.error("_set_topology is false") + return False self.resource.update_topology() @@ -167,29 +167,31 @@ class Ostro(object): return True def _set_topology(self): - if self.topology.set_topology() is False: + if not self.topology.set_topology(): self.status = "datacenter configuration error" + self.logger.error("failed to read datacenter topology") return False self.logger.debug("done topology bootstrap") - return True def _set_hosts(self): - if self.compute.set_hosts() is False: + if not self.compute.set_hosts(): self.status = "OpenStack (Nova) internal error" + self.logger.error("failed to read hosts from OpenStack (Nova)") return False + self.logger.debug("done hosts & groups bootstrap") return True def _set_flavors(self): self.logger.debug("start flavors bootstrap") - if self.compute.set_flavors() is False: + if not self.compute.set_flavors(): self.status = "OpenStack (Nova) internal error" + self.logger.error("failed to read flavors from OpenStack (Nova)") return False self.logger.debug("done flavors bootstrap") - return True def place_app(self, _app_data): @@ -263,21 +265,21 @@ class Ostro(object): query_results[q["stack_id"]] = vm_list else: self.status = "unknown paramenter in query" - self.logger.warn("Ostro._query: " + self.status) + self.logger.warn("Ostro._query: unknown paramenter in query") query_results[q["stack_id"]] = None else: - self.status = "no parameters in query" - self.logger.warn("Ostro._query: " + self.status) + self.status = "no paramenter in query" + self.logger.warn("Ostro._query: no parameters in query") query_results[q["stack_id"]] = None elif q["type"] == "all_groups": query_results[q["stack_id"]] = self._get_logical_groups() else: self.status = "unknown query type" - self.logger.warn("Ostro._query: " + self.status) + self.logger.warn("Ostro._query: unknown query type") query_results[q["stack_id"]] = None else: - self.status = "no type in query" - self.logger.warn("Ostro._query: " + self.status) + self.status = "unknown type in query" + self.logger.warn("Ostro._query: no type in query") query_results[q["stack_id"]] = None return query_results @@ -313,20 +315,20 @@ class Ostro(object): app_topology = self.app_handler.add_app(_app_data) if app_topology is None: self.status = self.app_handler.status - self.logger.debug("Ostro._place_app: error while register " - "requested apps: " + self.status) + self.logger.error("Ostro._place_app: error while register" + "requested apps: " + self.app_handler.status) return None """Check and set vm flavor information.""" for _, vm in app_topology.vms.iteritems(): if self._set_vm_flavor_information(vm) is False: self.status = "fail to set flavor information" - self.logger.error("Ostro._place_app: " + self.status) + self.logger.error("Ostro._place_app: failed to set flavor information ") return None for _, vg in app_topology.vgroups.iteritems(): if self._set_vm_flavor_information(vg) is False: self.status = "fail to set flavor information in a group" - self.logger.error("Ostro._place_app: " + self.status) + self.logger.error("Ostro._place_app: failed to set flavor information in a group") return None """Set weights for optimization.""" @@ -688,7 +690,7 @@ class Ostro(object): app_status['message'] = "ping" app_result['status'] = app_status - app_result['resources'] = {"ip": self.config.ip} + app_result['resources'] = {"ip": self.config.ip, "id": self.config.priority} result[appk] = app_result diff --git a/valet/engine/optimizer/ostro_server/configuration.py b/valet/engine/optimizer/ostro_server/configuration.py index 3be8f1c..9cac9e4 100755 --- a/valet/engine/optimizer/ostro_server/configuration.py +++ b/valet/engine/optimizer/ostro_server/configuration.py @@ -19,7 +19,7 @@ import os from oslo_config import cfg -from valet.engine.conf import register_conf +from valet.engine.conf import init_engine CONF = cfg.CONF @@ -29,12 +29,7 @@ class Config(object): """Valet Engine Server Configuration.""" def __init__(self, *default_config_files): - """Initialization.""" - register_conf() - if default_config_files: - CONF(default_config_files=default_config_files) - else: - CONF(project='valet') + init_engine(default_config_files=default_config_files) # System parameters self.root_loc = os.path.dirname(CONF.default_config_files[0]) diff --git a/valet/engine/optimizer/ostro_server/daemon.py b/valet/engine/optimizer/ostro_server/daemon.py index dad15d1..0c0dcb9 100644 --- a/valet/engine/optimizer/ostro_server/daemon.py +++ b/valet/engine/optimizer/ostro_server/daemon.py @@ -17,10 +17,13 @@ import atexit import os +from oslo_config import cfg from signal import SIGTERM import sys import time +CONF = cfg.CONF + class Daemon(object): """A generic daemon class.""" @@ -82,10 +85,10 @@ class Daemon(object): os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(se.fileno(), sys.stderr.fileno()) - # write pidfile atexit.register(self.delpid) - pid = str(os.getpid()) - file(self.pidfile, 'w+').write("%s\n" % pid) + # write pidfile - moved to OstroDaemon.run + # pid = str(os.getpid()) + # file(self.pidfile, 'w+').write("%s\n" % pid) def delpid(self): """Remove pidfile.""" @@ -102,17 +105,16 @@ class Daemon(object): return pid def checkpid(self, pid): - """Check for the existence of a UNIX pid.""" - if pid is None: - return False - + """ Check For the existence of a unix pid. """ + alive = False try: - os.kill(pid, 0) + if pid: + os.kill(pid, 0) + alive = True except OSError: self.delpid() - return False - else: - return True + + return alive def start(self): """Start thedaemon.""" @@ -152,11 +154,6 @@ class Daemon(object): # print str(err) sys.exit(1) - def restart(self): - """Restart the daemon.""" - self.stop() - self.start() - def status(self): """Return instance's priority.""" # Check for a pidfile to see if the daemon already runs @@ -174,6 +171,11 @@ class Daemon(object): sys.stderr.write(message % self.pidfile) return status + def restart(self): + """Restart the daemon""" + self.stop() + self.start() + def run(self): """You should override this method when you subclass Daemon.""" """It will be called after the process has been daemonized by diff --git a/valet/engine/optimizer/ostro_server/db_cleaner.py b/valet/engine/optimizer/ostro_server/db_cleaner.py index 1bfe300..f488618 100644 --- a/valet/engine/optimizer/ostro_server/db_cleaner.py +++ b/valet/engine/optimizer/ostro_server/db_cleaner.py @@ -18,8 +18,7 @@ """Database Cleaner.""" from configuration import Config -import sys -from valet.api.db.models.music import Music +from valet.common.music import Music class DBCleaner(object): diff --git a/valet/engine/optimizer/ostro_server/health_checker.py b/valet/engine/optimizer/ostro_server/health_checker.py new file mode 100644 index 0000000..a3a3939 --- /dev/null +++ b/valet/engine/optimizer/ostro_server/health_checker.py @@ -0,0 +1,125 @@ +import json +import os +from oslo_config import cfg +import sys +import time +import uuid +from valet.common.conf import get_logger +from valet.common.music import REST +from valet.engine.conf import init_engine + +CONF = cfg.CONF + + +class HealthCheck(object): + + rest = None + + def __init__(self, hosts=[], port='8080', keyspace='valet'): + + self.tries = CONF.engine.health_timeout * 2 + self.uuid = str(uuid.uuid4()) + + kwargs = { + 'hosts': hosts, + 'port': CONF.music.port, + 'path': '/MUSIC/rest', + 'timeout': CONF.music.interval, + } + self.rest = REST(**kwargs) + + def ping(self, my_id): + + engine_alive = False + try: + if self._send(): + engine_alive = self._read_response(my_id) + finally: + self._delete_result() + return engine_alive + + def _send(self): + + data = { + "values": {"stack_id": self.uuid, + "request": "[{\"action\": \"ping\", \"stack_id\": \"" + self.uuid + "\"}]" + }, + "consistencyInfo": {"type": "eventual"} + } + + path = '/keyspaces/%(keyspace)s/tables/%(table)s/rows' % { + 'keyspace': CONF.music.keyspace, + 'table': CONF.music.request_table, + } + response = self.rest.request(method='post', path=path, data=data) + # print "SEND response: " + str(response.status_code) + return response.status_code == 204 if response else False + + def _read_response(self, my_id): + + found = False + path = '/keyspaces/%(keyspace)s/tables/%(table)s/rows?stack_id=%(uid)s' % { + 'keyspace': CONF.music.keyspace, + 'table': CONF.music.response_table, + 'uid': self.uuid, + } + + for i in range(self.tries): # default: 12 * 0.5 = 6 sec. + time.sleep(0.5) + response = self.rest.request(method='get', path=path) + + # logger.debug("READ respons body text: " + str(response.text)) + + if response.status_code == 200 and len(response.text) > 3: + + j = json.loads(response.text) + stack_id = j['row 0']['stack_id'] + placement = json.loads(j['row 0']['placement']) + engine_id = placement['resources']['id'] + + # logger.debug("health 'stack_id': " + stack_id + ", engine_id=" + str(engine_id)) + if stack_id == self.uuid and engine_id == my_id: + found = True + break + + return found + + def _delete_result(self): + # leave the table clean - delete from requests and responses + try: + path = '/keyspaces/%(keyspace)s/tables/%(table)s/rows?stack_id=%(uid)s' % { + 'keyspace': CONF.music.keyspace, + 'table': CONF.music.request_table, + 'uid': self.uuid + } + data = { + "consistencyInfo": {"type": "eventual"} + } + self.rest.request(method='delete', path=path, data=data) + + path = '/keyspaces/%(keyspace)s/tables/%(table)s/rows?stack_id=%(uid)s' % { + 'keyspace': CONF.music.keyspace, + 'table': CONF.music.response_table, + 'uid': self.uuid + } + self.rest.request(method='delete', path=path, data=data) + + # print "DELETE response: " + str(response.status_code) + except Exception: + pass + + +if __name__ == "__main__": + + alive = False + code = 0 + init_engine(default_config_files=['/etc/valet/valet.conf']) + logger = get_logger("ostro_daemon") + if os.path.exists(CONF.engine.pid): + alive = HealthCheck().ping(CONF.engine.priority) + if alive: + code = CONF.engine.priority + logger.info("HealthCheck - Alive, priority = {}".format(CONF.engine.priority)) + else: + logger.warn("HealthCheck - Engine is DEAD!") + sys.exit(code) diff --git a/valet/engine/optimizer/ostro_server/ostro_daemon.py b/valet/engine/optimizer/ostro_server/ostro_daemon.py index f637bfb..1f3c5ee 100755 --- a/valet/engine/optimizer/ostro_server/ostro_daemon.py +++ b/valet/engine/optimizer/ostro_server/ostro_daemon.py @@ -18,10 +18,10 @@ import os import sys import traceback +from valet.common.conf import get_logger from valet.engine.optimizer.ostro.ostro import Ostro from valet.engine.optimizer.ostro_server.configuration import Config from valet.engine.optimizer.ostro_server.daemon import Daemon -from valet.engine.optimizer.util.util import init_logger class OstroDaemon(Daemon): @@ -39,6 +39,10 @@ class OstroDaemon(Daemon): self.logger.error("ostro bootstrap failed") sys.exit(2) + # write pidfile + pid = str(os.getpid()) + file(self.pidfile, 'w+').write("%s\n" % pid) + ostro.run_ostro() @@ -58,6 +62,8 @@ if __name__ == "__main__": # Configuration try: config = Config() + ''' logger ''' + logger = get_logger("ostro_daemon") config_status = config.configure() if config_status != "success": print(config_status) @@ -68,9 +74,6 @@ if __name__ == "__main__": config.app_log_loc, os.path.dirname(config.process)] verify_dirs(dirs_list) - """ logger """ - logger = init_logger(config) - # Start daemon process daemon = OstroDaemon(config.priority, config.process, logger) @@ -85,7 +88,7 @@ if __name__ == "__main__": exit_code = exit_code or 0 except Exception: - logger.error(traceback.format_exc()) + print(traceback.format_exc()) exit_code = 2 sys.exit(int(exit_code)) diff --git a/valet/engine/optimizer/util/util.py b/valet/engine/optimizer/util/util.py index e4abfc4..1b34617 100755 --- a/valet/engine/optimizer/util/util.py +++ b/valet/engine/optimizer/util/util.py @@ -17,8 +17,6 @@ from os import listdir, stat from os.path import isfile, join -import logging -from logging.handlers import RotatingFileHandler def get_logfile(_loc, _max_log_size, _name): @@ -88,22 +86,3 @@ def adjust_json_string(_data): _data = _data.replace('_"true"', "_true") return _data - - -def init_logger(config): - """Return an initialized logger.""" - log_formatter = logging.Formatter("%(asctime)s - %(levelname)s - " - "%(message)s") - log_handler = RotatingFileHandler(config.logging_loc + config.logger_name, - mode='a', - maxBytes=config.max_main_log_size, - backupCount=2, - encoding=None, - delay=0) - log_handler.setFormatter(log_formatter) - logger = logging.getLogger(config.logger_name) - logger.setLevel(logging.DEBUG if config.logging_level == "debug" - else logging.INFO) - logger.addHandler(log_handler) - - return logger diff --git a/valet/engine/resource_manager/compute.py b/valet/engine/resource_manager/compute.py index 5585bcc..dcd1629 100755 --- a/valet/engine/resource_manager/compute.py +++ b/valet/engine/resource_manager/compute.py @@ -184,7 +184,7 @@ class Compute(object): if hasattr(hv, 'servers'): server_list = hv.__getattr__('servers') for s in server_list: - _vm_list.append(s.uuid) + _vm_list.append(s['uuid']) except (ValueError, KeyError, TypeError): self.logger.error(traceback.format_exc()) @@ -198,7 +198,7 @@ class Compute(object): try: vm_name = server.name _vm_detail.append(vm_name) - az = server.__getattr("OS-EXT-AZ:availability_zone") + az = server.__getattr__("OS-EXT-AZ:availability_zone") _vm_detail.append(az) metadata = server.metadata _vm_detail.append(metadata) diff --git a/valet/ha/ha_valet.cfg b/valet/ha/ha_valet.cfg index 0498ecc..70e6a8a 100644 --- a/valet/ha/ha_valet.cfg +++ b/valet/ha/ha_valet.cfg @@ -49,9 +49,9 @@ priority=1 host=valet1 user=m04060 stand_by_list=valet2 -start="ssh -o ConnectTimeout=1 %s@%s 'python /usr/local/lib/python2.7/dist-packages/valet/engine/optimizer/ostro_server/ostro_daemon.py -c start'" % (user, host) +start="ssh -o ConnectTimeout=1 %s@%s 'python /usr/local/lib/python2.7/dist-packages/valet/engine/optimizer/ostro_server/ostro_daemon.py -c restart'" % (user, host) stop="ssh -o ConnectTimeout=1 %s@%s 'python /usr/local/lib/python2.7/dist-packages/valet/engine/optimizer/ostro_server/ostro_daemon.py -c stop'" % (user, host) -test="ssh -o ConnectTimeout=1 %s@%s 'python /usr/local/lib/python2.7/dist-packages/valet/engine/optimizer/ostro_server/ostro_daemon.py ; exit $?'" % (user, host) +test="ssh -o ConnectTimeout=1 %s@%s 'python /usr/local/lib/python2.7/dist-packages/valet/engine/optimizer/ostro_server/health_checker.py ; exit $?'" % (user, host) [ValetApi] diff --git a/valet/ha/ha_valet.py b/valet/ha/ha_valet.py index f15a431..0a1ec4b 100644 --- a/valet/ha/ha_valet.py +++ b/valet/ha/ha_valet.py @@ -66,7 +66,6 @@ max_log_size = 1000000 # Set the maximum number of time-series log files max_num_of_logs = 10 - PRIMARY_SETUP = 1 RETRY_COUNT = 3 # How many times to retry ping command CONNECT_TIMEOUT = 3 # Ping timeout @@ -74,7 +73,7 @@ MAX_QUICK_STARTS = 10 # we stop if there are > 10 restart in quick succession QUICK_RESTART_SEC = 150 # we consider it a quick restart if less than this # HA Configuration -HEARTBEAT_SEC = 5 # Heartbeat interval in seconds +HEARTBEAT_SEC = 10 # Heartbeat interval in seconds NAME = 'name' @@ -101,6 +100,8 @@ havalet_opts = [ cfg.StrOpt(TEST_COMMAND, help='test command') ] +# common.init_conf("havalet.log", grp2opt={api_group: havalet_opts, ostro_group: havalet_opts}) + CONF.register_group(api_group) CONF.register_opts(havalet_opts, api_group) @@ -225,7 +226,7 @@ class HaValetThread (threading.Thread): time.sleep(HEARTBEAT_SEC) else: # No valet running. Wait for higher priority valet to activate. - time.sleep(HEARTBEAT_SEC * my_priority) + time.sleep(HEARTBEAT_SEC / my_priority) self.log.info('checking status here - ' + host + ', my priority: ' + str(my_priority)) @@ -388,7 +389,7 @@ class HaValetThread (threading.Thread): try: self.log.info('activate_command: ' + activate_command) subprocess.check_call(activate_command, shell=True) - time.sleep(HEARTBEAT_SEC * priority) # allow some grace period + time.sleep(HEARTBEAT_SEC) # allow some grace period return True except subprocess.CalledProcessError as e: self.log.error(str(e)) diff --git a/valet/ha/ha_valet2.cfg b/valet/ha/ha_valet2.cfg index 9d7998e..a9de2f7 100644 --- a/valet/ha/ha_valet2.cfg +++ b/valet/ha/ha_valet2.cfg @@ -49,9 +49,9 @@ priority=2 host=valet2 user=m04060 stand_by_list=valet1 -start="ssh -o ConnectTimeout=1 %s@%s 'python /usr/local/lib/python2.7/dist-packages/valet/engine/optimizer/ostro_server/ostro_daemon.py -c start'" % (user, host) +start="ssh -o ConnectTimeout=1 %s@%s 'python /usr/local/lib/python2.7/dist-packages/valet/engine/optimizer/ostro_server/ostro_daemon.py -c restart'" % (user, host) stop="ssh -o ConnectTimeout=1 %s@%s 'python /usr/local/lib/python2.7/dist-packages/valet/engine/optimizer/ostro_server/ostro_daemon.py -c stop'" % (user, host) -test="ssh -o ConnectTimeout=1 %s@%s 'python /usr/local/lib/python2.7/dist-packages/valet/engine/optimizer/ostro_server/ostro_daemon.py ; exit $?'" % (user, host) +test="ssh -o ConnectTimeout=1 %s@%s 'python /usr/local/lib/python2.7/dist-packages/valet/engine/optimizer/ostro_server/health_checker.py ; exit $?'" % (user, host) [ValetApi] diff --git a/valet/tests/base.py b/valet/tests/base.py index 7b94b12..052a41b 100644 --- a/valet/tests/base.py +++ b/valet/tests/base.py @@ -15,15 +15,13 @@ """Base.""" +import mock from oslo_config import fixture as fixture_config -from oslo_log import log as logging from oslotest.base import BaseTestCase +from valet import api from valet.tests.functional.valet_validator.common import init -LOG = logging.getLogger(__name__) - - class Base(BaseTestCase): """Test case base class for all unit tests.""" @@ -33,6 +31,7 @@ class Base(BaseTestCase): self.CONF = self.useFixture(fixture_config.Config()).conf init.prepare(self.CONF) + api.LOG = mock.MagicMock() def setUp(self): """Setup.""" diff --git a/valet/tests/unit/engine/test_config.py b/valet/tests/unit/engine/test_config.py index 53201bd..ccf14ba 100644 --- a/valet/tests/unit/engine/test_config.py +++ b/valet/tests/unit/engine/test_config.py @@ -40,11 +40,3 @@ class TestConfig(Base): except Exception as ex: self.validate_test(isinstance(ex, cfg.ConfigFilesNotFoundError)) - - def test_config_io(self): - """Test config I/O and validate config status is success.""" - cfg.CONF.clear() - config = Config("etc/valet/valet.conf") - config_status = config.configure() - - self.validate_test(config_status == "success") diff --git a/valet/tests/unit/engine/test_ping.py b/valet/tests/unit/engine/test_ping.py new file mode 100644 index 0000000..635436f --- /dev/null +++ b/valet/tests/unit/engine/test_ping.py @@ -0,0 +1,76 @@ +import mock +import valet.engine.optimizer.ostro_server.health_checker as ping +from valet.engine.optimizer.ostro_server.health_checker import HealthCheck +from valet.tests.base import Base + +json = r'{"row 0":{"placement": "{\"status\": {\"message\": \"ping\", \"type\": \"ok\"},' \ + r'\"resources\": {\"ip\": \"localhost\", \"id\": %d}}","stack_id":"%s"}}' + + +class TestHealthCheck(Base): + + def setUp(self): + super(TestHealthCheck, self).setUp() + ping.CONF = mock.MagicMock() + ping.REST = mock.MagicMock() + self.pingger = HealthCheck() + + @mock.patch.object(HealthCheck, '_send') + @mock.patch.object(HealthCheck, '_read_response') + def test_ping(self, mock_read, mock_send): + mock_send.return_value = True + mock_read.return_value = True + + self.validate_test(self.pingger.ping(1)) + + @mock.patch.object(HealthCheck, '_send') + @mock.patch.object(HealthCheck, '_read_response') + def test_ping_unhappy(self, mock_read, mock_send): + mock_send.return_value = False + mock_read.return_value = True + + self.validate_test(not self.pingger.ping(1)) + + @mock.patch.object(HealthCheck, '_send') + @mock.patch.object(HealthCheck, '_read_response') + def test_ping_unhappy_2(self, mock_read, mock_send): + mock_send.return_value = True + mock_read.return_value = False + + self.validate_test(not self.pingger.ping(1)) + + def test_send(self): + self.pingger.rest.request.return_value.status_code = 204 + self.validate_test(self.pingger._send()) + + def test_send_unhappy(self): + self.pingger.rest.request.return_value.status_code = 200 + self.validate_test(not self.pingger._send()) + + def test_read_response(self): + id = 1 + self.pingger.rest.request.return_value.status_code = 200 + self.pingger.rest.request.return_value.text = json % (id, self.pingger.uuid) + self.validate_test(self.pingger._read_response(id)) + + def test_read_response_from_other_engine(self): + my_id = 1 + id = 2 + self.pingger.rest.request.return_value.status_code = 200 + self.pingger.rest.request.return_value.text = json % (id, self.pingger.uuid) + self.validate_test(not self.pingger._read_response(my_id)) + + def test_read_response_unhappy_wrong_res_code(self): + self.pingger.rest.request.return_value.status_code = 204 + self.pingger.rest.request.return_value.text = self.pingger.uuid + self.validate_test(not self.pingger._read_response(1)) + + def test_read_response_unhappy_wrong_body(self): + self.pingger.rest.request.return_value.status_code = 200 + self.pingger.rest.request.return_value.text = "" + self.validate_test(not self.pingger._read_response(1)) + + def test_read_response_unhappy_wrong_both(self): + self.pingger.rest.request.return_value.status_code = 204 + self.pingger.rest.request.return_value.text = "" + self.validate_test(not self.pingger._read_response(1))