diff --git a/orm/common/orm_common/model/__init__.py b/orm/common/orm_common/model/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orm/common/orm_common/model/models.py b/orm/common/orm_common/model/models.py new file mode 100755 index 00000000..85eb6042 --- /dev/null +++ b/orm/common/orm_common/model/models.py @@ -0,0 +1,42 @@ +class ResourceStatusModel(object): + def __init__(self, + timestamp, + region, + status, + resource_id, + operation, + err_msg): + self.timestamp = timestamp + self.region = region + self.status = status + self.resource_id = resource_id + self.operation = operation + self.error_msg = err_msg + + def as_dict(self): + return self.__dict__ + + +class StatusModel(object): + def __init__(self, status): + self.regions_status = status + self.status = self._get_aggregated_status() + + def _get_aggregated_status(self): + is_pending = False + for region in self.regions_status: + if (region.status == 'Error' and + region.operation.strip() != 'delete'): + # If a region had an error, the aggregated status is 'Error' + return 'Error' + elif region.status == 'Submitted': + # Just set the flag but don't return, because there might be + # an error in any of the next iterations + is_pending = True + + if is_pending: + return 'Pending' + else: + # If self.regions_status is empty, the result will still be + # 'Success' but the server returns 404 Not Found + return 'Success' diff --git a/orm/common/orm_common/sql_alchemy/__init__.py b/orm/common/orm_common/sql_alchemy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/orm/common/orm_common/sql_alchemy/db_models.py b/orm/common/orm_common/sql_alchemy/db_models.py new file mode 100644 index 00000000..16b1626a --- /dev/null +++ b/orm/common/orm_common/sql_alchemy/db_models.py @@ -0,0 +1,56 @@ +from oslo_db.sqlalchemy import models +from sqlalchemy import BigInteger, Column, ForeignKey, Integer, Text +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() + + +class CommonBaseModel(models.ModelBase): + __table_args__ = {'mysql_engine': 'InnoDB'} + + +class ImageMetadData(Base, CommonBaseModel): + __tablename__ = 'image_metadata' + + image_meta_data_id = Column(ForeignKey('resource_status.id'), + primary_key=True) + checksum = Column(Text, primary_key=False) + virtual_size = Column(Text, primary_key=False) + size = Column(Text, primary_key=False) + + def __json__(self): + return dict( + image_meta_data_id=self.image_meta_data_id, + checksum=self.checksum, + virtual_size=self.virtual_size, + size=self.size + ) + + +class ResourceStatus(Base, CommonBaseModel): + __tablename__ = 'resource_status' + + id = Column(Integer, autoincrement=True, primary_key=True) + timestamp = Column(BigInteger, primary_key=False) + region = Column(Text, primary_key=False) + status = Column(Text, primary_key=False) + transaction_id = Column(Text, primary_key=False) + resource_id = Column(Text, primary_key=False) + ord_notifier = Column(Text, primary_key=False) + err_code = Column(Text, primary_key=False) + err_msg = Column(Text, primary_key=False) + operation = Column(Text, primary_key=False) + + def __json__(self): + return dict( + id=self.id, + timestamp=self.timestamp, + region=self.region, + status=self.status, + transaction_id=self.transaction_id, + resource_id=self.resource_id, + ord_notifier=self.ord_notifier, + err_code=self.err_code, + err_msg=self.err_msg, + operation=self.operation + ) diff --git a/orm/common/orm_common/sql_alchemy/resource_status_record.py b/orm/common/orm_common/sql_alchemy/resource_status_record.py new file mode 100755 index 00000000..a1c684fb --- /dev/null +++ b/orm/common/orm_common/sql_alchemy/resource_status_record.py @@ -0,0 +1,86 @@ +import logging +from orm.common.orm_common.model.models import ResourceStatusModel, StatusModel +from orm.common.orm_common.sql_alchemy.db_models import ResourceStatus +import time + +logger = logging.getLogger(__name__) + + +class ResourceStatusRecord: + def __init__(self, session): + + self.__resource_status = ResourceStatus() + self.__TableName = "resource_status" + + if session: + self.setDBSession(session) + + def setDBSession(self, session): + self.session = session + + @property + def resource_status(self): + return self.__resource_status + + @resource_status.setter + def resource_status(self, resource_status): + self.__resource_status = resource_status + + def read_resource_status(self, resource_uuids): + records_model = {} + statuses_model = {} + + timestamp = int(time.time()) * 1000 + # assume same time period for all resource types + # max_interval_time_in_seconds = \ + # conf.region_resource_id_status.max_interval_time.default * 60 + max_interval_time_in_seconds = 2 * 60 + ref_timestamp = (int(time.time()) - + max_interval_time_in_seconds) * 1000 + + try: + records = self.session.query( + ResourceStatus.id, + ResourceStatus.resource_id, + ResourceStatus.region, + ResourceStatus.status, + ResourceStatus.timestamp, + ResourceStatus.operation, + ResourceStatus.err_msg).filter( + ResourceStatus.resource_id.in_(resource_uuids)).all() + + if records: + for id, resource_id, region, status, timestamp, operation, \ + err_msg in records: + if (status == "Submitted" and + timestamp < ref_timestamp): + timestamp = timestamp + status = "Error" + err_msg = "Status updated to 'Error'. " \ + "Too long 'Submitted' status" + + status_model = ResourceStatusModel( + timestamp, + region, + status, + resource_id, + operation, + err_msg) + + if resource_id in records_model: + records_model[resource_id].append(status_model) + else: + records_model[resource_id] = [status_model] + + for id, model in records_model.items(): + statuses_model[id] = StatusModel(model) + + else: + logger.debug("No resource status records found") + + return statuses_model + except Exception as ex: + message = "Failed to read resource status for id {}: {}".format( + resource_id, str(ex)) + logger.exception(message) + raise diff --git a/orm/common/orm_common/utils/utils.py b/orm/common/orm_common/utils/utils.py index 5115adec..fc569fca 100755 --- a/orm/common/orm_common/utils/utils.py +++ b/orm/common/orm_common/utils/utils.py @@ -1,10 +1,12 @@ import logging import pprint +import requests import time -import requests - from orm.common.client.audit.audit_client.api import audit +from orm.common.orm_common.sql_alchemy.resource_status_record \ + import ResourceStatusRecord + from pecan import conf # from cms_rest.logger import get_logger @@ -56,7 +58,8 @@ def create_or_validate_uuid(uuid, uuid_type): LOG.critical( 'CRITICAL|{}|Failed in make_uuid: connection error: {}'.format( nagios, str(exp))) - exp.message = 'connection error: Failed to get uuid: unable to connect to server' + exp.message = 'connection error: Failed to get uuid: \ + unable to connect to server' raise except Exception as e: LOG.info('Failed in make_uuid:' + str(e)) @@ -80,11 +83,14 @@ def make_transid(): try: LOG.debug('Requesting transaction ID from: {}'.format(url)) - resp = requests.post(url, data={'uuid_type': 'transaction'}, verify=conf.verify) + resp = requests.post(url, data={'uuid_type': 'transaction'}, + verify=conf.verify) except requests.exceptions.ConnectionError as exp: nagios = 'CON{}UUIDGEN001'.format(conf.server.name.upper()) - LOG.critical('CRITICAL|{}|Failed in make_transid: connection error: {}'.format(nagios, str(exp))) - exp.message = 'connection error: Failed to get uuid: unable to connect to server' + LOG.critical('CRITICAL|{}|Failed in make_transid: ' + 'connection error: {}'.format(nagios, str(exp))) + exp.message = 'connection error: Failed to get uuid: \ + unable to connect to server' raise except Exception as e: LOG.info('Failed in make_transid:' + str(e)) @@ -206,6 +212,11 @@ def get_resource_status(resource_id): return result.json() +def get_resource_status_from_db(session, resource_ids): + resource_status_record = ResourceStatusRecord(session) + return resource_status_record.read_resource_status(resource_ids) + + def get_time_human(): """this function return the timestamp for output JSON :return: timestamp in wanted format diff --git a/orm/services/customer_manager/cms_rest/data/sql_alchemy/customer_record.py b/orm/services/customer_manager/cms_rest/data/sql_alchemy/customer_record.py index 34dcf6bd..30b6be1f 100755 --- a/orm/services/customer_manager/cms_rest/data/sql_alchemy/customer_record.py +++ b/orm/services/customer_manager/cms_rest/data/sql_alchemy/customer_record.py @@ -4,6 +4,7 @@ from orm.services.customer_manager.cms_rest.data.sql_alchemy.models import (CmsU CustomerMetadata, CustomerRegion, Region, UserRole) from orm.services.customer_manager.cms_rest.logger import get_logger +from sqlalchemy import or_ LOG = get_logger(__name__) @@ -66,6 +67,19 @@ class CustomerRecord: LOG.log_exception(message, exception) raise + def read_customer_by_uuid_or_name(self, customer_uuid): + try: + customer = self.session.query(Customer).filter( + or_(Customer.uuid == customer_uuid, + Customer.name == customer_uuid)) + + return customer.first() + + except Exception as exception: + message = "Failed to get customer by uuid or name: %s " % customer_uuid + LOG.log_exception(message, exception) + raise + def get_customer_id_from_uuid(self, uuid): result = self.session.connection().scalar("SELECT id from customer WHERE uuid = \"{}\"".format(uuid)) # nosec @@ -79,6 +93,7 @@ class CustomerRecord: " FROM rds_resource_status_view WHERE resource_id IN ({})".format(uuid_str)) cust_region_dict = {} if results: + resource_status_dict = dict((id, (resource_id, region, status)) for id, resource_id, region, status in results) # using resource_status_dict, create cust_region_dict with resource_id as key and (region, status) as value for v in list(resource_status_dict.values()): diff --git a/orm/services/customer_manager/cms_rest/data/sql_alchemy/models.py b/orm/services/customer_manager/cms_rest/data/sql_alchemy/models.py index f8a54a89..70fa0f03 100755 --- a/orm/services/customer_manager/cms_rest/data/sql_alchemy/models.py +++ b/orm/services/customer_manager/cms_rest/data/sql_alchemy/models.py @@ -15,6 +15,7 @@ class CMSBaseModel(models.ModelBase): __table_args__ = {'mysql_engine': 'InnoDB'} + ''' ' CmsDomain is a DataObject and contains all the fields defined in cms_domain table record. ' defined as SqlAlchemy model map to a table @@ -438,6 +439,7 @@ class GroupsUser(Base, CMSBaseModel): user = GroupWsmeModels.User(id=id, domain=domain) return user + ''' ' CmsRole is a DataObject and contains all the fields defined in CmsRole ' table record, defined as SqlAlchemy model map to a table @@ -643,7 +645,8 @@ class CustomerRegion(Base, CMSBaseModel): customer_id=self.customer_id, region_id=self.region_id, customer_region_quotas=[quota.__json__() for quota in self.customer_region_quotas], - customer_region_user_roles=[user_role.__json__() for user_role in self.customer_region_user_roles] + customer_region_user_roles=[user_role.__json__() for user_role in self.customer_region_user_roles], + region=self.region.name ) def get_proxy_dict(self): diff --git a/orm/services/customer_manager/cms_rest/logic/customer_logic.py b/orm/services/customer_manager/cms_rest/logic/customer_logic.py index d9a6b50b..31415e11 100755 --- a/orm/services/customer_manager/cms_rest/logic/customer_logic.py +++ b/orm/services/customer_manager/cms_rest/logic/customer_logic.py @@ -1,7 +1,6 @@ import oslo_db import pecan from pecan import conf, request -import requests from orm.common.orm_common.utils import utils from orm.common.orm_common.utils.cross_api_utils import (get_regions_of_group, @@ -627,34 +626,42 @@ class CustomerLogic(object): finally: datamanager.close() + def set_resource_status(self, sql_customer, status_model): + wsme_customer = sql_customer.to_wsme() + wsme_customer.status = 'no regions' + + if status_model and status_model.regions_status: + for region in wsme_customer.regions: + for status in status_model.regions_status: + if status.region == region.name: + region.status = status.status + if status.error_msg: + region.error_message = status.error_msg + + wsme_customer.status = status_model.status + + return wsme_customer + def get_customer(self, customer): datamanager = DataManager() - - sql_customer = datamanager.get_customer_by_uuid_or_name(customer) + customer_record = datamanager.get_record('customer') + sql_customer = customer_record.read_customer_by_uuid_or_name(customer) if not sql_customer: raise ErrorStatus(404, 'customer: {0} not found'.format(customer)) - ret_customer = sql_customer.to_wsme() + # if we have regions in sql_customer if sql_customer.get_real_customer_regions(): - # if we have regions in sql_customer - - resp = requests.get(conf.api.rds_server.base + - conf.api.rds_server.status + - sql_customer.uuid, verify=conf.verify).json() - - for item in ret_customer.regions: - for status in resp['regions']: - if status['region'] == item.name: - item.status = status['status'] - if status['error_msg']: - item.error_message = status['error_msg'] - ret_customer.status = resp['status'] + uuid = [sql_customer.uuid] + resource_status_dict = utils.get_resource_status_from_db( + datamanager.get_session(), uuid) + status_model = resource_status_dict.get(sql_customer.uuid) + wsme_customer = self.set_resource_status(sql_customer, status_model) else: - ret_customer.status = 'no regions' - - return ret_customer + wsme_customer = sql_customer.to_wsme() + wsme_customer.status = 'no regions' + return wsme_customer def get_customer_list_by_criteria(self, region, user, starts_with, contains, metadata, start=0, limit=0): @@ -667,37 +674,20 @@ class CustomerLogic(object): metadata=metadata, start=start, limit=limit) + response = CustomerSummaryResponse() if sql_customers: - uuids = ','.join(str("\'" + sql_customer.uuid + "\'") - for sql_customer in sql_customers if sql_customer and sql_customer.uuid) - resource_status_dict = customer_record.get_customers_status_by_uuids(uuids) + uuids = [sql_customer.uuid for sql_customer in sql_customers] + + resource_status_dict = utils.get_resource_status_from_db( + datamanager.get_session(), uuids) for sql_customer in sql_customers: - customer = CustomerSummary.from_db_model(sql_customer) - if sql_customer.uuid: - # rds_region_list contains tuples - each containing the region associated - # with the customer along with the region status - rds_region_list = resource_status_dict.get(sql_customer.uuid) + status_model = resource_status_dict.get(sql_customer.uuid) + wsme_customer = self.set_resource_status(sql_customer, status_model) + customer_summary = CustomerSummary.from_wsme(wsme_customer) + response.customers.append(customer_summary) - if rds_region_list and customer.regions: - # set customer.status to 'error' if any of the regions has an 'Error' status' - # else, if any region status shows 'Submitted' then set customer status to 'Pending' - # otherwise customer status is 'Success' - error_status = [item for item in rds_region_list if item[1] == 'Error'] - submitted_status = [item for item in rds_region_list if item[1] == 'Submitted'] - success_status = [item for item in rds_region_list if item[1] == 'Success'] - - if len(error_status) > 0: - customer.status = 'Error' - elif len(submitted_status) > 0: - customer.status = 'Pending' - elif len(success_status) > 0: - customer.status = 'Success' - else: - customer.status = 'no regions' - - response.customers.append(customer) return response def enable(self, customer_uuid, enabled, transaction_id): @@ -745,38 +735,25 @@ class CustomerLogic(object): "delete the customer.") else: expected_status = 'Success' - invalid_status = 'N/A' - # Get status from RDS - resp = RdsProxy.get_status(sql_customer.uuid) - if resp.status_code == 200: - status_resp = resp.json() - if 'status' in list(status_resp.keys()): - LOG.debug( - 'RDS returned status: {}'.format( - status_resp['status'])) - status = status_resp['status'] - else: - # Invalid response from RDS - LOG.error('Response from RDS did not contain status') - status = invalid_status - elif resp.status_code == 404: - # Customer not found in RDS, that means it never had any regions - # So it is OK to delete it - LOG.debug( - 'Resource not found in RDS, so it is OK to delete') - status = expected_status - else: - # Invalid status code from RDS - log_message = 'Invalid response code from RDS: {}'.format( - resp.status_code) - log_message = log_message.replace('\n', '_').replace('\r', - '_') - LOG.warning(log_message) - status = invalid_status - if status == invalid_status: - raise ErrorStatus(500, "Could not get customer status") - elif status != expected_status: + # Get status from resource status table + uuid = [sql_customer.uuid] + resource_status_dict = utils.get_resource_status_from_db( + datamanager.get_session(), uuid) + status_model = resource_status_dict.get(sql_customer.uuid) + + if status_model: + status = status_model.status + LOG.debug( + 'Status from resource_status table: {}'.format(status)) + else: + # Customer not found in table, that means it never + # had any regions. So it is OK to delete it + status = expected_status + LOG.debug( + 'Resource not found in table, so it is OK to delete') + + if status != expected_status: raise ErrorStatus(409, "The customer has not been deleted " "successfully from all of its regions " diff --git a/orm/services/customer_manager/cms_rest/model/Models.py b/orm/services/customer_manager/cms_rest/model/Models.py index e9eae600..cc20bf87 100755 --- a/orm/services/customer_manager/cms_rest/model/Models.py +++ b/orm/services/customer_manager/cms_rest/model/Models.py @@ -418,6 +418,21 @@ class CustomerSummary(Model): return customer + @staticmethod + def from_wsme(wsme_customer): + regions = [region.name for region in + wsme_customer.regions] + customer = CustomerSummary() + customer.id = wsme_customer.uuid + customer.name = wsme_customer.name + customer.description = wsme_customer.description + customer.enabled = wsme_customer.enabled + customer.num_regions = len(regions) + customer.regions = regions + customer.status = wsme_customer.status + + return customer + class CustomerSummaryResponse(Model): customers = wsme.wsattr([CustomerSummary], mandatory=True) diff --git a/orm/services/flavor_manager/fms_rest/logic/flavor_logic.py b/orm/services/flavor_manager/fms_rest/logic/flavor_logic.py index b2d872e3..7e4e9c4e 100755 --- a/orm/services/flavor_manager/fms_rest/logic/flavor_logic.py +++ b/orm/services/flavor_manager/fms_rest/logic/flavor_logic.py @@ -1,7 +1,7 @@ from orm.services.flavor_manager.fms_rest.data.sql_alchemy.db_models import ( FlavorRegion, FlavorTenant) from orm.services.flavor_manager.fms_rest.data.wsme.models import ( - ExtraSpecsWrapper, Flavor, + ExtraSpecsWrapper, FlavorListFullResponse, FlavorWrapper, Region, RegionWrapper, TagsWrapper, TenantWrapper) @@ -9,6 +9,7 @@ from orm.services.flavor_manager.fms_rest.logger import get_logger from orm.services.flavor_manager.fms_rest.logic.error_base import ( ConflictError, ErrorStatus, NotFoundError) from orm.common.orm_common.injector import injector +from orm.common.orm_common.utils import utils from oslo_config import cfg import oslo_db @@ -27,15 +28,16 @@ def validate_tenants_regions_list(requested_tenants, requested_regions, valid_tenants_list, valid_regions_list = [], [] if results: - # the first element in the results tuple is the tenant - # prep result_tenant_list from result_rows and remove NoneTypes from list + # the first element in the results tuple is the tenant prep + # result_tenant_list from result_rows and remove NoneTypes from list result_tenant_list = [x[0] for x in results] result_tenant_list = filter(None, result_tenant_list) # lastly clean up valid_tenants_list list by removing duplicate items valid_tenants_list = list(dict.fromkeys(result_tenant_list)) - # second element in the results tuple is the region - compile the region - # data into valid_regions_list and eliminate duplicates from the list + # second element in the results tuple is the region - compile the + # region data into valid_regions_list and eliminate duplicates + # from the list valid_regions_list = [x[1] for x in results] valid_regions_list = list(dict.fromkeys(valid_regions_list)) @@ -170,10 +172,9 @@ def update_flavor(flavor, flavor_uuid, transaction_id): # pragma: no cover datamanager.close() -@di.dependsOn('rds_proxy') @di.dependsOn('data_manager') def delete_flavor_by_uuid(flavor_uuid): # , transaction_id): - rds_proxy, DataManager = di.resolver.unpack(delete_flavor_by_uuid) + DataManager = di.resolver.unpack(delete_flavor_by_uuid) datamanager = DataManager() try: @@ -195,35 +196,24 @@ def delete_flavor_by_uuid(flavor_uuid): # , transaction_id): raise ErrorStatus(405, msg) else: expected_status = 'Success' - invalid_status = 'N/A' - # Get status from RDS - resp = rds_proxy.get_status(sql_flavor.id) - if resp.status_code == 200: - status_resp = resp.json() - if 'status' in list(status_resp.keys()): - LOG.debug( - 'RDS returned status:{}'.format(status_resp['status'])) - status = status_resp['status'] - else: - # Invalid response from RDS - LOG.warning('Response from RDS did not contain status') - status = invalid_status - elif resp.status_code == 404: - # Flavor not found in RDS, that means it never had any regions - # So it is OK to delete it - LOG.debug('Resource not found in RDS, so it is OK to delete') - status = expected_status - else: - # Invalid status code from RDS - LOG.warning('Invalid response code from RDS: {}'.format( - resp.status_code)) - status = invalid_status - if status == invalid_status: - LOG.error('Invalid flavor status received from RDS') - raise ErrorStatus(500, - "Invalid flavor status received from RDS") - elif status != expected_status: + # Get status from resource status table + uuid = [sql_flavor.id] + resource_status_dict = utils.get_resource_status_from_db( + datamanager.get_session(), uuid) + status_model = resource_status_dict.get(sql_flavor.id) + + if status_model: + status = status_model.status + LOG.debug( + 'Status from resource_status table: {}'.format(status)) + else: + # Flavor not found in table, that means it never had any + # regions. So it is OK to delete it + status = expected_status + LOG.debug('Resource not found in table, so it is OK to delete') + + if status != expected_status: msg = "The flavor has not been deleted " \ "successfully from all of its regions " \ "(either the deletion failed on one of the " \ @@ -447,7 +437,8 @@ def delete_tenant(flavor_uuid, tenant_id, transaction_id): LOG.log_exception("FlavorLogic - Tenant not found", str(exp)) raise else: - LOG.log_exception("FlavorLogic - failed to delete tenant", str(exp)) + LOG.log_exception( + "FlavorLogic - failed to delete tenant", str(exp)) raise except Exception as exp: LOG.log_exception("FlavorLogic - Failed to delete tenant", str(exp)) @@ -558,9 +549,8 @@ def delete_extra_specs(flavor_id, transaction_id, extra_spec=None): @di.dependsOn('data_manager') -@di.dependsOn('rds_proxy') def get_tags(flavor_uuid): - DataManager, rds_proxy = di.resolver.unpack(get_flavor_by_uuid) + DataManager = di.resolver.unpack(get_tags) datamanager = DataManager() flavor_record = datamanager.get_record('flavor') @@ -760,21 +750,26 @@ def update_extra_specs(flavor_id, extra_specs, transaction_id): @di.dependsOn('data_manager') -@di.dependsOn('rds_proxy') def get_flavor_by_uuid(flavor_uuid): - DataManager, rds_proxy = di.resolver.unpack(get_flavor_by_uuid) + DataManager = di.resolver.unpack(get_flavor_by_uuid) datamanager = DataManager() - flavor_record = datamanager.get_record('flavor') + try: + flavor_record = datamanager.get_record('flavor') - sql_flavor = flavor_record.get_flavor_by_id(flavor_uuid) + sql_flavor = flavor_record.get_flavor_by_id(flavor_uuid) - if not sql_flavor: - raise ErrorStatus(404, 'flavor id {0} not found'.format(flavor_uuid)) + if not sql_flavor: + raise ErrorStatus( + 404, 'flavor id {0} not found'.format(flavor_uuid)) - flavor_wrapper = FlavorWrapper.from_db_model(sql_flavor) + flavor_wrapper = get_flavor_status( + sql_flavor, datamanager.get_session()) + + except Exception as exp: + LOG.log_exception("Failed to get_flavor_by_uuid", exp) + raise - update_region_statuses(flavor_wrapper.flavor, sql_flavor) return flavor_wrapper @@ -820,14 +815,32 @@ def add_tags(flavor_id, tags, transaction_id): return result +def get_flavor_status(sql_flavor, session): + + flavor_wrapper = FlavorWrapper.from_db_model(sql_flavor) + region_names = [reg.region_name for reg in sql_flavor.flavor_regions] + + if len(region_names): + uuid = [sql_flavor.id] + resource_status_dict = utils.get_resource_status_from_db( + session, uuid) + + status_model = resource_status_dict.get(sql_flavor.id) + update_region_statuses( + flavor_wrapper.flavor, region_names, status_model) + else: + # flavor has no regions + flavor_wrapper.flavor.status = "no regions" + + return flavor_wrapper + + @di.dependsOn('data_manager') -@di.dependsOn('rds_proxy') def get_flavor_by_uuid_or_name(flavor_uuid_or_name): - DataManager, rds_proxy = di.resolver.unpack(get_flavor_by_uuid) + DataManager = di.resolver.unpack(get_flavor_by_uuid_or_name) datamanager = DataManager() try: - flavor_record = datamanager.get_record('flavor') sql_flavor = flavor_record.get_flavor_by_id_or_name( @@ -838,9 +851,9 @@ def get_flavor_by_uuid_or_name(flavor_uuid_or_name): 404, 'flavor id or name {0} not found'.format(flavor_uuid_or_name)) - flavor_wrapper = FlavorWrapper.from_db_model(sql_flavor) + flavor_wrapper = get_flavor_status( + sql_flavor, datamanager.get_session()) - update_region_statuses(flavor_wrapper.flavor, sql_flavor) except Exception as exp: LOG.log_exception("Failed to get_flavor_by_uuid_or_name", exp) raise @@ -850,56 +863,37 @@ def get_flavor_by_uuid_or_name(flavor_uuid_or_name): return flavor_wrapper -@di.dependsOn('rds_proxy') -def update_region_statuses(flavor, sql_flavor): - rds_proxy = di.resolver.unpack(update_region_statuses) +def update_region_statuses(flavor, region_names, status_model): # remove the regions comes from database and return the regions which # return from rds, because there might be group region there (in the db) # and in the response from the rds we will have a list of all regions # belong to this group flavor.regions[:] = [] - resp = rds_proxy.get_status(sql_flavor.id) - if resp.status_code == 200: - rds_status_resp = resp.json() - # store all regions for the flavor in the flavor_regions_list - flavor_regions_list = [] - for region_data in sql_flavor.flavor_regions: - flavor_regions_list.append(region_data.region_name) + flavor.status = 'no regions' + error_status = submitted_status = success_status = 0 + + if status_model and status_model.regions_status: # get region status if region in flavor_regions_list - if flavor_regions_list and 'regions' in list(rds_status_resp.keys()): - for rds_region_status in rds_status_resp['regions']: + if region_names: + for status in status_model.regions_status: # check that the region read from RDS is in the # flavor_regions_list - if rds_region_status['region'] in flavor_regions_list: + if status.region in region_names: flavor.regions.append( - Region(name=rds_region_status['region'], type="single", - status=rds_region_status['status'], - error_message=rds_region_status['error_msg'])) + Region(name=status.region, type="single", + status=status.status, + error_message=status.error_msg)) - if 'status' in list(rds_status_resp.keys()): - # if flavor not assigned to region set flavor.status to - # 'no regions' - if flavor_regions_list: - flavor.status = rds_status_resp['status'] - else: - flavor.status = 'no regions' - - elif resp.status_code == 404: - # flavor id not in rds resource_status table as no region is assigned - flavor.status = "no regions" - else: - flavor.status = "N/A" + flavor.status = status_model.status @di.dependsOn('data_manager') -@di.dependsOn('rds_proxy') def get_flavor_list_by_params(visibility, region, tenant, series, vm_type, vnf_name, starts_with, contains, alias): - DataManager, rds_proxy = di.resolver.unpack(get_flavor_list_by_params) + DataManager = di.resolver.unpack(get_flavor_list_by_params) datamanager = DataManager() - try: flavor_record = datamanager.get_record('flavor') sql_flavors = flavor_record.get_flavors_by_criteria( @@ -914,53 +908,23 @@ def get_flavor_list_by_params(visibility, region, tenant, series, vm_type, alias=alias) response = FlavorListFullResponse() - if sql_flavors: - uuids = ','.join(str("\'" + sql_flavor.id + "\'") - for sql_flavor in sql_flavors - if sql_flavor and sql_flavor.id) - resource_status_dict = flavor_record.get_flavors_status_by_uuids( - uuids) + if sql_flavors: + uuids = [sql_flavor.id for sql_flavor in sql_flavors] + + resource_status_dict = utils.get_resource_status_from_db( + datamanager.get_session(), uuids) for sql_flavor in sql_flavors: - flavor = Flavor.from_db_model(sql_flavor) - if sql_flavor.id: - # rds_region_list contains tuples - each containing the - # region associated with the flavor along with the region - # status - rds_region_list = resource_status_dict.get(sql_flavor.id) + flavor_wrapper = FlavorWrapper.from_db_model(sql_flavor) + region_names = [reg.region_name for reg + in sql_flavor.flavor_regions] - # determine flavor overall status by checking its region - # statuses: - if rds_region_list and flavor.regions: - # set image.status to 'error' if any of the regions has - # an 'Error' status' else, if any region status shows - # 'Submitted' then set image status to 'Pending' - # otherwise image status = 'Success' - error_status = [item for item in rds_region_list - if item[1] == 'Error'] - submitted_status = [item for item in rds_region_list - if item[1] == 'Submitted'] - success_status = [item for item in rds_region_list - if item[1] == 'Success'] + status_model = resource_status_dict.get(sql_flavor.id) + update_region_statuses( + flavor_wrapper.flavor, region_names, status_model) - if len(error_status) > 0: - flavor.status = 'Error' - elif len(submitted_status) > 0: - flavor.status = 'Pending' - elif len(success_status) > 0: - flavor.status = 'Success' - - # use rds_region_list to format the regions' statuses - # in flavor record - for rgn in flavor.regions: - for rds_row_items in rds_region_list: - if rgn.name == rds_row_items[0]: - rgn.status = rds_row_items[1] - else: - flavor.status = 'no regions' - - response.flavors.append(flavor) + response.flavors.append(flavor_wrapper.flavor) except Exception as exp: LOG.log_exception("Fail to get_flavor_list_by_params", exp) diff --git a/orm/tests/unit/cms/test_customer_logic.py b/orm/tests/unit/cms/test_customer_logic.py index 4e889711..eb62e200 100755 --- a/orm/tests/unit/cms/test_customer_logic.py +++ b/orm/tests/unit/cms/test_customer_logic.py @@ -33,6 +33,19 @@ class RdsStatus(object): return {"status": self.status} +class ResourceStatusModel(object): + def __init__(self, region='DPK', status='Success', err_msg=None): + self.region = region + self.status = status + self.error_msg = err_msg + + +class StatusModel(object): + def __init__(self, status='Success'): + self.regions_status = [ResourceStatusModel()] + self.status = status + + class TestCustomerLogic(FunctionalTest): def setUp(self): global customer @@ -46,6 +59,8 @@ class TestCustomerLogic(FunctionalTest): customer_logic.utils.audit_trail.return_value = None customer_logic.utils.make_uuid.return_value = 'some_uuid' customer_logic.utils.get_time_human.return_value = '1337' + customer_logic.utils.get_resource_status_from_db.return_value = \ + {'1337': StatusModel()} customer_logic.RdsProxy = mock.MagicMock() customer_logic.RdsProxy.send_customer.return_value = None @@ -384,12 +399,13 @@ class TestCustomerLogic(FunctionalTest): {"key:value"}) def test_delete_customer_by_uuid_success(self): - logic = customer_logic.CustomerLogic() - logic.delete_customer_by_uuid('customer_id') + global flow_type + flow_type = 3 - # Customer found in CMS DB but not found in RDS - customer_logic.RdsProxy.get_status.return_value = RdsStatus( - status_code=404) + customer_logic.utils.get_resource_status_from_db.return_value = \ + {'1337': StatusModel()} + + logic = customer_logic.CustomerLogic() logic.delete_customer_by_uuid('customer_id') def test_delete_customer_by_uuid_not_found(self): @@ -403,31 +419,23 @@ class TestCustomerLogic(FunctionalTest): logic.delete_customer_by_uuid('customer_id') def test_delete_customer_by_uuid_errors(self): + global flow_type + flow_type = 3 + global mock_returns_error mock_returns_error = True logic = customer_logic.CustomerLogic() self.assertRaises(SystemError, logic.delete_customer_by_uuid, 'customer_id') - # RDS returned an empty json mock_returns_error = False - customer_logic.RdsProxy.get_status.return_value = RdsStatus(oy=True) + customer_logic.utils.get_resource_status_from_db.return_value = \ + {'1337': StatusModel(status='Error')} + self.assertRaises(customer_logic.ErrorStatus, logic.delete_customer_by_uuid, 'customer_id') - # RDS returned 500 - customer_logic.RdsProxy.get_status.return_value = RdsStatus( - status_code=500) - self.assertRaises(customer_logic.ErrorStatus, - logic.delete_customer_by_uuid, - 'customer_id') - - # RDS returned Error status - customer_logic.RdsProxy.get_status.return_value = RdsStatus( - status='Error') - self.assertRaises(customer_logic.ErrorStatus, - logic.delete_customer_by_uuid, - 'customer_id') + self.assertTrue(record_mock.read_customer_by_uuid.called) def test_delete_customer_by_uuid_conflict(self): global flow_type @@ -454,20 +462,14 @@ class TestCustomerLogic(FunctionalTest): def test_get_customer_success(self): logic = customer_logic.CustomerLogic() - get_mock = mock.MagicMock() - get_mock.json.return_value = STATUS_JSON - customer_logic.requests.get = mock.MagicMock(return_value=get_mock) logic.get_customer('customer_id') - - self.assertTrue(data_manager_mock.get_customer_by_uuid_or_name.called) + self.assertTrue(record_mock.read_customer_by_uuid_or_name.called) def test_get_customer_not_found(self): global flow_type flow_type = 1 - # customer_logic.requests.get = mock.MagicMock(return_value=None) logic = customer_logic.CustomerLogic() - self.assertRaises(ErrorStatus, logic.get_customer, 'id') @@ -507,7 +509,14 @@ def get_mock_datamanager(): sql_customer.uuid = '1337' sql_customer.status = 'Success' sql_customer.name = 'DPK' + return sql_customer + def _get_customer_no_regions(): + sql_customer = sql_models.Customer() + sql_customer.customer_customer_regions = [] + sql_customer.uuid = '1337' + sql_customer.status = 'Success' + sql_customer.name = 'DPK' return sql_customer def _add_customer(*args, **kwargs): @@ -547,17 +556,20 @@ def get_mock_datamanager(): data_manager_mock.update_customer = _update_customer data_manager_mock.add_region = _add_region data_manager_mock.add_user.return_value = sql_models.CmsUser() - data_manager_mock.get_customer_by_uuid_or_name.return_value = _get_customer() record_mock.delete_region_for_customer.return_value = None record_mock.delete_customer_by_uuid.return_value = None + record_mock.read_customer_by_uuid_or_name.return_value = _get_customer() + if flow_type == 1: record_mock.read_customer_by_uuid.return_value = None - data_manager_mock.get_customer_by_uuid_or_name.return_value = None + record_mock.read_customer_by_uuid_or_name.return_value = None elif flow_type == 2: q = mock.MagicMock() q.get_real_customer_regions.return_value = [mock.MagicMock()] record_mock.read_customer_by_uuid.return_value = q + elif flow_type == 3: + record_mock.read_customer_by_uuid.return_value = _get_customer_no_regions() record_mock.delete_user_from_region.return_value = resultobj(rowcount) else: diff --git a/orm/tests/unit/fms/test_flavor_logic.py b/orm/tests/unit/fms/test_flavor_logic.py index 0ef60802..5ebc7d0c 100755 --- a/orm/tests/unit/fms/test_flavor_logic.py +++ b/orm/tests/unit/fms/test_flavor_logic.py @@ -1,4 +1,5 @@ from orm.common.orm_common.injector import injector +from orm.common.orm_common.utils import utils from orm.services.flavor_manager.fms_rest.data.sql_alchemy import db_models from orm.services.flavor_manager.fms_rest.data.wsme import models from orm.services.flavor_manager.fms_rest.data.wsme.models import * @@ -20,6 +21,7 @@ class OES(): def to_db_model(self): return None + error = None FLAVOR_MOCK = None @@ -343,45 +345,44 @@ class TestFlavorLogic(FunctionalTest): self.assertRaises(flavor_logic.ErrorStatus, flavor_logic.delete_tags, 'a', None, 'a') - def test_delete_flavor_by_uuid_success(self): + @patch.object(utils, 'get_resource_status_from_db') + def test_delete_flavor_by_uuid_success(self, mock_get_resource): global error - error = 31 - injector.override_injected_dependency( - ('data_manager', get_datamanager_mock)) - injector.override_injected_dependency( - ('rds_proxy', get_rds_proxy_mock())) + error = 10 - flavor_logic.delete_flavor_by_uuid('some_id') - - error = 33 - injector.override_injected_dependency( - ('rds_proxy', get_rds_proxy_mock())) - flavor_logic.delete_flavor_by_uuid('some_id') - - def test_delete_flavor_by_uuid_bad_status(self): - global error injector.override_injected_dependency( ('data_manager', get_datamanager_mock)) - # Run once for response with no status and once for an invalid - # response code - for error_value in (32, 40,): - error = error_value - injector.override_injected_dependency( - ('rds_proxy', get_rds_proxy_mock())) + status_model = MagicMock() - try: - flavor_logic.delete_flavor_by_uuid('some_id') - self.fail('ErrorStatus not raised!') - except flavor_logic.ErrorStatus as e: - self.assertEqual(e.status_code, 500) + # status model has success status + status_model.status = 'Success' + resource_status_dict = {'some_id': status_model} + mock_get_resource.return_value = resource_status_dict + flavor_logic.delete_flavor_by_uuid('some_uuid') + + # status model not found + resource_status_dict = {} + mock_get_resource.return_value = resource_status_dict + flavor_logic.delete_flavor_by_uuid('some_id') + + @patch.object(utils, 'get_resource_status_from_db') + def test_delete_flavor_by_uuid_bad_status(self, mock_get_resource): + global error + error = 10 - # RDS returned OK, but the resource status is Error - error = 34 injector.override_injected_dependency( - ('rds_proxy', get_rds_proxy_mock())) + ('data_manager', get_datamanager_mock)) + + status_model = MagicMock() + + # status model has error status + status_model.status = 'Error' + resource_status_dict = {'some_id': status_model} + mock_get_resource.return_value = resource_status_dict + try: - flavor_logic.delete_flavor_by_uuid('some_id') + flavor_logic.delete_flavor_by_uuid('some_uuid') self.fail('ErrorStatus not raised!') except flavor_logic.ErrorStatus as e: self.assertEqual(e.status_code, 409) @@ -691,6 +692,9 @@ def get_datamanager_mock(): elif error == 9: record.get_flavor_by_id.side_effect = flavor_logic.ErrorStatus( 500) + elif error == 10: + record.get_flavor_by_id.return_value = db_models.Flavor( + id='some_id') else: record.get_flavor_by_id.return_value = MagicMock() return record