From 2cba5155c7ae9ec48708c18232668fa284113d46 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Fri, 21 Feb 2020 16:20:10 -0800 Subject: [PATCH] Removed RDS service call (cont'd) Change-Id: I0c5bebd53faca231f58b0280ef1423ef0d5d50fb --- orm/base_config.py | 3 + orm/common/orm_common/model/models.py | 35 ++- .../orm_common/sql_alchemy/db_models.py | 20 +- .../sql_alchemy/resource_status_record.py | 39 ++-- .../data/sql_alchemy/customer_record.py | 16 -- .../cms_rest/data/sql_alchemy/group_record.py | 2 +- .../cms_rest/logic/customer_logic.py | 4 +- .../db_scripts/ranger_cms_create_db.sql | 5 - .../data/sql_alchemy/flavor/flavor_record.py | 18 -- .../fms_rest/logic/flavor_logic.py | 4 +- .../db_scripts/ranger_fms_create_db.sql | 11 - .../image_manager/ims/logic/image_logic.py | 204 +++++++++--------- .../sql_alchemy/image/image_record.py | 15 -- .../ims/persistency/wsme/models.py | 13 ++ .../scripts/db_scripts/create_db.sql | 10 - .../model/region_resource_id_status.py | 69 ------ .../mysql/region_resource_id_status.py | 3 +- orm/tests/unit/cms/test_customer_logic.py | 2 +- orm/tests/unit/ims/logic/test_image_logic.py | 118 +++++----- .../v1/status/test_get_resource_status.py | 3 +- .../model/test_region_resource_id_status.py | 23 +- .../unit/rds/services/test_create_resource.py | 6 +- 22 files changed, 245 insertions(+), 378 deletions(-) diff --git a/orm/base_config.py b/orm/base_config.py index 11474ffa..70ecd750 100644 --- a/orm/base_config.py +++ b/orm/base_config.py @@ -44,6 +44,9 @@ OrmOpts = [ cfg.StrOpt('ranger_agent_client_cert_path', default='', help='Ranger Agent certificate path.'), + cfg.IntOpt('resource_status_max_interval', + default=2, + help='Interval in seconds.'), cfg.StrOpt('log_location', default='/var/log/ranger', help='Orm log directory.'), diff --git a/orm/common/orm_common/model/models.py b/orm/common/orm_common/model/models.py index 85eb6042..5ca97054 100755 --- a/orm/common/orm_common/model/models.py +++ b/orm/common/orm_common/model/models.py @@ -1,17 +1,42 @@ +class ResourceMetaData(object): + def __init__(self, checksum, virtual_size, size): + self.size = size + self.virtual_size = virtual_size + self.checksum = checksum + + def as_dict(self): + return self.__dict__ + + class ResourceStatusModel(object): def __init__(self, timestamp, region, status, + transaction_id, resource_id, + ord_notifier, + err_msg, + err_code, operation, - err_msg): + resource_extra_metadata=None): self.timestamp = timestamp self.region = region self.status = status + self.ord_transaction_id = transaction_id self.resource_id = resource_id - self.operation = operation + self.ord_notifier_id = ord_notifier self.error_msg = err_msg + self.error_code = err_code + self.operation = operation + + if resource_extra_metadata: + self.resource_extra_metadata = ResourceMetaData( + checksum=resource_extra_metadata[0].checksum, + virtual_size=resource_extra_metadata[0].virtual_size, + size=resource_extra_metadata[0].size) + else: + self.resource_extra_metadata = None def as_dict(self): return self.__dict__ @@ -19,12 +44,12 @@ class ResourceStatusModel(object): class StatusModel(object): def __init__(self, status): - self.regions_status = status + self.regions = status self.status = self._get_aggregated_status() def _get_aggregated_status(self): is_pending = False - for region in self.regions_status: + for region in self.regions: if (region.status == 'Error' and region.operation.strip() != 'delete'): # If a region had an error, the aggregated status is 'Error' @@ -37,6 +62,6 @@ class StatusModel(object): if is_pending: return 'Pending' else: - # If self.regions_status is empty, the result will still be + # If self.regions is empty, the result will still be # 'Success' but the server returns 404 Not Found return 'Success' diff --git a/orm/common/orm_common/sql_alchemy/db_models.py b/orm/common/orm_common/sql_alchemy/db_models.py index 16b1626a..b2a6d89b 100644 --- a/orm/common/orm_common/sql_alchemy/db_models.py +++ b/orm/common/orm_common/sql_alchemy/db_models.py @@ -1,5 +1,5 @@ from oslo_db.sqlalchemy import models -from sqlalchemy import BigInteger, Column, ForeignKey, Integer, Text +from sqlalchemy import BigInteger, Column, Integer, Text from sqlalchemy.ext.declarative import declarative_base Base = declarative_base() @@ -9,24 +9,6 @@ class CommonBaseModel(models.ModelBase): __table_args__ = {'mysql_engine': 'InnoDB'} -class ImageMetadData(Base, CommonBaseModel): - __tablename__ = 'image_metadata' - - image_meta_data_id = Column(ForeignKey('resource_status.id'), - primary_key=True) - checksum = Column(Text, primary_key=False) - virtual_size = Column(Text, primary_key=False) - size = Column(Text, primary_key=False) - - def __json__(self): - return dict( - image_meta_data_id=self.image_meta_data_id, - checksum=self.checksum, - virtual_size=self.virtual_size, - size=self.size - ) - - class ResourceStatus(Base, CommonBaseModel): __tablename__ = 'resource_status' diff --git a/orm/common/orm_common/sql_alchemy/resource_status_record.py b/orm/common/orm_common/sql_alchemy/resource_status_record.py index a1c684fb..d8a45677 100755 --- a/orm/common/orm_common/sql_alchemy/resource_status_record.py +++ b/orm/common/orm_common/sql_alchemy/resource_status_record.py @@ -3,6 +3,8 @@ from orm.common.orm_common.model.models import ResourceStatusModel, StatusModel from orm.common.orm_common.sql_alchemy.db_models import ResourceStatus import time +from oslo_config import cfg + logger = logging.getLogger(__name__) @@ -32,26 +34,26 @@ class ResourceStatusRecord: timestamp = int(time.time()) * 1000 # assume same time period for all resource types - # max_interval_time_in_seconds = \ - # conf.region_resource_id_status.max_interval_time.default * 60 - max_interval_time_in_seconds = 2 * 60 - ref_timestamp = (int(time.time()) - - max_interval_time_in_seconds) * 1000 + max_interval_in_seconds = cfg.CONF.resource_status_max_interval * 60 + ref_timestamp = (int(time.time()) - max_interval_in_seconds) * 1000 try: records = self.session.query( ResourceStatus.id, - ResourceStatus.resource_id, + ResourceStatus.timestamp, ResourceStatus.region, ResourceStatus.status, - ResourceStatus.timestamp, - ResourceStatus.operation, - ResourceStatus.err_msg).filter( + ResourceStatus.transaction_id, + ResourceStatus.resource_id, + ResourceStatus.ord_notifier, + ResourceStatus.err_msg, + ResourceStatus.err_code, + ResourceStatus.operation).filter( ResourceStatus.resource_id.in_(resource_uuids)).all() if records: - for id, resource_id, region, status, timestamp, operation, \ - err_msg in records: + for id, timestamp, region, status, transaction, resource, \ + notifier, err_msg, err_code, operation in records: if (status == "Submitted" and timestamp < ref_timestamp): timestamp = timestamp @@ -63,14 +65,17 @@ class ResourceStatusRecord: timestamp, region, status, - resource_id, - operation, - err_msg) + transaction, + resource, + notifier, + err_msg, + err_code, + operation) - if resource_id in records_model: - records_model[resource_id].append(status_model) + if resource in records_model: + records_model[resource].append(status_model) else: - records_model[resource_id] = [status_model] + records_model[resource] = [status_model] for id, model in records_model.items(): statuses_model[id] = StatusModel(model) diff --git a/orm/services/customer_manager/cms_rest/data/sql_alchemy/customer_record.py b/orm/services/customer_manager/cms_rest/data/sql_alchemy/customer_record.py index 30b6be1f..34cb2c46 100755 --- a/orm/services/customer_manager/cms_rest/data/sql_alchemy/customer_record.py +++ b/orm/services/customer_manager/cms_rest/data/sql_alchemy/customer_record.py @@ -88,22 +88,6 @@ class CustomerRecord: else: return None - def get_customers_status_by_uuids(self, uuid_str): - results = self.session.connection().execute("SELECT id, resource_id, region, status" # nosec - " FROM rds_resource_status_view WHERE resource_id IN ({})".format(uuid_str)) - cust_region_dict = {} - if results: - - resource_status_dict = dict((id, (resource_id, region, status)) for id, resource_id, region, status in results) - # using resource_status_dict, create cust_region_dict with resource_id as key and (region, status) as value - for v in list(resource_status_dict.values()): - if v[0] in cust_region_dict: - cust_region_dict[v[0]].append(v[1:]) - else: - cust_region_dict[v[0]] = [v[1:]] - results.close() - return cust_region_dict - def delete_customer_by_uuid(self, uuid): try: result = self.session.query(Customer).filter( diff --git a/orm/services/customer_manager/cms_rest/data/sql_alchemy/group_record.py b/orm/services/customer_manager/cms_rest/data/sql_alchemy/group_record.py index 41de1710..76e57afd 100755 --- a/orm/services/customer_manager/cms_rest/data/sql_alchemy/group_record.py +++ b/orm/services/customer_manager/cms_rest/data/sql_alchemy/group_record.py @@ -79,7 +79,7 @@ class GroupRecord: def get_groups_status_by_uuids(self, uuid_str): cmd = "SELECT id, resource_id, region, status FROM " \ - "rds_resource_status_view WHERE resource_id IN (%s)" + "resource_status WHERE resource_id IN (%s)" results = self.session.connection().execute(cmd % uuid_str) group_region = {} diff --git a/orm/services/customer_manager/cms_rest/logic/customer_logic.py b/orm/services/customer_manager/cms_rest/logic/customer_logic.py index 31415e11..3fa8ed17 100755 --- a/orm/services/customer_manager/cms_rest/logic/customer_logic.py +++ b/orm/services/customer_manager/cms_rest/logic/customer_logic.py @@ -630,9 +630,9 @@ class CustomerLogic(object): wsme_customer = sql_customer.to_wsme() wsme_customer.status = 'no regions' - if status_model and status_model.regions_status: + if status_model and status_model.regions: for region in wsme_customer.regions: - for status in status_model.regions_status: + for status in status_model.regions: if status.region == region.name: region.status = status.status if status.error_msg: diff --git a/orm/services/customer_manager/scripts/db_scripts/ranger_cms_create_db.sql b/orm/services/customer_manager/scripts/db_scripts/ranger_cms_create_db.sql index b94e7046..b64438d3 100755 --- a/orm/services/customer_manager/scripts/db_scripts/ranger_cms_create_db.sql +++ b/orm/services/customer_manager/scripts/db_scripts/ranger_cms_create_db.sql @@ -90,11 +90,6 @@ create table if not exists user_role index region_id (region_id), index user_id (user_id)); -create or replace view rds_resource_status_view AS - ( - SELECT id, resource_id, region, status, - err_code, operation from resource_status); - create table if not exists cms_domain ( id integer auto_increment not null, diff --git a/orm/services/flavor_manager/fms_rest/data/sql_alchemy/flavor/flavor_record.py b/orm/services/flavor_manager/fms_rest/data/sql_alchemy/flavor/flavor_record.py index 7cb1c6f5..db03ab9b 100755 --- a/orm/services/flavor_manager/fms_rest/data/sql_alchemy/flavor/flavor_record.py +++ b/orm/services/flavor_manager/fms_rest/data/sql_alchemy/flavor/flavor_record.py @@ -147,24 +147,6 @@ class FlavorRecord: LOG.log_exception(message, exception) raise - def get_flavors_status_by_uuids(self, uuid_str): - results = self.session.connection().execute("SELECT id, resource_id, region, status" # nosec - " FROM rds_resource_status_view WHERE resource_id IN ({})".format(uuid_str)) - - flvr_region_dict = {} - - if results: - resource_status_dict = dict((id, (resource_id, region, status)) for id, resource_id, region, status in results) - # using resource_status_dict, create flvr_region_dict with resource_id as key and (region, status) as value - for v in list(resource_status_dict.values()): - if v[0] in flvr_region_dict: - flvr_region_dict[v[0]].append(v[1:]) - else: - flvr_region_dict[v[0]] = [v[1:]] - - results.close() - return flvr_region_dict - def get_flavors_by_criteria(self, **criteria): try: diff --git a/orm/services/flavor_manager/fms_rest/logic/flavor_logic.py b/orm/services/flavor_manager/fms_rest/logic/flavor_logic.py index 7e4e9c4e..da4c6768 100755 --- a/orm/services/flavor_manager/fms_rest/logic/flavor_logic.py +++ b/orm/services/flavor_manager/fms_rest/logic/flavor_logic.py @@ -873,10 +873,10 @@ def update_region_statuses(flavor, region_names, status_model): flavor.status = 'no regions' error_status = submitted_status = success_status = 0 - if status_model and status_model.regions_status: + if status_model and status_model.regions: # get region status if region in flavor_regions_list if region_names: - for status in status_model.regions_status: + for status in status_model.regions: # check that the region read from RDS is in the # flavor_regions_list if status.region in region_names: diff --git a/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql b/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql index fe0dab1c..932504e1 100755 --- a/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql +++ b/orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql @@ -99,14 +99,3 @@ create table if not exists flavor_option foreign key (flavor_internal_id) references flavor(internal_id) ON DELETE CASCADE ON UPDATE NO ACTION, primary key (flavor_internal_id,key_name) ); -# - - -#***** -#* MySql script for Creating View rds_resource_status_view -#***** - -create or replace view rds_resource_status_view AS - ( - SELECT ID, RESOURCE_ID, REGION,STATUS, - ERR_CODE,OPERATION from resource_status); diff --git a/orm/services/image_manager/ims/logic/image_logic.py b/orm/services/image_manager/ims/logic/image_logic.py index bbd56ee3..9be47b0c 100755 --- a/orm/services/image_manager/ims/logic/image_logic.py +++ b/orm/services/image_manager/ims/logic/image_logic.py @@ -1,11 +1,14 @@ import time from orm.common.orm_common.injector import injector +from orm.common.orm_common.utils import utils from orm.services.image_manager.ims.logger import get_logger -from orm.services.image_manager.ims.logic.error_base import ErrorStatus, NotFoundError -from orm.services.image_manager.ims.persistency.sql_alchemy.db_models import ImageCustomer, ImageRegion -from orm.services.image_manager.ims.persistency.wsme.models import (ImageSummary, ImageSummaryResponse, - ImageWrapper, RegionWrapper) +from orm.services.image_manager.ims.logic.error_base import \ + ErrorStatus, NotFoundError +from orm.services.image_manager.ims.persistency.sql_alchemy.db_models import \ + ImageCustomer, ImageRegion +from orm.services.image_manager.ims.persistency.wsme.models import ( + ImageSummary, ImageSummaryResponse, ImageWrapper, RegionWrapper) from orm.services.image_manager.ims.utils import utils as ImsUtils import oslo_db @@ -63,13 +66,13 @@ def send_to_rds_if_needed(sql_image, existing_region_names, http_action, image_dict = sql_image.get_proxy_dict() update_region_actions(image_dict, existing_region_names, http_action) if image_dict['regions'] or len(existing_region_names) > 0: - LOG.debug("Image is valid to send to RDS - sending to RDS Proxy ") + LOG.debug("Image is valid, sending to RDS Proxy ") rds_proxy.send_image(image_dict, transaction_id, http_action) else: - LOG.debug("Group with no regions - wasn't send to RDS Proxy " + str( + LOG.debug("Group with no regions, not sending to RDS Proxy " + str( sql_image)) else: - LOG.debug("Image with no regions - wasn't send to RDS Proxy " + str( + LOG.debug("Image with no regions, not sending to RDS Proxy " + str( sql_image)) @@ -87,9 +90,10 @@ def update_image(image_wrapper, image_uuid, transaction_id, http_action="put"): sql_image = image_rec.get_image_by_id(image_uuid) if sql_image is None: - raise NotFoundError(status_code=404, - message="Image {0} does not exist for update".format( - image_uuid)) + raise NotFoundError( + status_code=404, + message="Image {0} does not exist for update".format( + image_uuid)) image_wrapper.validate_update(sql_image, new_image) @@ -120,51 +124,48 @@ def update_image(image_wrapper, image_uuid, transaction_id, http_action="put"): raise -@di.dependsOn('rds_proxy') @di.dependsOn('data_manager') def delete_image_by_uuid(image_uuid, transaction_id): - rds_proxy, DataManager = di.resolver.unpack(delete_image_by_uuid) + DataManager = di.resolver.unpack(delete_image_by_uuid) datamanager = DataManager() try: - datamanager.begin_transaction() image_rec = datamanager.get_record('image') sql_image = image_rec.get_image_by_id(image_uuid) if sql_image is None: - raise NotFoundError(message="Image '{}' not found".format(image_uuid)) + raise NotFoundError( + message="Image '{}' not found".format(image_uuid)) image_existing_region_names = sql_image.get_existing_region_names() if len(image_existing_region_names) > 0: - # Do not delete a flavor that still has some regions + # Do not delete an image that still has region(s) raise ErrorStatus(405, "Cannot delete a image with regions. " "Please delete the regions first and then " "delete the image. ") - # Get status from RDS - image_status = rds_proxy.get_status(sql_image.id, False) - - status_resp = None - - if image_status.status_code == 200: - status_resp = image_status.json()['status'] - LOG.debug('RDS returned status: {}'.format(status_resp)) - - elif image_status.status_code == 404: - status_resp = 'Success' + # Get status from resource status table + uuid = [sql_image.id] + resource_status_dict = utils.get_resource_status_from_db( + datamanager.get_session(), uuid) + status_model = resource_status_dict.get(sql_image.id) + if status_model: + status = status_model.status + LOG.debug('Status from resource_status table: {}'.format(status)) else: - # fail to get status from rds - raise ErrorStatus(500, "fail to get status for this resource " - "deleting image not allowed ") + # Image not found in table, that means it never had any + # regions. So it is OK to delete it + status = 'Success' + LOG.debug('Resource not found in table, so it is OK to delete') - if status_resp != 'Success': + if status != 'Success': raise ErrorStatus(405, "not allowed as aggregate status " - "have to be Success (either the deletion" - " failed on one of the regions or it is " - "still in progress)") + "have to be Success (either the deletion " + "failed on one of the regions or it is " + "still in progress)") image_rec.delete_image_by_id(image_uuid) datamanager.flush() # i want to get any exception created by this @@ -193,7 +194,8 @@ def add_regions(image_uuid, regions, transaction_id): existing_region_names = sql_image.get_existing_region_names() for region in regions.regions: - db_region = ImageRegion(region_name=region.name, region_type=region.type) + db_region = ImageRegion(region_name=region.name, + region_type=region.type) sql_image.add_region(db_region) datamanager.flush() # i want to get any exception created by @@ -236,7 +238,8 @@ def replace_regions(image_uuid, regions, transaction_id): datamanager.flush() for region in regions.regions: - db_region = ImageRegion(region_name=region.name, region_type=region.type) + db_region = ImageRegion(region_name=region.name, + region_type=region.type) sql_image.add_region(db_region) datamanager.flush() # i want to get any exception created by # previous actions against the database @@ -335,7 +338,8 @@ def add_customers(image_uuid, customers, transaction_id): return ret_image except Exception as exp: - if 'conflicts with persistent instance' in str(exp) or 'Duplicate entry' in str(exp): + if 'conflicts with persistent instance' in str(exp) or \ + 'Duplicate entry' in str(exp): raise ErrorStatus(409, "Duplicate Customer for Image") LOG.log_exception("ImageLogic - Failed to add Customers", exp) datamanager.rollback() @@ -373,7 +377,8 @@ def replace_customers(image_uuid, customers, transaction_id): return ret_image except Exception as exp: - if 'conflicts with persistent instance' in str(exp) or 'Duplicate entry' in str(exp): + if 'conflicts with persistent instance' in str(exp) or \ + 'Duplicate entry' in str(exp): raise ErrorStatus(409, "Duplicate Customer for Image") LOG.log_exception("ImageLogic - Failed to add Customers", exp) datamanager.rollback() @@ -392,7 +397,8 @@ def delete_customer(image_uuid, customer_id, transaction_id): raise ErrorStatus(404, 'image {0} not found'.format(image_uuid)) # if trying to delete the only one Customer then return value error if sql_image.visibility == "public": - raise ValueError("Image {} is public, no customers".format(image_uuid)) + raise ValueError( + "Image {} is public, no customers".format(image_uuid)) if len(sql_image.customers) == 1 and \ sql_image.customers[0].customer_id == customer_id: @@ -414,15 +420,32 @@ def delete_customer(image_uuid, customer_id, transaction_id): raise +def set_resource_status(sql_image, status_model): + image_wrapper = ImageWrapper.from_db_model(sql_image) + image_wrapper.image.status = 'no regions' + + if status_model and status_model.regions: + for region in image_wrapper.image.regions: + for status in status_model.regions: + if status.region == region.name: + region.status = status.status + if status.error_msg: + region.set_error_message(status.error_msg) + + image_wrapper.image.status = status_model.status + + return image_wrapper + + @di.dependsOn('data_manager') -@di.dependsOn('rds_proxy') def get_image_by_uuid(image_uuid, query_by_id_or_name=False): - """This function includes an optional boolean parameter "query_by_id_or_name". - If query_by_id_or_name evaluates to true, IMS logic will fetch the image - record whose "image_uuid" parameter value matches either the image id or name value. - Otherwise it defaults to query by image ID value only. + """This function includes an optional boolean parameter + query_by_id_or_name. If query_by_id_or_name evaluates to true, IMS logic + will fetch the image record whose "image_uuid" parameter value matches + either the image id or name value. Otherwise, it defaults to query by + image ID value only. """ - DataManager, rds_proxy = di.resolver.unpack(get_image_by_uuid) + DataManager = di.resolver.unpack(get_image_by_uuid) datamanager = DataManager() LOG.debug("Get image by uuid : {}".format(image_uuid)) @@ -431,51 +454,36 @@ def get_image_by_uuid(image_uuid, query_by_id_or_name=False): datamanager.begin_transaction() image_rec = datamanager.get_record('image') - # Only the get_image API will pass the optional parameter and set it to true + # Only the get_image API will pass the optional parameter and + # set it to true if query_by_id_or_name: sql_image = image_rec.get_image(image_uuid) else: # all other image APIs will NOT pass the optional parameter sql_image = image_rec.get_image_by_id(image_uuid) - if not sql_image: - raise NotFoundError(status_code=404, - message="Image {0} not found ".format( - image_uuid)) - image_wrapper = ImageWrapper.from_db_model(sql_image) + if not sql_image: + raise NotFoundError( + status_code=404, + message="Image {0} not found ".format(image_uuid)) + + # Get the status from resource table + uuid = [sql_image.id] + resource_status_dict = utils.get_resource_status_from_db( + datamanager.get_session(), uuid) + status_model = resource_status_dict.get(sql_image.id) + + image_wrapper = set_resource_status(sql_image, status_model) # get image own link - image_wrapper.image.links, image_wrapper.image.self_link = ImsUtils.get_server_links(image_uuid) + image_wrapper.image.links, image_wrapper.image.self_link = \ + ImsUtils.get_server_links(image_uuid) + # convert time stamp format to human readable time image_wrapper.image.created_at = ImsUtils.convert_time_human( image_wrapper.image.created_at) image_wrapper.image.updated_at = ImsUtils.convert_time_human( image_wrapper.image.updated_at) - # Get the status from RDS - image_status = rds_proxy.get_status(image_wrapper.image.id, False) - - if image_status.status_code == 404: - # image not on rds resource table - applicable to images created with no region assigned - image_wrapper.image.status = 'no regions' - - elif image_status.status_code == 200: - image_status = image_status.json() - if image_wrapper.image.regions: - image_wrapper.image.status = image_status['status'] - else: - image_wrapper.image.status = 'no regions' - - # update status for all regions - for result_regions in image_wrapper.image.regions: - for status_region in image_status['regions']: - if result_regions.name == status_region['region']: - result_regions.status = status_region['status'] - if status_region['error_msg']: - result_regions.set_error_message(status_region['error_msg']) - - # status codes not falling under 404 (not found) or 200 (success) - else: - raise ErrorStatus(500, "unsuccessful GET - failed to get status for this resource") except NotFoundError as exp: datamanager.rollback() @@ -491,11 +499,10 @@ def get_image_by_uuid(image_uuid, query_by_id_or_name=False): @di.dependsOn('data_manager') -@di.dependsOn('rds_proxy') def get_image_list_by_params(visibility, region, Customer): - DataManager, rds_proxy = di.resolver.unpack(get_image_list_by_params) - + DataManager = di.resolver.unpack(get_image_list_by_params) datamanager = DataManager() + try: image_record = datamanager.get_record('image') sql_images = image_record.get_images_by_criteria(visibility=visibility, @@ -504,35 +511,16 @@ def get_image_list_by_params(visibility, region, Customer): response = ImageSummaryResponse() if sql_images: - uuids = ','.join(str("\'" + sql_image.id + "\'") - for sql_image in sql_images if sql_image and sql_image.id) - resource_status_dict = image_record.get_images_status_by_uuids(uuids) + uuids = [sql_image.id for sql_image in sql_images] + resource_status_dict = utils.get_resource_status_from_db( + datamanager.get_session(), uuids) for sql_image in sql_images: - image = ImageSummary.from_db_model(sql_image) - if sql_image.id: - # rds_region_list contains tuples - each containing the regions associated with the image - # along with the region status - rds_region_list = resource_status_dict.get(sql_image.id) + status_model = resource_status_dict.get(sql_image.id) + wsme_image = set_resource_status(sql_image, status_model) + image_summary = ImageSummary.from_wsme(wsme_image) + response.images.append(image_summary) - if rds_region_list and image.regions: - # set image.status to 'error' if any of the regions has an 'Error' status' - # else, if any region status shows 'Submitted' then set image status to 'Pending' - # otherwise image status is 'Success' - error_status = [item for item in rds_region_list if item[1] == 'Error'] - submitted_status = [item for item in rds_region_list if item[1] == 'Submitted'] - success_status = [item for item in rds_region_list if item[1] == 'Success'] - - if len(error_status) > 0: - image.status = 'Error' - elif len(submitted_status) > 0: - image.status = 'Pending' - elif len(success_status) > 0: - image.status = 'Success' - else: - image.status = 'no regions' - - response.images.append(image) return response except ErrorStatus as exp: @@ -605,10 +593,12 @@ def enable_image(image_uuid, int_enabled, transaction_id): return ret_image except ErrorStatus as exp: - LOG.log_exception("ImageLogic - Failed to change image activation value", exp) + LOG.log_exception( + "ImageLogic - Failed to change image activation value", exp) datamanager.rollback() raise exp except Exception as exp: - LOG.log_exception("ImageLogic - Failed to change image activation value", exp) + LOG.log_exception( + "ImageLogic - Failed to change image activation value", exp) datamanager.rollback() raise exp diff --git a/orm/services/image_manager/ims/persistency/sql_alchemy/image/image_record.py b/orm/services/image_manager/ims/persistency/sql_alchemy/image/image_record.py index 0d2bdf13..7838fc59 100755 --- a/orm/services/image_manager/ims/persistency/sql_alchemy/image/image_record.py +++ b/orm/services/image_manager/ims/persistency/sql_alchemy/image/image_record.py @@ -94,21 +94,6 @@ class ImageRecord(Record): LOG.log_exception(message, exception) raise - def get_images_status_by_uuids(self, uuid_str): - results = self.session.connection().execute("SELECT id, resource_id, region, status" # nosec - " FROM rds_resource_status_view WHERE resource_id IN ({})".format(uuid_str)) - img_region_dict = {} - if results: - resource_status_dict = dict((id, (resource_id, region, status)) for id, resource_id, region, status in results) - # using resource_status_dict, create img_region_dict with resource_id as key and (region, status) as value - for v in list(resource_status_dict.values()): - if v[0] in img_region_dict: - img_region_dict[v[0]].append(v[1:]) - else: - img_region_dict[v[0]] = [v[1:]] - results.close() - return img_region_dict - def create_images_by_visibility_query(self, visibility): try: query = self.session.query(Image).filter(Image.visibility == visibility) diff --git a/orm/services/image_manager/ims/persistency/wsme/models.py b/orm/services/image_manager/ims/persistency/wsme/models.py index 3e447994..0987e988 100755 --- a/orm/services/image_manager/ims/persistency/wsme/models.py +++ b/orm/services/image_manager/ims/persistency/wsme/models.py @@ -494,6 +494,19 @@ class ImageSummary(Model): return image + @staticmethod + def from_wsme(wsme_image): + image = ImageSummary() + image.status = wsme_image.image.status + image.id = wsme_image.image.id + image.name = wsme_image.image.name + image.visibility = wsme_image.image.visibility + image.regions = [] + for wsme_region in wsme_image.image.regions: + image.regions.append(wsme_region.name) + + return image + class ImageSummaryResponse(Model): images = wsme.wsattr([ImageSummary], mandatory=True) diff --git a/orm/services/image_manager/scripts/db_scripts/create_db.sql b/orm/services/image_manager/scripts/db_scripts/create_db.sql index 9bee7f60..fd4dc74a 100755 --- a/orm/services/image_manager/scripts/db_scripts/create_db.sql +++ b/orm/services/image_manager/scripts/db_scripts/create_db.sql @@ -84,13 +84,3 @@ create table if not exists image_customer primary key (image_id,customer_id), foreign key (image_id) references image(id) ON DELETE CASCADE ON UPDATE NO ACTION ); -# - -#***** -#* MySql script for Creating View rds_resource_status_view -#***** - -create or replace view rds_resource_status_view AS - ( - SELECT ID, RESOURCE_ID, REGION,STATUS, - ERR_CODE,OPERATION from resource_status); diff --git a/orm/services/resource_distributor/rds/services/model/region_resource_id_status.py b/orm/services/resource_distributor/rds/services/model/region_resource_id_status.py index 582e0119..c3ac4349 100755 --- a/orm/services/resource_distributor/rds/services/model/region_resource_id_status.py +++ b/orm/services/resource_distributor/rds/services/model/region_resource_id_status.py @@ -1,50 +1,5 @@ -class ResourceMetaData(object): - def __init__(self, checksum, virtual_size, size): - self.size = size - self.virtual_size = virtual_size - self.checksum = checksum - - def as_dict(self): - return self.__dict__ - - -class ResourceStatusModel(object): - def __init__(self, - timestamp, - region, - status, - transaction_id, - resource_id, - ord_notifier, - err_msg, - err_code, - operation, - resource_extra_metadata=None): - self.timestamp = timestamp - self.region = region - self.status = status - self.ord_transaction_id = transaction_id - self.resource_id = resource_id - self.ord_notifier_id = ord_notifier - self.error_msg = err_msg - self.error_code = err_code - self.operation = operation - - if resource_extra_metadata: - self.resource_extra_metadata = ResourceMetaData( - checksum=resource_extra_metadata[0].checksum, - virtual_size=resource_extra_metadata[0].virtual_size, - size=resource_extra_metadata[0].size - ) - else: - self.resource_extra_metadata = None - - def as_dict(self): - return self.__dict__ - - class ResourceTemplateModel(object): def __init__(self, resource_id, @@ -62,30 +17,6 @@ class ResourceTemplateModel(object): return self.__dict__ -class StatusModel(object): - def __init__(self, status): - self.regions = status - self.status = self._get_aggregated_status() - - def _get_aggregated_status(self): - is_pending = False - for region in self.regions: - if region.status == 'Error' and region.operation.strip() != 'delete': - # If a region had an error, the aggregated status is 'Error' - return 'Error' - elif region.status == 'Submitted': - # Just set the flag but don't return, because there might be - # an error in any of the next iterations - is_pending = True - - if is_pending: - return 'Pending' - else: - # If self.regions is empty, the result will still be 'Success' but the - # server returns 404 Not Found - return 'Success' - - class RegionEndPointData(object): """class method endpoints data""" diff --git a/orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py b/orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py index 1fcb8deb..a67644cc 100755 --- a/orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py +++ b/orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py @@ -5,8 +5,9 @@ import time from oslo_db.sqlalchemy.enginefacade import LegacyEngineFacade from pecan import conf +from orm.common.orm_common.model.models import ResourceStatusModel, StatusModel from orm.services.resource_distributor.rds.services.model.region_resource_id_status \ - import ResourceStatusModel, StatusModel, RegionEndPointData + import RegionEndPointData from orm.services.resource_distributor.rds.storage import region_resource_id_status from sqlalchemy import BigInteger, BLOB, Column, ForeignKey, Integer, String, Text from sqlalchemy.ext.declarative.api import declarative_base diff --git a/orm/tests/unit/cms/test_customer_logic.py b/orm/tests/unit/cms/test_customer_logic.py index eb62e200..dbc5057f 100755 --- a/orm/tests/unit/cms/test_customer_logic.py +++ b/orm/tests/unit/cms/test_customer_logic.py @@ -42,7 +42,7 @@ class ResourceStatusModel(object): class StatusModel(object): def __init__(self, status='Success'): - self.regions_status = [ResourceStatusModel()] + self.regions = [ResourceStatusModel()] self.status = status diff --git a/orm/tests/unit/ims/logic/test_image_logic.py b/orm/tests/unit/ims/logic/test_image_logic.py index 5ebd1abd..8c2914b3 100755 --- a/orm/tests/unit/ims/logic/test_image_logic.py +++ b/orm/tests/unit/ims/logic/test_image_logic.py @@ -1,3 +1,4 @@ +from orm.common.orm_common.utils import utils from orm.services.image_manager.ims.logic import image_logic from orm.services.image_manager.ims.persistency.sql_alchemy.db_models import Image from orm.services.image_manager.ims.persistency.wsme import models @@ -81,51 +82,46 @@ image_status_dict = {'regions': [{ class TestImageLogic(FunctionalTest): @mock.patch.object(image_logic, 'di') def test_get_image_by_uuid_image_not_found(self, mock_di): - mock_rds_proxy = mock.MagicMock() - my_get_image = mock.MagicMock() - my_get_image.get_image_by_id.return_value = None - my_get_record = mock.MagicMock() - my_get_record.get_record.return_value = my_get_image - my_dm = mock.MagicMock(return_value=my_get_record) + _, mock_data_manager = \ + get_data_manager_mock(mock_sql_image=None) + mock_di.resolver.unpack.return_value = (mock_data_manager) - mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy try: image_logic.get_image_by_uuid('te') except image_logic.ErrorStatus as e: self.assertEqual(e.status_code, 404) + @mock.patch.object(utils, 'get_resource_status_from_db') @mock.patch.object(image_logic.ImsUtils, 'get_server_links', return_value=["ip", "path"]) @mock.patch.object(image_logic, 'di') @mock.patch.object(image_logic, 'ImageWrapper') - def test_get_image_by_uuid_image_no_status(self, mock_image, - mock_di, mock_links): - mock_rds_proxy = mock.MagicMock() - mock_rds_proxy.get_status.return_value = RDSGetStatus(status_code=404) + def test_get_image_by_uuid_image_no_status( + self, mock_image, mock_di, mock_links, mock_get_resource): + _, mock_data_manager = get_data_manager_mock() + mock_di.resolver.unpack.return_value = (mock_data_manager) mock_image.from_db_model.return_value = ImageWrapperTest() - my_get_image = mock.MagicMock() - my_get_record = mock.MagicMock() - my_get_record.get_record.return_value = my_get_image - my_dm = mock.MagicMock(return_value=my_get_record) - mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy + mock_get_resource.return_value = {} + result = image_logic.get_image_by_uuid('test') self.assertEqual(result.image.status, 'no regions') + @mock.patch.object(utils, 'get_resource_status_from_db') @mock.patch.object(image_logic.ImsUtils, 'get_server_links', return_value=["ip", "path"]) @mock.patch.object(image_logic, 'di') @mock.patch.object(image_logic, 'ImageWrapper') - def test_get_image_by_uuid_image_sanity(self, mock_image, - mock_di, mock_links): - mock_rds_proxy = mock.MagicMock() - mock_rds_proxy.get_status.return_value = RDSGetStatus() - my_get_image = mock.MagicMock() - my_get_record = mock.MagicMock() - my_get_record.get_record.return_value = my_get_image - my_dm = mock.MagicMock(return_value=my_get_record) + def test_get_image_by_uuid_image_sanity( + self, mock_image, mock_di, mock_links, mock_get_resource): + _, mock_data_manager = get_data_manager_mock() + mock_di.resolver.unpack.return_value = (mock_data_manager) + + status_model = mock.MagicMock() + status_model.status = 'Success' + resource_status_dict = {'some_id': status_model} + mock_get_resource.return_value = resource_status_dict - mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy result = image_logic.get_image_by_uuid('test') self.assertEqual(result.image.status, 'Success') @@ -133,50 +129,50 @@ class TestImageLogic(FunctionalTest): class TestDeleteImageLogic(FunctionalTest): """test delete image.""" - @mock.patch.object(image_logic, 'update_region_actions', return_value=True) + @mock.patch.object(utils, 'get_resource_status_from_db') @mock.patch.object(image_logic, 'di') - def test_delete_image_success(self, mock_di, mock_update_region): - mock_rds_proxy, mock_data_manager = get_data_manager_mock( + def test_delete_image_success(self, mock_di, mock_get_resource): + _, mock_data_manager = get_data_manager_mock( get_existing_region_names=[]) - mock_rds_proxy.get_status.return_value = RdsResponse() - mock_di.resolver.unpack.return_value = (mock_rds_proxy, - mock_data_manager) - global regions - regions = [] + mock_di.resolver.unpack.return_value = (mock_data_manager) + + status_model = mock.MagicMock() + status_model.status = 'Success' + resource_status_dict = {'some_id': status_model} + mock_get_resource.return_value = resource_status_dict + image_logic.delete_image_by_uuid("image_uuid", "transaction_id") - @mock.patch.object(image_logic, 'update_region_actions', return_value=True) + @mock.patch.object(utils, 'get_resource_status_from_db') @mock.patch.object(image_logic, 'di') - def test_delete_image_success_nords(self, mock_di, mock_update_region): - mock_rds_proxy, mock_data_manager = \ - get_data_manager_mock(imagejson={"regions": {}}, - get_existing_region_names=[]) - mock_rds_proxy.get_status.return_value = RdsResponse() - mock_di.resolver.unpack.return_value = (mock_rds_proxy, - mock_data_manager) + def test_delete_image_success_nords(self, mock_di, mock_get_resource): + _, mock_data_manager = get_data_manager_mock( + get_existing_region_names=[]) + mock_di.resolver.unpack.return_value = (mock_data_manager) + + mock_get_resource.return_value = {} image_logic.delete_image_by_uuid("image_uuid", "transaction_id") - @mock.patch.object(image_logic, 'update_region_actions', return_value=True) @mock.patch.object(image_logic, 'di') - def test_delete_image_notfound_error(self, mock_di, mock_update_region): - mock_rds_proxy, mock_data_manager = \ + def test_delete_image_notfound_error(self, mock_di): + _, mock_data_manager = \ get_data_manager_mock(mock_sql_image=None) - mock_di.resolver.unpack.return_value = (mock_rds_proxy, - mock_data_manager) + mock_di.resolver.unpack.return_value = (mock_data_manager) + try: image_logic.delete_image_by_uuid("image_uuid", "transaction_id") except Exception as e: self.assertEqual(404, e.status_code) - @mock.patch.object(image_logic, 'update_region_actions', - side_effect=ValueError('test')) @mock.patch.object(image_logic, 'di') - def test_delete_image_other_error(self, mock_di, mock_update_region): - mock_rds_proxy, mock_data_manager = get_data_manager_mock() - mock_di.resolver.unpack.return_value = (mock_rds_proxy, - mock_data_manager) - self.assertRaises(image_logic.ErrorStatus, image_logic.delete_image_by_uuid, - "image_uuid", "transaction_id") + def test_delete_image_other_error(self, mock_di): + _, mock_data_manager = get_data_manager_mock() + mock_di.resolver.unpack.return_value = (mock_data_manager) + + self.assertRaises(image_logic.ErrorStatus, + image_logic.delete_image_by_uuid, + "image_uuid", + "transaction_id") class TestUpdateImage(FunctionalTest): @@ -310,14 +306,13 @@ class TestActivateImageLogic(FunctionalTest): class TestListImageLogic(FunctionalTest): @mock.patch.object(image_logic, 'di') def test_list_image_not_found(self, mock_di): - mock_rds_proxy = mock.MagicMock() my_get_image = mock.MagicMock() my_get_image.get_image.return_value = None my_get_record = mock.MagicMock() my_get_record.get_record.side_effect = image_logic.ErrorStatus(404, 'a') my_dm = mock.MagicMock(return_value=my_get_record) - mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy + mock_di.resolver.unpack.return_value = my_dm try: image_logic.get_image_list_by_params('a', 'b', 'c') except image_logic.ErrorStatus as e: @@ -326,13 +321,12 @@ class TestListImageLogic(FunctionalTest): @mock.patch.object(image_logic, 'di') @mock.patch.object(image_logic, 'ImageWrapper') def test_list_image_error(self, mock_image, mock_di): - mock_rds_proxy = mock.MagicMock() my_get_image = mock.MagicMock() my_get_record = mock.MagicMock() my_get_record.get_record.side_effect = SystemError() my_dm = mock.MagicMock(return_value=my_get_record) - mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy + mock_di.resolver.unpack.return_value = my_dm try: image_logic.get_image_list_by_params('a', 'b', 'c') except Exception as e: @@ -341,17 +335,12 @@ class TestListImageLogic(FunctionalTest): @mock.patch.object(image_logic, 'di') @mock.patch.object(image_logic, 'ImageWrapper') def test_list_image_sanity(self, mock_image, mock_di): - imagejson = [{"regions": {"name": "mdt1"}}] - mock_rds_proxy = mock.MagicMock() - mock_rds_proxy.get_status.return_value = RDSGetStatus() mock_data_manager = mock.MagicMock() mock_image_rec = mock.MagicMock() - image_json = mock.MagicMock() - image_json.return_value = imagejson mock_image_rec.get_images_by_criteria.return_value = Image() my_dm = mock.MagicMock(mock_data_manager) - mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy + mock_di.resolver.unpack.return_value = my_dm result = image_logic.get_image_list_by_params('a', 'b', 'c') self.assertEqual(len(result.images), 0) @@ -688,6 +677,7 @@ def get_data_manager_mock(get_existing_region_names={"name": "mdt1"}, if mock_sql_image: mock_sql_image = mock.MagicMock() mock_sql_image.__json__ = image_json + mock_sql_image.id = 'some_id' mock_sql_image.visibility = visibility mock_sql_image.protected = protected mock_sql_image.get_proxy_dict = mock.MagicMock(return_value={'regions': regions}) diff --git a/orm/tests/unit/rds/controllers/v1/status/test_get_resource_status.py b/orm/tests/unit/rds/controllers/v1/status/test_get_resource_status.py index f4fa02bf..f90b2885 100755 --- a/orm/tests/unit/rds/controllers/v1/status/test_get_resource_status.py +++ b/orm/tests/unit/rds/controllers/v1/status/test_get_resource_status.py @@ -1,9 +1,10 @@ """unittest get resource status.""" from mock import MagicMock import orm.services.resource_distributor.rds.controllers.v1.status.get_resource as resource -from orm.services.resource_distributor.rds.services.model.region_resource_id_status import ResourceStatusModel, StatusModel from orm.tests.unit.rds.controllers.v1.functional_test import FunctionalTest +from orm.common.orm_common.model.models import ResourceStatusModel, StatusModel + class EmptyModel(object): """mock class.""" diff --git a/orm/tests/unit/rds/services/model/test_region_resource_id_status.py b/orm/tests/unit/rds/services/model/test_region_resource_id_status.py index f48b28db..4ace1190 100755 --- a/orm/tests/unit/rds/services/model/test_region_resource_id_status.py +++ b/orm/tests/unit/rds/services/model/test_region_resource_id_status.py @@ -1,12 +1,13 @@ import unittest +from orm.common.orm_common.model import models from orm.services.resource_distributor.rds.services.model import region_resource_id_status class TestModel(unittest.TestCase): def test_model_as_dict(self): - model = region_resource_id_status.ResourceStatusModel(1, 2, 3, 4, 5, 6, 7, 8, - 'create') + model = models.ResourceStatusModel( + 1, 2, 3, 4, 5, 6, 7, 8, 'create') expected_dict = { 'timestamp': 1, 'region': 2, @@ -42,19 +43,19 @@ class TestResourceTemplateModel(unittest.TestCase): class TestStatusModel(unittest.TestCase): def test_get_aggregated_status_error(self): - model = region_resource_id_status.ResourceStatusModel(1, 2, 'Error', 4, 5, 6, 7, 8, - 'create') - status_model = region_resource_id_status.StatusModel([model]) + model = models.ResourceStatusModel( + 1, 2, 'Error', 4, 5, 6, 7, 8, 'create') + status_model = models.StatusModel([model]) self.assertEqual(status_model.status, 'Error') def test_get_aggregated_status_pending(self): - model = region_resource_id_status.ResourceStatusModel(1, 2, 'Submitted', 4, 5, 6, 7, - 8, 'create') - status_model = region_resource_id_status.StatusModel([model]) + model = models.ResourceStatusModel( + 1, 2, 'Submitted', 4, 5, 6, 7, 8, 'create') + status_model = models.StatusModel([model]) self.assertEqual(status_model.status, 'Pending') def test_get_aggregated_status_success(self): - model = region_resource_id_status.ResourceStatusModel(1, 2, 'Success', 4, 5, 6, 7, 8, - 'create') - status_model = region_resource_id_status.StatusModel([model]) + model = models.ResourceStatusModel( + 1, 2, 'Success', 4, 5, 6, 7, 8, 'create') + status_model = models.StatusModel([model]) self.assertEqual(status_model.status, 'Success') diff --git a/orm/tests/unit/rds/services/test_create_resource.py b/orm/tests/unit/rds/services/test_create_resource.py index 5e7a46fe..a15219b3 100755 --- a/orm/tests/unit/rds/services/test_create_resource.py +++ b/orm/tests/unit/rds/services/test_create_resource.py @@ -2,10 +2,10 @@ import unittest from unittest.mock import patch +from orm.common.orm_common.model.models import (ResourceStatusModel, + ResourceMetaData, + StatusModel) from orm.services.resource_distributor.rds.services import resource as ResourceService -from orm.services.resource_distributor.rds.services.model.region_resource_id_status import (ResourceStatusModel, - ResourceMetaData, - StatusModel) result = ResourceStatusModel( status="success", timestamp="123456789", region="name",