x/ranger
Code Issues Proposed changes

Removed RDS service call (cont'd)

Browse Source 
Change-Id: I0c5bebd53faca231f58b0280ef1423ef0d5d50fb
This commit is contained in:
Chi Lo 2020-02-21 16:20:10 -08:00
parent d026199784
commit 2cba5155c7
22 changed files with 245 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
orm/base_config.py
View File

@ -44,6 +44,9 @@ OrmOpts = [
cfg.StrOpt('ranger_agent_client_cert_path',
default='',
help='Ranger Agent certificate path.'),
cfg.IntOpt('resource_status_max_interval',
default=2,
help='Interval in seconds.'),
cfg.StrOpt('log_location',
default='/var/log/ranger',
help='Orm log directory.'),

35
orm/common/orm_common/model/models.py
View File

@ -1,17 +1,42 @@
class ResourceMetaData(object):
def __init__(self, checksum, virtual_size, size):
self.size = size
self.virtual_size = virtual_size
self.checksum = checksum
def as_dict(self):
return self.__dict__
class ResourceStatusModel(object):
def __init__(self,
timestamp,
region,
status,
transaction_id,
resource_id,
ord_notifier,
err_msg,
err_code,
operation,
err_msg):
resource_extra_metadata=None):
self.timestamp = timestamp
self.region = region
self.status = status
self.ord_transaction_id = transaction_id
self.resource_id = resource_id
self.operation = operation
self.ord_notifier_id = ord_notifier
self.error_msg = err_msg
self.error_code = err_code
self.operation = operation
if resource_extra_metadata:
self.resource_extra_metadata = ResourceMetaData(
checksum=resource_extra_metadata[0].checksum,
virtual_size=resource_extra_metadata[0].virtual_size,
size=resource_extra_metadata[0].size)
else:
self.resource_extra_metadata = None
def as_dict(self):
return self.__dict__
@ -19,12 +44,12 @@ class ResourceStatusModel(object):
class StatusModel(object):
def __init__(self, status):
self.regions_status = status
self.regions = status
self.status = self._get_aggregated_status()
def _get_aggregated_status(self):
is_pending = False
for region in self.regions_status:
for region in self.regions:
if (region.status == 'Error' and
region.operation.strip() != 'delete'):
# If a region had an error, the aggregated status is 'Error'
@ -37,6 +62,6 @@ class StatusModel(object):
if is_pending:
return 'Pending'
else:
# If self.regions_status is empty, the result will still be
# If self.regions is empty, the result will still be
# 'Success' but the server returns 404 Not Found
return 'Success'

20
orm/common/orm_common/sql_alchemy/db_models.py
View File

@ -1,5 +1,5 @@
from oslo_db.sqlalchemy import models
from sqlalchemy import BigInteger, Column, ForeignKey, Integer, Text
from sqlalchemy import BigInteger, Column, Integer, Text
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
@ -9,24 +9,6 @@ class CommonBaseModel(models.ModelBase):
__table_args__ = {'mysql_engine': 'InnoDB'}
class ImageMetadData(Base, CommonBaseModel):
__tablename__ = 'image_metadata'
image_meta_data_id = Column(ForeignKey('resource_status.id'),
primary_key=True)
checksum = Column(Text, primary_key=False)
virtual_size = Column(Text, primary_key=False)
size = Column(Text, primary_key=False)
def __json__(self):
return dict(
image_meta_data_id=self.image_meta_data_id,
checksum=self.checksum,
virtual_size=self.virtual_size,
size=self.size
)
class ResourceStatus(Base, CommonBaseModel):
__tablename__ = 'resource_status'

39
orm/common/orm_common/sql_alchemy/resource_status_record.py
View File

@ -3,6 +3,8 @@ from orm.common.orm_common.model.models import ResourceStatusModel, StatusModel
from orm.common.orm_common.sql_alchemy.db_models import ResourceStatus
import time
from oslo_config import cfg
logger = logging.getLogger(__name__)
@ -32,26 +34,26 @@ class ResourceStatusRecord:
timestamp = int(time.time()) * 1000
# assume same time period for all resource types
# max_interval_time_in_seconds = \
# conf.region_resource_id_status.max_interval_time.default * 60
max_interval_time_in_seconds = 2 * 60
ref_timestamp = (int(time.time()) -
max_interval_time_in_seconds) * 1000
max_interval_in_seconds = cfg.CONF.resource_status_max_interval * 60
ref_timestamp = (int(time.time()) - max_interval_in_seconds) * 1000
try:
records = self.session.query(
ResourceStatus.id,
ResourceStatus.resource_id,
ResourceStatus.timestamp,
ResourceStatus.region,
ResourceStatus.status,
ResourceStatus.timestamp,
ResourceStatus.operation,
ResourceStatus.err_msg).filter(
ResourceStatus.transaction_id,
ResourceStatus.resource_id,
ResourceStatus.ord_notifier,
ResourceStatus.err_msg,
ResourceStatus.err_code,
ResourceStatus.operation).filter(
ResourceStatus.resource_id.in_(resource_uuids)).all()
if records:
for id, resource_id, region, status, timestamp, operation, \
err_msg in records:
for id, timestamp, region, status, transaction, resource, \
notifier, err_msg, err_code, operation in records:
if (status == "Submitted" and
timestamp < ref_timestamp):
timestamp = timestamp
@ -63,14 +65,17 @@ class ResourceStatusRecord:
timestamp,
region,
status,
resource_id,
operation,
err_msg)
transaction,
resource,
notifier,
err_msg,
err_code,
operation)
if resource_id in records_model:
records_model[resource_id].append(status_model)
if resource in records_model:
records_model[resource].append(status_model)
else:
records_model[resource_id] = [status_model]
records_model[resource] = [status_model]
for id, model in records_model.items():
statuses_model[id] = StatusModel(model)

16
orm/services/customer_manager/cms_rest/data/sql_alchemy/customer_record.py
View File

@ -88,22 +88,6 @@ class CustomerRecord:
else:
return None
def get_customers_status_by_uuids(self, uuid_str):
results = self.session.connection().execute("SELECT id, resource_id, region, status" # nosec
" FROM rds_resource_status_view WHERE resource_id IN ({})".format(uuid_str))
cust_region_dict = {}
if results:
resource_status_dict = dict((id, (resource_id, region, status)) for id, resource_id, region, status in results)
# using resource_status_dict, create cust_region_dict with resource_id as key and (region, status) as value
for v in list(resource_status_dict.values()):
if v[0] in cust_region_dict:
cust_region_dict[v[0]].append(v[1:])
else:
cust_region_dict[v[0]] = [v[1:]]
results.close()
return cust_region_dict
def delete_customer_by_uuid(self, uuid):
try:
result = self.session.query(Customer).filter(

2
orm/services/customer_manager/cms_rest/data/sql_alchemy/group_record.py
View File

@ -79,7 +79,7 @@ class GroupRecord:
def get_groups_status_by_uuids(self, uuid_str):
cmd = "SELECT id, resource_id, region, status FROM " \
"rds_resource_status_view WHERE resource_id IN (%s)"
"resource_status WHERE resource_id IN (%s)"
results = self.session.connection().execute(cmd % uuid_str)
group_region = {}

4
orm/services/customer_manager/cms_rest/logic/customer_logic.py
View File

@ -630,9 +630,9 @@ class CustomerLogic(object):
wsme_customer = sql_customer.to_wsme()
wsme_customer.status = 'no regions'
if status_model and status_model.regions_status:
if status_model and status_model.regions:
for region in wsme_customer.regions:
for status in status_model.regions_status:
for status in status_model.regions:
if status.region == region.name:
region.status = status.status
if status.error_msg:

5
orm/services/customer_manager/scripts/db_scripts/ranger_cms_create_db.sql
View File

@ -90,11 +90,6 @@ create table if not exists user_role
index region_id (region_id),
index user_id (user_id));
create or replace view rds_resource_status_view AS
(
SELECT id, resource_id, region, status,
err_code, operation from resource_status);
create table if not exists cms_domain
(
id integer auto_increment not null,

18
orm/services/flavor_manager/fms_rest/data/sql_alchemy/flavor/flavor_record.py
View File

@ -147,24 +147,6 @@ class FlavorRecord:
LOG.log_exception(message, exception)
raise
def get_flavors_status_by_uuids(self, uuid_str):
results = self.session.connection().execute("SELECT id, resource_id, region, status" # nosec
" FROM rds_resource_status_view WHERE resource_id IN ({})".format(uuid_str))
flvr_region_dict = {}
if results:
resource_status_dict = dict((id, (resource_id, region, status)) for id, resource_id, region, status in results)
# using resource_status_dict, create flvr_region_dict with resource_id as key and (region, status) as value
for v in list(resource_status_dict.values()):
if v[0] in flvr_region_dict:
flvr_region_dict[v[0]].append(v[1:])
else:
flvr_region_dict[v[0]] = [v[1:]]
results.close()
return flvr_region_dict
def get_flavors_by_criteria(self, **criteria):
try:

4
orm/services/flavor_manager/fms_rest/logic/flavor_logic.py
View File

@ -873,10 +873,10 @@ def update_region_statuses(flavor, region_names, status_model):
flavor.status = 'no regions'
error_status = submitted_status = success_status = 0
if status_model and status_model.regions_status:
if status_model and status_model.regions:
# get region status if region in flavor_regions_list
if region_names:
for status in status_model.regions_status:
for status in status_model.regions:
# check that the region read from RDS is in the
# flavor_regions_list
if status.region in region_names:

11
orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql
View File

@ -99,14 +99,3 @@ create table if not exists flavor_option
foreign key (flavor_internal_id) references flavor(internal_id) ON DELETE CASCADE ON UPDATE NO ACTION,
primary key (flavor_internal_id,key_name)
);
#
#*****
#* MySql script for Creating View rds_resource_status_view
#*****
create or replace view rds_resource_status_view AS
(
SELECT ID, RESOURCE_ID, REGION,STATUS,
ERR_CODE,OPERATION from resource_status);

204
orm/services/image_manager/ims/logic/image_logic.py
View File

@ -1,11 +1,14 @@
import time
from orm.common.orm_common.injector import injector
from orm.common.orm_common.utils import utils
from orm.services.image_manager.ims.logger import get_logger
from orm.services.image_manager.ims.logic.error_base import ErrorStatus, NotFoundError
from orm.services.image_manager.ims.persistency.sql_alchemy.db_models import ImageCustomer, ImageRegion
from orm.services.image_manager.ims.persistency.wsme.models import (ImageSummary, ImageSummaryResponse,
ImageWrapper, RegionWrapper)
from orm.services.image_manager.ims.logic.error_base import \
ErrorStatus, NotFoundError
from orm.services.image_manager.ims.persistency.sql_alchemy.db_models import \
ImageCustomer, ImageRegion
from orm.services.image_manager.ims.persistency.wsme.models import (
ImageSummary, ImageSummaryResponse, ImageWrapper, RegionWrapper)
from orm.services.image_manager.ims.utils import utils as ImsUtils
import oslo_db
@ -63,13 +66,13 @@ def send_to_rds_if_needed(sql_image, existing_region_names, http_action,
image_dict = sql_image.get_proxy_dict()
update_region_actions(image_dict, existing_region_names, http_action)
if image_dict['regions'] or len(existing_region_names) > 0:
LOG.debug("Image is valid to send to RDS - sending to RDS Proxy ")
LOG.debug("Image is valid, sending to RDS Proxy ")
rds_proxy.send_image(image_dict, transaction_id, http_action)
else:
LOG.debug("Group with no regions - wasn't send to RDS Proxy " + str(
LOG.debug("Group with no regions, not sending to RDS Proxy " + str(
sql_image))
else:
LOG.debug("Image with no regions - wasn't send to RDS Proxy " + str(
LOG.debug("Image with no regions, not sending to RDS Proxy " + str(
sql_image))
@ -87,9 +90,10 @@ def update_image(image_wrapper, image_uuid, transaction_id, http_action="put"):
sql_image = image_rec.get_image_by_id(image_uuid)
if sql_image is None:
raise NotFoundError(status_code=404,
message="Image {0} does not exist for update".format(
image_uuid))
raise NotFoundError(
status_code=404,
message="Image {0} does not exist for update".format(
image_uuid))
image_wrapper.validate_update(sql_image, new_image)
@ -120,51 +124,48 @@ def update_image(image_wrapper, image_uuid, transaction_id, http_action="put"):
raise
@di.dependsOn('rds_proxy')
@di.dependsOn('data_manager')
def delete_image_by_uuid(image_uuid, transaction_id):
rds_proxy, DataManager = di.resolver.unpack(delete_image_by_uuid)
DataManager = di.resolver.unpack(delete_image_by_uuid)
datamanager = DataManager()
try:
datamanager.begin_transaction()
image_rec = datamanager.get_record('image')
sql_image = image_rec.get_image_by_id(image_uuid)
if sql_image is None:
raise NotFoundError(message="Image '{}' not found".format(image_uuid))
raise NotFoundError(
message="Image '{}' not found".format(image_uuid))
image_existing_region_names = sql_image.get_existing_region_names()
if len(image_existing_region_names) > 0:
# Do not delete a flavor that still has some regions
# Do not delete an image that still has region(s)
raise ErrorStatus(405,
"Cannot delete a image with regions. "
"Please delete the regions first and then "
"delete the image. ")
# Get status from RDS
image_status = rds_proxy.get_status(sql_image.id, False)
status_resp = None
if image_status.status_code == 200:
status_resp = image_status.json()['status']
LOG.debug('RDS returned status: {}'.format(status_resp))
elif image_status.status_code == 404:
status_resp = 'Success'
# Get status from resource status table
uuid = [sql_image.id]
resource_status_dict = utils.get_resource_status_from_db(
datamanager.get_session(), uuid)
status_model = resource_status_dict.get(sql_image.id)
if status_model:
status = status_model.status
LOG.debug('Status from resource_status table: {}'.format(status))
else:
# fail to get status from rds
raise ErrorStatus(500, "fail to get status for this resource "
"deleting image not allowed ")
# Image not found in table, that means it never had any
# regions. So it is OK to delete it
status = 'Success'
LOG.debug('Resource not found in table, so it is OK to delete')
if status_resp != 'Success':
if status != 'Success':
raise ErrorStatus(405, "not allowed as aggregate status "
"have to be Success (either the deletion"
" failed on one of the regions or it is "
"still in progress)")
"have to be Success (either the deletion "
"failed on one of the regions or it is "
"still in progress)")
image_rec.delete_image_by_id(image_uuid)
datamanager.flush() # i want to get any exception created by this
@ -193,7 +194,8 @@ def add_regions(image_uuid, regions, transaction_id):
existing_region_names = sql_image.get_existing_region_names()
for region in regions.regions:
db_region = ImageRegion(region_name=region.name, region_type=region.type)
db_region = ImageRegion(region_name=region.name,
region_type=region.type)
sql_image.add_region(db_region)
datamanager.flush() # i want to get any exception created by
@ -236,7 +238,8 @@ def replace_regions(image_uuid, regions, transaction_id):
datamanager.flush()
for region in regions.regions:
db_region = ImageRegion(region_name=region.name, region_type=region.type)
db_region = ImageRegion(region_name=region.name,
region_type=region.type)
sql_image.add_region(db_region)
datamanager.flush() # i want to get any exception created by
# previous actions against the database
@ -335,7 +338,8 @@ def add_customers(image_uuid, customers, transaction_id):
return ret_image
except Exception as exp:
if 'conflicts with persistent instance' in str(exp) or 'Duplicate entry' in str(exp):
if 'conflicts with persistent instance' in str(exp) or \
'Duplicate entry' in str(exp):
raise ErrorStatus(409, "Duplicate Customer for Image")
LOG.log_exception("ImageLogic - Failed to add Customers", exp)
datamanager.rollback()
@ -373,7 +377,8 @@ def replace_customers(image_uuid, customers, transaction_id):
return ret_image
except Exception as exp:
if 'conflicts with persistent instance' in str(exp) or 'Duplicate entry' in str(exp):
if 'conflicts with persistent instance' in str(exp) or \
'Duplicate entry' in str(exp):
raise ErrorStatus(409, "Duplicate Customer for Image")
LOG.log_exception("ImageLogic - Failed to add Customers", exp)
datamanager.rollback()
@ -392,7 +397,8 @@ def delete_customer(image_uuid, customer_id, transaction_id):
raise ErrorStatus(404, 'image {0} not found'.format(image_uuid))
# if trying to delete the only one Customer then return value error
if sql_image.visibility == "public":
raise ValueError("Image {} is public, no customers".format(image_uuid))
raise ValueError(
"Image {} is public, no customers".format(image_uuid))
if len(sql_image.customers) == 1 and \
sql_image.customers[0].customer_id == customer_id:
@ -414,15 +420,32 @@ def delete_customer(image_uuid, customer_id, transaction_id):
raise
def set_resource_status(sql_image, status_model):
image_wrapper = ImageWrapper.from_db_model(sql_image)
image_wrapper.image.status = 'no regions'
if status_model and status_model.regions:
for region in image_wrapper.image.regions:
for status in status_model.regions:
if status.region == region.name:
region.status = status.status
if status.error_msg:
region.set_error_message(status.error_msg)
image_wrapper.image.status = status_model.status
return image_wrapper
@di.dependsOn('data_manager')
@di.dependsOn('rds_proxy')
def get_image_by_uuid(image_uuid, query_by_id_or_name=False):
"""This function includes an optional boolean parameter "query_by_id_or_name".
If query_by_id_or_name evaluates to true, IMS logic will fetch the image
record whose "image_uuid" parameter value matches either the image id or name value.
Otherwise it defaults to query by image ID value only.
"""This function includes an optional boolean parameter
query_by_id_or_name. If query_by_id_or_name evaluates to true, IMS logic
will fetch the image record whose "image_uuid" parameter value matches
either the image id or name value. Otherwise, it defaults to query by
image ID value only.
"""
DataManager, rds_proxy = di.resolver.unpack(get_image_by_uuid)
DataManager = di.resolver.unpack(get_image_by_uuid)
datamanager = DataManager()
LOG.debug("Get image by uuid : {}".format(image_uuid))
@ -431,51 +454,36 @@ def get_image_by_uuid(image_uuid, query_by_id_or_name=False):
datamanager.begin_transaction()
image_rec = datamanager.get_record('image')
# Only the get_image API will pass the optional parameter and set it to true
# Only the get_image API will pass the optional parameter and
# set it to true
if query_by_id_or_name:
sql_image = image_rec.get_image(image_uuid)
else:
# all other image APIs will NOT pass the optional parameter
sql_image = image_rec.get_image_by_id(image_uuid)
if not sql_image:
raise NotFoundError(status_code=404,
message="Image {0} not found ".format(
image_uuid))
image_wrapper = ImageWrapper.from_db_model(sql_image)
if not sql_image:
raise NotFoundError(
status_code=404,
message="Image {0} not found ".format(image_uuid))
# Get the status from resource table
uuid = [sql_image.id]
resource_status_dict = utils.get_resource_status_from_db(
datamanager.get_session(), uuid)
status_model = resource_status_dict.get(sql_image.id)
image_wrapper = set_resource_status(sql_image, status_model)
# get image own link
image_wrapper.image.links, image_wrapper.image.self_link = ImsUtils.get_server_links(image_uuid)
image_wrapper.image.links, image_wrapper.image.self_link = \
ImsUtils.get_server_links(image_uuid)
# convert time stamp format to human readable time
image_wrapper.image.created_at = ImsUtils.convert_time_human(
image_wrapper.image.created_at)
image_wrapper.image.updated_at = ImsUtils.convert_time_human(
image_wrapper.image.updated_at)
# Get the status from RDS
image_status = rds_proxy.get_status(image_wrapper.image.id, False)
if image_status.status_code == 404:
# image not on rds resource table - applicable to images created with no region assigned
image_wrapper.image.status = 'no regions'
elif image_status.status_code == 200:
image_status = image_status.json()
if image_wrapper.image.regions:
image_wrapper.image.status = image_status['status']
else:
image_wrapper.image.status = 'no regions'
# update status for all regions
for result_regions in image_wrapper.image.regions:
for status_region in image_status['regions']:
if result_regions.name == status_region['region']:
result_regions.status = status_region['status']
if status_region['error_msg']:
result_regions.set_error_message(status_region['error_msg'])
# status codes not falling under 404 (not found) or 200 (success)
else:
raise ErrorStatus(500, "unsuccessful GET - failed to get status for this resource")
except NotFoundError as exp:
datamanager.rollback()
@ -491,11 +499,10 @@ def get_image_by_uuid(image_uuid, query_by_id_or_name=False):
@di.dependsOn('data_manager')
@di.dependsOn('rds_proxy')
def get_image_list_by_params(visibility, region, Customer):
DataManager, rds_proxy = di.resolver.unpack(get_image_list_by_params)
DataManager = di.resolver.unpack(get_image_list_by_params)
datamanager = DataManager()
try:
image_record = datamanager.get_record('image')
sql_images = image_record.get_images_by_criteria(visibility=visibility,
@ -504,35 +511,16 @@ def get_image_list_by_params(visibility, region, Customer):
response = ImageSummaryResponse()
if sql_images:
uuids = ','.join(str("\'" + sql_image.id + "\'")
for sql_image in sql_images if sql_image and sql_image.id)
resource_status_dict = image_record.get_images_status_by_uuids(uuids)
uuids = [sql_image.id for sql_image in sql_images]
resource_status_dict = utils.get_resource_status_from_db(
datamanager.get_session(), uuids)
for sql_image in sql_images:
image = ImageSummary.from_db_model(sql_image)
if sql_image.id:
# rds_region_list contains tuples - each containing the regions associated with the image
# along with the region status
rds_region_list = resource_status_dict.get(sql_image.id)
status_model = resource_status_dict.get(sql_image.id)
wsme_image = set_resource_status(sql_image, status_model)
image_summary = ImageSummary.from_wsme(wsme_image)
response.images.append(image_summary)
if rds_region_list and image.regions:
# set image.status to 'error' if any of the regions has an 'Error' status'
# else, if any region status shows 'Submitted' then set image status to 'Pending'
# otherwise image status is 'Success'
error_status = [item for item in rds_region_list if item[1] == 'Error']
submitted_status = [item for item in rds_region_list if item[1] == 'Submitted']
success_status = [item for item in rds_region_list if item[1] == 'Success']
if len(error_status) > 0:
image.status = 'Error'
elif len(submitted_status) > 0:
image.status = 'Pending'
elif len(success_status) > 0:
image.status = 'Success'
else:
image.status = 'no regions'
response.images.append(image)
return response
except ErrorStatus as exp:
@ -605,10 +593,12 @@ def enable_image(image_uuid, int_enabled, transaction_id):
return ret_image
except ErrorStatus as exp:
LOG.log_exception("ImageLogic - Failed to change image activation value", exp)
LOG.log_exception(
"ImageLogic - Failed to change image activation value", exp)
datamanager.rollback()
raise exp
except Exception as exp:
LOG.log_exception("ImageLogic - Failed to change image activation value", exp)
LOG.log_exception(
"ImageLogic - Failed to change image activation value", exp)
datamanager.rollback()
raise exp

15
orm/services/image_manager/ims/persistency/sql_alchemy/image/image_record.py
View File

@ -94,21 +94,6 @@ class ImageRecord(Record):
LOG.log_exception(message, exception)
raise
def get_images_status_by_uuids(self, uuid_str):
results = self.session.connection().execute("SELECT id, resource_id, region, status" # nosec
" FROM rds_resource_status_view WHERE resource_id IN ({})".format(uuid_str))
img_region_dict = {}
if results:
resource_status_dict = dict((id, (resource_id, region, status)) for id, resource_id, region, status in results)
# using resource_status_dict, create img_region_dict with resource_id as key and (region, status) as value
for v in list(resource_status_dict.values()):
if v[0] in img_region_dict:
img_region_dict[v[0]].append(v[1:])
else:
img_region_dict[v[0]] = [v[1:]]
results.close()
return img_region_dict
def create_images_by_visibility_query(self, visibility):
try:
query = self.session.query(Image).filter(Image.visibility == visibility)

13
orm/services/image_manager/ims/persistency/wsme/models.py
View File

@ -494,6 +494,19 @@ class ImageSummary(Model):
return image
@staticmethod
def from_wsme(wsme_image):
image = ImageSummary()
image.status = wsme_image.image.status
image.id = wsme_image.image.id
image.name = wsme_image.image.name
image.visibility = wsme_image.image.visibility
image.regions = []
for wsme_region in wsme_image.image.regions:
image.regions.append(wsme_region.name)
return image
class ImageSummaryResponse(Model):
images = wsme.wsattr([ImageSummary], mandatory=True)

10
orm/services/image_manager/scripts/db_scripts/create_db.sql
View File

@ -84,13 +84,3 @@ create table if not exists image_customer
primary key (image_id,customer_id),
foreign key (image_id) references image(id) ON DELETE CASCADE ON UPDATE NO ACTION
);
#
#*****
#* MySql script for Creating View rds_resource_status_view
#*****
create or replace view rds_resource_status_view AS
(
SELECT ID, RESOURCE_ID, REGION,STATUS,
ERR_CODE,OPERATION from resource_status);

69
orm/services/resource_distributor/rds/services/model/region_resource_id_status.py
View File

@ -1,50 +1,5 @@
class ResourceMetaData(object):
def __init__(self, checksum, virtual_size, size):
self.size = size
self.virtual_size = virtual_size
self.checksum = checksum
def as_dict(self):
return self.__dict__
class ResourceStatusModel(object):
def __init__(self,
timestamp,
region,
status,
transaction_id,
resource_id,
ord_notifier,
err_msg,
err_code,
operation,
resource_extra_metadata=None):
self.timestamp = timestamp
self.region = region
self.status = status
self.ord_transaction_id = transaction_id
self.resource_id = resource_id
self.ord_notifier_id = ord_notifier
self.error_msg = err_msg
self.error_code = err_code
self.operation = operation
if resource_extra_metadata:
self.resource_extra_metadata = ResourceMetaData(
checksum=resource_extra_metadata[0].checksum,
virtual_size=resource_extra_metadata[0].virtual_size,
size=resource_extra_metadata[0].size
)
else:
self.resource_extra_metadata = None
def as_dict(self):
return self.__dict__
class ResourceTemplateModel(object):
def __init__(self,
resource_id,
@ -62,30 +17,6 @@ class ResourceTemplateModel(object):
return self.__dict__
class StatusModel(object):
def __init__(self, status):
self.regions = status
self.status = self._get_aggregated_status()
def _get_aggregated_status(self):
is_pending = False
for region in self.regions:
if region.status == 'Error' and region.operation.strip() != 'delete':
# If a region had an error, the aggregated status is 'Error'
return 'Error'
elif region.status == 'Submitted':
# Just set the flag but don't return, because there might be
# an error in any of the next iterations
is_pending = True
if is_pending:
return 'Pending'
else:
# If self.regions is empty, the result will still be 'Success' but the
# server returns 404 Not Found
return 'Success'
class RegionEndPointData(object):
"""class method endpoints data"""

3
orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py
View File

@ -5,8 +5,9 @@ import time
from oslo_db.sqlalchemy.enginefacade import LegacyEngineFacade
from pecan import conf
from orm.common.orm_common.model.models import ResourceStatusModel, StatusModel
from orm.services.resource_distributor.rds.services.model.region_resource_id_status \
import ResourceStatusModel, StatusModel, RegionEndPointData
import RegionEndPointData
from orm.services.resource_distributor.rds.storage import region_resource_id_status
from sqlalchemy import BigInteger, BLOB, Column, ForeignKey, Integer, String, Text
from sqlalchemy.ext.declarative.api import declarative_base

2
orm/tests/unit/cms/test_customer_logic.py
View File

@ -42,7 +42,7 @@ class ResourceStatusModel(object):
class StatusModel(object):
def __init__(self, status='Success'):
self.regions_status = [ResourceStatusModel()]
self.regions = [ResourceStatusModel()]
self.status = status

118
orm/tests/unit/ims/logic/test_image_logic.py
View File

@ -1,3 +1,4 @@
from orm.common.orm_common.utils import utils
from orm.services.image_manager.ims.logic import image_logic
from orm.services.image_manager.ims.persistency.sql_alchemy.db_models import Image
from orm.services.image_manager.ims.persistency.wsme import models
@ -81,51 +82,46 @@ image_status_dict = {'regions': [{
class TestImageLogic(FunctionalTest):
@mock.patch.object(image_logic, 'di')
def test_get_image_by_uuid_image_not_found(self, mock_di):
mock_rds_proxy = mock.MagicMock()
my_get_image = mock.MagicMock()
my_get_image.get_image_by_id.return_value = None
my_get_record = mock.MagicMock()
my_get_record.get_record.return_value = my_get_image
my_dm = mock.MagicMock(return_value=my_get_record)
_, mock_data_manager = \
get_data_manager_mock(mock_sql_image=None)
mock_di.resolver.unpack.return_value = (mock_data_manager)
mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy
try:
image_logic.get_image_by_uuid('te')
except image_logic.ErrorStatus as e:
self.assertEqual(e.status_code, 404)
@mock.patch.object(utils, 'get_resource_status_from_db')
@mock.patch.object(image_logic.ImsUtils, 'get_server_links',
return_value=["ip", "path"])
@mock.patch.object(image_logic, 'di')
@mock.patch.object(image_logic, 'ImageWrapper')
def test_get_image_by_uuid_image_no_status(self, mock_image,
mock_di, mock_links):
mock_rds_proxy = mock.MagicMock()
mock_rds_proxy.get_status.return_value = RDSGetStatus(status_code=404)
def test_get_image_by_uuid_image_no_status(
self, mock_image, mock_di, mock_links, mock_get_resource):
_, mock_data_manager = get_data_manager_mock()
mock_di.resolver.unpack.return_value = (mock_data_manager)
mock_image.from_db_model.return_value = ImageWrapperTest()
my_get_image = mock.MagicMock()
my_get_record = mock.MagicMock()
my_get_record.get_record.return_value = my_get_image
my_dm = mock.MagicMock(return_value=my_get_record)
mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy
mock_get_resource.return_value = {}
result = image_logic.get_image_by_uuid('test')
self.assertEqual(result.image.status, 'no regions')
@mock.patch.object(utils, 'get_resource_status_from_db')
@mock.patch.object(image_logic.ImsUtils, 'get_server_links',
return_value=["ip", "path"])
@mock.patch.object(image_logic, 'di')
@mock.patch.object(image_logic, 'ImageWrapper')
def test_get_image_by_uuid_image_sanity(self, mock_image,
mock_di, mock_links):
mock_rds_proxy = mock.MagicMock()
mock_rds_proxy.get_status.return_value = RDSGetStatus()
my_get_image = mock.MagicMock()
my_get_record = mock.MagicMock()
my_get_record.get_record.return_value = my_get_image
my_dm = mock.MagicMock(return_value=my_get_record)
def test_get_image_by_uuid_image_sanity(
self, mock_image, mock_di, mock_links, mock_get_resource):
_, mock_data_manager = get_data_manager_mock()
mock_di.resolver.unpack.return_value = (mock_data_manager)
status_model = mock.MagicMock()
status_model.status = 'Success'
resource_status_dict = {'some_id': status_model}
mock_get_resource.return_value = resource_status_dict
mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy
result = image_logic.get_image_by_uuid('test')
self.assertEqual(result.image.status, 'Success')
@ -133,50 +129,50 @@ class TestImageLogic(FunctionalTest):
class TestDeleteImageLogic(FunctionalTest):
"""test delete image."""
@mock.patch.object(image_logic, 'update_region_actions', return_value=True)
@mock.patch.object(utils, 'get_resource_status_from_db')
@mock.patch.object(image_logic, 'di')
def test_delete_image_success(self, mock_di, mock_update_region):
mock_rds_proxy, mock_data_manager = get_data_manager_mock(
def test_delete_image_success(self, mock_di, mock_get_resource):
_, mock_data_manager = get_data_manager_mock(
get_existing_region_names=[])
mock_rds_proxy.get_status.return_value = RdsResponse()
mock_di.resolver.unpack.return_value = (mock_rds_proxy,
mock_data_manager)
global regions
regions = []
mock_di.resolver.unpack.return_value = (mock_data_manager)
status_model = mock.MagicMock()
status_model.status = 'Success'
resource_status_dict = {'some_id': status_model}
mock_get_resource.return_value = resource_status_dict
image_logic.delete_image_by_uuid("image_uuid", "transaction_id")
@mock.patch.object(image_logic, 'update_region_actions', return_value=True)
@mock.patch.object(utils, 'get_resource_status_from_db')
@mock.patch.object(image_logic, 'di')
def test_delete_image_success_nords(self, mock_di, mock_update_region):
mock_rds_proxy, mock_data_manager = \
get_data_manager_mock(imagejson={"regions": {}},
get_existing_region_names=[])
mock_rds_proxy.get_status.return_value = RdsResponse()
mock_di.resolver.unpack.return_value = (mock_rds_proxy,
mock_data_manager)
def test_delete_image_success_nords(self, mock_di, mock_get_resource):
_, mock_data_manager = get_data_manager_mock(
get_existing_region_names=[])
mock_di.resolver.unpack.return_value = (mock_data_manager)
mock_get_resource.return_value = {}
image_logic.delete_image_by_uuid("image_uuid", "transaction_id")
@mock.patch.object(image_logic, 'update_region_actions', return_value=True)
@mock.patch.object(image_logic, 'di')
def test_delete_image_notfound_error(self, mock_di, mock_update_region):
mock_rds_proxy, mock_data_manager = \
def test_delete_image_notfound_error(self, mock_di):
_, mock_data_manager = \
get_data_manager_mock(mock_sql_image=None)
mock_di.resolver.unpack.return_value = (mock_rds_proxy,
mock_data_manager)
mock_di.resolver.unpack.return_value = (mock_data_manager)
try:
image_logic.delete_image_by_uuid("image_uuid", "transaction_id")
except Exception as e:
self.assertEqual(404, e.status_code)
@mock.patch.object(image_logic, 'update_region_actions',
side_effect=ValueError('test'))
@mock.patch.object(image_logic, 'di')
def test_delete_image_other_error(self, mock_di, mock_update_region):
mock_rds_proxy, mock_data_manager = get_data_manager_mock()
mock_di.resolver.unpack.return_value = (mock_rds_proxy,
mock_data_manager)
self.assertRaises(image_logic.ErrorStatus, image_logic.delete_image_by_uuid,
"image_uuid", "transaction_id")
def test_delete_image_other_error(self, mock_di):
_, mock_data_manager = get_data_manager_mock()
mock_di.resolver.unpack.return_value = (mock_data_manager)
self.assertRaises(image_logic.ErrorStatus,
image_logic.delete_image_by_uuid,
"image_uuid",
"transaction_id")
class TestUpdateImage(FunctionalTest):
@ -310,14 +306,13 @@ class TestActivateImageLogic(FunctionalTest):
class TestListImageLogic(FunctionalTest):
@mock.patch.object(image_logic, 'di')
def test_list_image_not_found(self, mock_di):
mock_rds_proxy = mock.MagicMock()
my_get_image = mock.MagicMock()
my_get_image.get_image.return_value = None
my_get_record = mock.MagicMock()
my_get_record.get_record.side_effect = image_logic.ErrorStatus(404, 'a')
my_dm = mock.MagicMock(return_value=my_get_record)
mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy
mock_di.resolver.unpack.return_value = my_dm
try:
image_logic.get_image_list_by_params('a', 'b', 'c')
except image_logic.ErrorStatus as e:
@ -326,13 +321,12 @@ class TestListImageLogic(FunctionalTest):
@mock.patch.object(image_logic, 'di')
@mock.patch.object(image_logic, 'ImageWrapper')
def test_list_image_error(self, mock_image, mock_di):
mock_rds_proxy = mock.MagicMock()
my_get_image = mock.MagicMock()
my_get_record = mock.MagicMock()
my_get_record.get_record.side_effect = SystemError()
my_dm = mock.MagicMock(return_value=my_get_record)
mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy
mock_di.resolver.unpack.return_value = my_dm
try:
image_logic.get_image_list_by_params('a', 'b', 'c')
except Exception as e:
@ -341,17 +335,12 @@ class TestListImageLogic(FunctionalTest):
@mock.patch.object(image_logic, 'di')
@mock.patch.object(image_logic, 'ImageWrapper')
def test_list_image_sanity(self, mock_image, mock_di):
imagejson = [{"regions": {"name": "mdt1"}}]
mock_rds_proxy = mock.MagicMock()
mock_rds_proxy.get_status.return_value = RDSGetStatus()
mock_data_manager = mock.MagicMock()
mock_image_rec = mock.MagicMock()
image_json = mock.MagicMock()
image_json.return_value = imagejson
mock_image_rec.get_images_by_criteria.return_value = Image()
my_dm = mock.MagicMock(mock_data_manager)
mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy
mock_di.resolver.unpack.return_value = my_dm
result = image_logic.get_image_list_by_params('a', 'b', 'c')
self.assertEqual(len(result.images), 0)
@ -688,6 +677,7 @@ def get_data_manager_mock(get_existing_region_names={"name": "mdt1"},
if mock_sql_image:
mock_sql_image = mock.MagicMock()
mock_sql_image.__json__ = image_json
mock_sql_image.id = 'some_id'
mock_sql_image.visibility = visibility
mock_sql_image.protected = protected
mock_sql_image.get_proxy_dict = mock.MagicMock(return_value={'regions': regions})

3
orm/tests/unit/rds/controllers/v1/status/test_get_resource_status.py
View File

@ -1,9 +1,10 @@
"""unittest get resource status."""
from mock import MagicMock
import orm.services.resource_distributor.rds.controllers.v1.status.get_resource as resource
from orm.services.resource_distributor.rds.services.model.region_resource_id_status import ResourceStatusModel, StatusModel
from orm.tests.unit.rds.controllers.v1.functional_test import FunctionalTest
from orm.common.orm_common.model.models import ResourceStatusModel, StatusModel
class EmptyModel(object):
"""mock class."""

23
orm/tests/unit/rds/services/model/test_region_resource_id_status.py
View File

@ -1,12 +1,13 @@
import unittest
from orm.common.orm_common.model import models
from orm.services.resource_distributor.rds.services.model import region_resource_id_status
class TestModel(unittest.TestCase):
def test_model_as_dict(self):
model = region_resource_id_status.ResourceStatusModel(1, 2, 3, 4, 5, 6, 7, 8,
'create')
model = models.ResourceStatusModel(
1, 2, 3, 4, 5, 6, 7, 8, 'create')
expected_dict = {
'timestamp': 1,
'region': 2,
@ -42,19 +43,19 @@ class TestResourceTemplateModel(unittest.TestCase):
class TestStatusModel(unittest.TestCase):
def test_get_aggregated_status_error(self):
model = region_resource_id_status.ResourceStatusModel(1, 2, 'Error', 4, 5, 6, 7, 8,
'create')
status_model = region_resource_id_status.StatusModel([model])
model = models.ResourceStatusModel(
1, 2, 'Error', 4, 5, 6, 7, 8, 'create')
status_model = models.StatusModel([model])
self.assertEqual(status_model.status, 'Error')
def test_get_aggregated_status_pending(self):
model = region_resource_id_status.ResourceStatusModel(1, 2, 'Submitted', 4, 5, 6, 7,
8, 'create')
status_model = region_resource_id_status.StatusModel([model])
model = models.ResourceStatusModel(
1, 2, 'Submitted', 4, 5, 6, 7, 8, 'create')
status_model = models.StatusModel([model])
self.assertEqual(status_model.status, 'Pending')
def test_get_aggregated_status_success(self):
model = region_resource_id_status.ResourceStatusModel(1, 2, 'Success', 4, 5, 6, 7, 8,
'create')
status_model = region_resource_id_status.StatusModel([model])
model = models.ResourceStatusModel(
1, 2, 'Success', 4, 5, 6, 7, 8, 'create')
status_model = models.StatusModel([model])
self.assertEqual(status_model.status, 'Success')

6
orm/tests/unit/rds/services/test_create_resource.py
View File

@ -2,10 +2,10 @@
import unittest
from unittest.mock import patch
from orm.common.orm_common.model.models import (ResourceStatusModel,
ResourceMetaData,
StatusModel)
from orm.services.resource_distributor.rds.services import resource as ResourceService
from orm.services.resource_distributor.rds.services.model.region_resource_id_status import (ResourceStatusModel,
ResourceMetaData,
StatusModel)
result = ResourceStatusModel(
status="success", timestamp="123456789", region="name",