x/ranger
Code Issues Proposed changes

Add customer's domain attribute in CMS API call

Browse Source 
When CMS API call is made for either create and update customer, a

A new optional customer domain field is introduced for CMS API call
to create or update customer. The customer domain value will be used
by Rds to set the customer and users domain value when generating the
heat template.  In the event customer domain is not provided by the
CMS API call, a default value will be retrieved from the config file.

Change-Id: I58dbae760d56b756b72c5d9eaf7c0b61cf193847
This commit is contained in:
Chi Lo 2020-08-27 15:40:42 -07:00
parent 88b95a0dcb
commit 8366b10bef
14 changed files with 65 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
orm/services/customer_manager/cms_rest/data/data_manager.py
View File

@ -52,7 +52,8 @@ class DataManager(object):
if not connection_string:
connection_string = conf.database.connection_string
self._engine_facade = LegacyEngineFacade(connection_string, autocommit=False)
self._engine_facade = LegacyEngineFacade(connection_string,
autocommit=False)
self._session = None
listen(self.session, 'before_flush', on_before_flush)
self.image_record = None
@ -245,7 +246,8 @@ class DataManager(object):
uuid=uuid,
name=customer.name,
enabled=customer.enabled,
description=customer.description
description=customer.description,
customer_domain=customer.customerDomain
)
self.session.add(sql_customer)

4
orm/services/customer_manager/cms_rest/data/sql_alchemy/models.py
View File

@ -491,6 +491,7 @@ class Customer(Base, CMSBaseModel):
uuid = Column(String(64), nullable=False, unique=True)
name = Column(String(64), nullable=False, unique=True)
description = Column(String(255), nullable=False)
customer_domain = Column(String(64), nullable=False)
enabled = Column(SmallInteger, nullable=False)
customer_customer_regions = relationship("CustomerRegion", cascade="all, delete, delete-orphan")
customer_metadata = relationship("CustomerMetadata", cascade="all, delete, delete-orphan")
@ -501,6 +502,7 @@ class Customer(Base, CMSBaseModel):
uuid=self.uuid,
name=self.name,
description=self.description,
customer_domain=self.customer_domain,
enabled=self.enabled,
customer_customer_regions=[customer_region.__json__() for customer_region in
self.customer_customer_regions],
@ -515,6 +517,7 @@ class Customer(Base, CMSBaseModel):
"uuid": self.uuid,
"name": self.name,
"description": self.description,
"customer_domain": self.customer_domain,
"enabled": 1 if self.enabled else 0
}
@ -558,6 +561,7 @@ class Customer(Base, CMSBaseModel):
name=name,
regions=regions,
users=defaultRegion[0].users if defaultRegion else [],
customerDomain=self.customer_domain,
metadata=metadata,
defaultQuotas=defaultRegion[0].quotas if defaultRegion else [],
custId=self.uuid,

4
orm/services/customer_manager/cms_rest/logic/customer_logic.py
View File

@ -27,6 +27,9 @@ class CustomerLogic(object):
if customer.name.strip() == '':
raise ErrorStatus(400, "Customer Name can not be blank.")
if not customer.customerDomain:
customer.customerDomain = conf.rds.customer_domain
try:
sql_customer = datamanager.add_customer(customer, uuid)
@ -165,7 +168,6 @@ class CustomerLogic(object):
customer.handle_region_group()
sql_customer = self.build_full_customer(customer, uuid, datamanager)
customer_result_wrapper = build_response(uuid, transaction_id, 'create')
if sql_customer.customer_customer_regions and len(sql_customer.customer_customer_regions) > 1:
customer_dict = sql_customer.get_proxy_dict()
for region in customer_dict["regions"]:

6
orm/services/customer_manager/cms_rest/model/Models.py
View File

@ -183,7 +183,7 @@ class User(Model):
"""Create a new compute.
:param id: user id
:param role: roles this use belong to
:param role: roles this user belong to
"""
self.id = id
self.role = role
@ -229,13 +229,14 @@ class Customer(Model):
metadata = wsme.wsattr(wsme.types.DictType(str, str), mandatory=False)
regions = wsme.wsattr([Region], mandatory=False)
users = wsme.wsattr([User], mandatory=True)
customerDomain = wsme.wsattr(str, mandatory=False)
defaultQuotas = wsme.wsattr([Quota], mandatory=True)
status = wsme.wsattr(wsme.types.text, mandatory=False)
custId = wsme.wsattr(wsme.types.text, mandatory=False)
uuid = wsme.wsattr(wsme.types.text, mandatory=False)
def __init__(self, description="", enabled=False, name="", metadata={}, regions=[], users=[],
defaultQuotas=[], status="", custId="", uuid=None):
customerDomain="", defaultQuotas=[], status="", custId="", uuid=None):
"""Create a new Customer.
:param description: Server name
@ -248,6 +249,7 @@ class Customer(Model):
self.metadata = metadata
self.regions = regions
self.users = users
self.customerDomain = customerDomain
self.defaultQuotas = defaultQuotas
self.status = status
self.custId = custId

1
orm/services/customer_manager/scripts/db_scripts/ranger_cms_create_db.sql
View File

@ -30,6 +30,7 @@ create table if not exists customer
uuid varchar(64) not null,
name varchar(64) not null,
description varchar(255) not null,
customer_domain varchar(64) not null,
enabled tinyint not null,
primary key (id),
unique uuid_idx (uuid),

23
orm/services/customer_manager/scripts/db_scripts/ranger_cms_update_db.sql
View File

@ -6,7 +6,6 @@ insert ignore into cms_region(id,name,type) values(-1, "DEFAULT", "single");
insert ignore into cms_domain(id,name) values(1, "default");
DROP PROCEDURE IF EXISTS MoveKeyToQuota;
DELIMITER ;;
CREATE PROCEDURE `MoveKeyToQuota`(p_field_key varchar(64), p_quota varchar(64))
BEGIN
DECLARE bDone INT;
@ -72,8 +71,7 @@ BEGIN
CLOSE security_elements;
COMMIT;
END;;
DELIMITER ;
END;
CALL MoveKeyToQuota('security_groups', 'network');
CALL MoveKeyToQuota('security_group_rules', 'network');
@ -82,11 +80,9 @@ SELECT "==========================" as "";
SELECT q.*, qfd.* FROM quota_field_detail qfd
left join quota q on (q.id = qfd.quota_id) where qfd.field_key like "security%%";
DELIMITER ;;
# account for the old procedure name in DROP PROCEDURE before the name change
DROP PROCEDURE IF EXISTS add_regoion_type ;;
DROP PROCEDURE IF EXISTS add_region_type ;;
DROP PROCEDURE IF EXISTS add_region_type;
CREATE PROCEDURE add_region_type()
BEGIN
@ -96,7 +92,18 @@ BEGIN
insert ignore into cms_region(id,name,type) values(-1, "DEFAULT", "single");
END IF;
END ;;
DELIMITER ;
END;
CALL add_region_type();
DROP PROCEDURE IF EXISTS add_customer_domain;
CREATE PROCEDURE add_customer_domain()
BEGIN
IF NOT EXISTS( (SELECT * FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE()
AND COLUMN_NAME='customer_domain' AND TABLE_NAME='customer') ) THEN
ALTER TABLE customer ADD customer_domain varchar(64);
END IF;
END;
CALL add_customer_domain();

9
orm/services/db_setup.py
View File

@ -36,17 +36,24 @@ def execute_app_custom_sql(conn):
config = ConfigParser()
config.read(CONF.config_file)
# Insert custom domain name into cms_domain.
if config.has_option("rds", "customer_domain"):
customer_domain = config.get("rds", "customer_domain")
customer_domain = re.sub(r'[\'\"]', '', customer_domain).strip()
# Insert custom domain name into cms_domain talbe.
sql = 'insert ignore into cms_domain(name) values(%s)'
conn.execute(sql, (customer_domain, ))
# Update domain_name column in region table
update_regions = "update region set domain_name = %s" \
" where domain_name is NULL or length(domain_name) = 0"
conn.execute(update_regions, customer_domain)
# Update users_domain column in customer table
update_customer_domain = "update customer set customer_domain = %s" \
" where customer_domain is NULL or length(customer_domain) = 0"
conn.execute(update_customer_domain, customer_domain)
def execute_purge_uuids_record(conn):
"""Execute custom SQL statements to purge records from uuids table.

22
orm/services/resource_distributor/rds/controllers/v1/resources/root.py
View File

@ -183,17 +183,12 @@ class CreateNewResource(rest.RestController):
uuid = resource.service_template.tracking.tracking_id
resource_type = resource.service_template.resource.resource_type
base_url = pecan.request.application_url
user_data = {}
if resource_type == 'customer':
user_data['user_domain'] = pecan.request.headers['User-Domain']
user_data['user_id'] = pecan.request.headers['X-RANGER-Client']
jsondata = ast.literal_eval(jsondata)
try:
resource_id = ResourceService.main(jsondata,
uuid,
resource_type,
'create',
user_data)
'create')
site_link = "%s/v1/rds/%s/%s" % (base_url,
resource_type,
resource_id)
@ -229,17 +224,12 @@ class CreateNewResource(rest.RestController):
resource_type = resource.service_template.resource.resource_type
base_url = pecan.request.application_url
jsondata = ast.literal_eval(jsondata)
user_data = {}
if resource_type == 'customer':
user_data['user_domain'] = pecan.request.headers['User-Domain']
user_data['user_id'] = pecan.request.headers['X-RANGER-Client']
try:
resource_id = ResourceService.main(jsondata,
uuid,
resource_type,
'modify',
user_data)
'modify')
my_logger.debug("data sent")
site_link = "%s/v1/rds/%s/%s" % (base_url,
resource_type,
@ -275,10 +265,7 @@ class CreateNewResource(rest.RestController):
jsondata = ast.literal_eval(jsondata)
resource_uuid = resource.service_template.tracking.tracking_id
resource_type = resource.service_template.resource.resource_type
user_data = {}
if resource_type == 'customer':
user_data['user_domain'] = pecan.request.headers['User-Domain']
user_data['user_id'] = pecan.request.headers['X-RANGER-Client']
if resource_type not in resources_operation_list or operation not in \
resources_operation_list[resource_type]:
raise NotAllowedError("delete Not allowed for this"
@ -287,8 +274,7 @@ class CreateNewResource(rest.RestController):
resource_id = ResourceService.main(jsondata,
resource_uuid,
resource_type,
operation,
user_data)
operation)
return resource_id
except ConflictValue as e:
my_logger.error("the request blocked need to wait"

7
orm/services/resource_distributor/rds/services/resource.py
View File

@ -126,8 +126,7 @@ def _create_template_data(input_data):
if target['action'] == "delete":
yamldata = "delete"
elif input_data.resource_type == "customer":
yamldata = yaml_customer_builder.yamlbuilder(jsondata, target,
input_data.user_data)
yamldata = yaml_customer_builder.yamlbuilder(jsondata, target)
elif input_data.resource_type == "group":
yamldata = yaml_group_builder.yamlbuilder(jsondata, target)
elif input_data.resource_type == "flavor":
@ -164,7 +163,6 @@ def _create_template_data(input_data):
targetslist.append({"region_id": target['name'],
"resource_type": input_data.resource_type,
"resource_name": input_data.resource_id,
"resource_domain": target['domain_name'],
"template_data": yamldata,
"operation": target['action']})
return targetslist
@ -264,7 +262,7 @@ def _generate_resource_data(input_data):
# User domain is only used in the case that a customer template is being generated,
# as certain builds of heat require user_domain in order to validate roles
def main(jsondata, external_transaction_id, resource_type, operation, user_data=None):
def main(jsondata, external_transaction_id, resource_type, operation):
"""main function handle resource operation."""
my_logger.info("got %s for %s resource" % (operation, resource_type))
try:
@ -277,7 +275,6 @@ def main(jsondata, external_transaction_id, resource_type, operation, user_data=
my_logger.debug("iterate through the regions see if none in submitted")
_check_resource_status(input_data)
my_logger.debug("get uuid from uuid generator")
input_data.user_data = user_data
input_data.transaction_id = uuid_utils.get_random_uuid()
my_logger.debug("uuid ={}".format(input_data.transaction_id))
# add regions status from rms (to check if it down)

13
orm/services/resource_distributor/rds/services/yaml_customer_builder.py
View File

@ -26,9 +26,9 @@ def _metadata_to_tags(metadata):
str(k) + '=' + str(v) for i in metadata for k, v in i.items()) + ']'
def yamlbuilder(alldata, region, user_data):
def yamlbuilder(alldata, region):
logger.info("building customer yaml")
logger.debug("start building flavor yaml for region %s" % region['name'])
logger.debug("start building customer yaml for region %s" % region['name'])
"""build cstomer yaml.
build yaml file from json
@ -40,8 +40,7 @@ def yamlbuilder(alldata, region, user_data):
resources = {}
yaml_version = conf.yaml_configs.customer_yaml.yaml_version
yaml_type = conf.yaml_configs.customer_yaml.yaml_options.type
domain = region['domain_name']
domain = alldata['customer_domain']
title = {'heat_template_version': yaml_version}
description = {'description': 'yaml file for region - %s' % region['name']}
jsondata = alldata
@ -104,12 +103,6 @@ def yamlbuilder(alldata, region, user_data):
'properties': {'name': user['id'],
'groups': user_group,
'roles': user_roles}}
else:
if user['id'] == user_data['user_id']:
resources['resources'][user['id']] = \
{'type': 'OS::Keystone::UserRoleAssignment\n',
'properties': {'user': (user['id'] + "{%s}" % user_data['user_domain']),
'roles': user_roles}}
else:
resources['resources'][user['id']] = \
{'type': 'OS::Keystone::UserRoleAssignment\n',

5
orm/tests/unit/cms/test_customer_logic.py
View File

@ -85,6 +85,7 @@ class TestCustomerLogic(FunctionalTest):
def test_create_customer_success_with_regions(self):
customer.regions = [models.Region(name="a")]
customer.name = 'Cust Name'
customer.customerDomain = 'domain'
logic = customer_logic.CustomerLogic()
logic.create_customer(customer, 'some_uuid', 'some_trans_id')
@ -103,6 +104,7 @@ class TestCustomerLogic(FunctionalTest):
def test_create_customer_add_all_default_users(self):
customer.name = 'Cust Name'
customer.customerDomain = 'domain'
logic = customer_logic.CustomerLogic()
logic.create_customer(customer, 'some_uuid', 'some_trans_id')
@ -113,6 +115,7 @@ class TestCustomerLogic(FunctionalTest):
mock_returns_error = True
customer.name = 'Cust Name'
customer.customerDomain = 'domain'
logic = customer_logic.CustomerLogic()
self.assertRaises(SystemError, logic.create_customer, customer, 'some_uuid', 'some_trans_id')
@ -130,6 +133,7 @@ class TestCustomerLogic(FunctionalTest):
def test_update_customer_success(self):
customer.name = 'Cust Name'
customer.customerDomain = 'domain'
logic = customer_logic.CustomerLogic()
logic.update_customer(customer, 'some_uuid', 'some_trans_id')
@ -189,6 +193,7 @@ class TestCustomerLogic(FunctionalTest):
default_region = models.Region()
customer.regions = [default_region]
customer.name = 'Cust Name'
customer.customerDomain = 'domain'
default_user = models.User()
default_user.role = ['user', 'admin']

2
orm/tests/unit/rds/services/test_create_resource.py
View File

@ -374,13 +374,11 @@ json_data = {
target_list = [{'region_id': 'regionname',
'resource_type': 'customer',
'resource_name': '1e24981a-fa51-11e5-86aa-5e5517507c66',
'resource_domain': 'default',
'template_data': ['anystring'],
'operation': 'create'},
{'region_id': 'regionnametest',
'resource_name': '1e24981a-fa51-11e5-86aa-5e5517507c66',
'resource_type': 'customer',
'resource_domain': 'default',
'template_data': 'delete',
'operation': 'delete'}]

19
orm/tests/unit/rds/services/test_customer_yaml.py
View File

@ -25,6 +25,7 @@ alldata = {
'network': '111',
'floatingip': '111'}}]},
'description': 'this is a description', 'enabled': 1,
'customer_domain': 'default',
'regions': [{'users': [{'id': 'userId1', 'roles': ['admin', 'other']},
{'id': 'userId2', 'roles': ['storage']}],
'name': 'regionname',
@ -287,8 +288,7 @@ class CreateResource(unittest.TestCase):
ver = mock_conf.yaml_configs.customer_yaml.yaml_version = '2015-1-1'
domain = mock_conf.yaml_configs.customer_yaml.customer_domain = 'default'
mock_conf.yaml_configs.customer_yaml.yaml_options.quotas = False
user_data = {'user_domain': 'default', 'user_id': 'user1'}
yamlfile = CustomerBuild.yamlbuilder(alldata, region_quotas, user_data)
yamlfile = CustomerBuild.yamlbuilder(alldata, region_quotas)
yamlfile_as_json = yaml.safe_load(yamlfile)
self.assertEqual(yamlfile_as_json['heat_template_version'], ver)
self.assertEqual(yaml.safe_load(yamlfile), yaml.safe_load(fullyaml_no_users_quotasoff))
@ -299,8 +299,7 @@ class CreateResource(unittest.TestCase):
ver = mock_conf.yaml_configs.customer_yaml.yaml_version = '2015-1-2'
domain = mock_conf.yaml_configs.customer_yaml.customer_domain = 'default'
mock_conf.yaml_configs.customer_yaml.yaml_options.quotas = False
user_data = {'user_domain': 'default', 'user_id': 'user1'}
yamlfile = CustomerBuild.yamlbuilder(alldata, region_users, user_data)
yamlfile = CustomerBuild.yamlbuilder(alldata, region_users)
yamlfile_as_json = yaml.safe_load(yamlfile)
self.assertEqual(yamlfile_as_json['heat_template_version'], ver)
self.assertEqual(yaml.safe_load(yamlfile), yaml.safe_load(fullyaml_with_users_quotasoff))
@ -311,8 +310,7 @@ class CreateResource(unittest.TestCase):
ver = mock_conf.yaml_configs.customer_yaml.yaml_version = '2015-1-1'
domain = mock_conf.yaml_configs.customer_yaml.customer_domain = 'default'
mock_conf.yaml_configs.customer_yaml.yaml_options.quotas = True
user_data = {'user_domain': 'default', 'user_id': 'user1'}
yamlfile = CustomerBuild.yamlbuilder(alldata, region_users, user_data)
yamlfile = CustomerBuild.yamlbuilder(alldata, region_users)
yamlfile_as_json = yaml.safe_load(yamlfile)
self.assertEqual(yamlfile_as_json['heat_template_version'], ver)
self.assertEqual(yaml.safe_load(yamlfile), yaml.safe_load(full_yaml_default_quotas))
@ -323,8 +321,7 @@ class CreateResource(unittest.TestCase):
ver = mock_conf.yaml_configs.customer_yaml.yaml_version = '2015-1-1'
domain = mock_conf.yaml_configs.customer_yaml.customer_domain = 'default'
mock_conf.yaml_configs.customer_yaml.yaml_options.quotas = True
user_data = {'user_domain': 'default', 'user_id': 'user1'}
yamlfile = CustomerBuild.yamlbuilder(alldata, region_quotas, user_data)
yamlfile = CustomerBuild.yamlbuilder(alldata, region_quotas)
yamlfile_as_json = yaml.safe_load(yamlfile)
self.assertEqual(yamlfile_as_json['heat_template_version'], ver)
self.assertEqual(yaml.safe_load(yamlfile), yaml.safe_load(full_yaml_quotas))
@ -338,8 +335,7 @@ class CreateResource(unittest.TestCase):
mock_conf.authentication.user_domain_name = 'default'
mock_conf.yaml_configs.customer_yaml.yaml_options.quotas = False
mock_conf.yaml_configs.customer_yaml.yaml_options.type = "ldap"
user_data = {'user_domain': 'default', 'user_id': 'user1'}
yamlfile = CustomerBuild.yamlbuilder(alldata, region_users, user_data)
yamlfile = CustomerBuild.yamlbuilder(alldata, region_users)
yamlfile_as_json = yaml.safe_load(yamlfile)
self.assertEqual(yamlfile_as_json['heat_template_version'], ver)
@ -350,8 +346,7 @@ class CreateResource(unittest.TestCase):
ver = mock_conf.yaml_configs.customer_yaml.yaml_version = '2015-1-2'
domain = mock_conf.yaml_configs.customer_yaml.customer_domain = 'default'
mock_conf.yaml_configs.customer_yaml.yaml_options.quotas = True
user_data = {'user_domain': 'default', 'user_id': 'user1'}
yamlfile = CustomerBuild.yamlbuilder(alldata, region_users_v4, user_data)
yamlfile = CustomerBuild.yamlbuilder(alldata, region_users_v4)
yamlfile_as_json = yaml.safe_load(yamlfile)
self.assertEqual(yamlfile_as_json['heat_template_version'], ver)
self.assertEqual(yaml.safe_load(yamlfile), yaml.safe_load(fullyaml_aic4))

2
requirements.txt
View File

@ -16,4 +16,4 @@ keystonemiddleware>=4.17.0 # Apache-2.0
python-keystoneclient>=3.8.0 # Apache-2.0
mysqlclient>=1.4.0
psutil>=3.2.2 # BSD
setuptools>=40.0.0
setuptools<50.0.0,>=40.0.0