x/ranger
Code Issues Proposed changes

Ranger: Merge multiple databases into one database

Browse Source 
Update Ranger to merge multpiple databases into one.  Also fix the
delete_region and delete_user issues that were discovered .

Change-Id: I0642ee771b18caae81bdb6882caa21ef376c78b6
This commit is contained in:
hosingh000 2018-10-02 07:42:52 -05:00 committed by stewie925
parent d376b60597
commit 287d68523c
21 changed files with 51 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
orm/cmd/db_drop.py
View File

@ -1,21 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2018 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from orm.services import db_cleanup
def main():
db_cleanup.main()

6
orm/services/audit_trail_manager/config.py
View File

@ -8,12 +8,8 @@ server = {
}
# DB configurations
db_url = config.db_connect
database = {
'url': db_url.endswith('/orm') and db_url.replace("/orm", "/orm_audit") or (db_url + 'orm_audit'),
'echo_statements': False
'url': config.db_connect, 'echo_statements': False
}
# Pecan Application Configurations

4
orm/services/audit_trail_manager/scripts/db_scripts/create_db.sql
View File

@ -1,8 +1,6 @@
SET sql_notes=0;
create database if not exists orm_audit;
use orm_audit;
use orm;
create table if not exists transactions(
id integer not null auto_increment,

4
orm/services/customer_manager/cms_rest/data/sql_alchemy/models.py
View File

@ -176,7 +176,7 @@ class CustomerRegion(Base, CMSBaseModel):
__tablename__ = "customer_region"
customer_id = Column(Integer, ForeignKey('customer.id'), primary_key=True, nullable=False)
region_id = Column(Integer, ForeignKey('region.id'), primary_key=True, nullable=False, index=True)
region_id = Column(Integer, ForeignKey('cms_region.id'), primary_key=True, nullable=False, index=True)
customer_region_quotas = relationship("Quota",
uselist=True,
@ -369,7 +369,7 @@ class QuotaFieldDetail(Base, CMSBaseModel):
class Region(Base, CMSBaseModel):
__tablename__ = "region"
__tablename__ = "cms_region"
id = Column(Integer, primary_key=True)
name = Column(String(64), nullable=False, unique=True)

4
orm/services/customer_manager/cms_rest/data/sql_alchemy/region_record.py
View File

@ -13,7 +13,7 @@ class RegionRecord:
# self.setRecordData(self.region)
# self.region.Clear()
self.__TableName = "region"
self.__TableName = "cms_region"
if (session):
self.session = session
@ -37,7 +37,7 @@ class RegionRecord:
raise
def get_region_id_from_name(self, region_name):
result = self.session.connection().scalar("SELECT id from region WHERE name = \"{}\"".format(region_name))
result = self.session.connection().scalar("SELECT id from cms_region WHERE name = \"{}\"".format(region_name))
if result is not None:
return int(result)
return result

21
orm/services/customer_manager/cms_rest/data/sql_alchemy/user_role_record.py
View File

@ -52,14 +52,25 @@ class UserRoleRecord:
region_record = RegionRecord(self.session)
region_id = region_record.get_region_id_from_name(region_id)
if region_id is None:
raise NotFound("region %s is not found" % region_query)
raise NotFound("region {} ".format(region_query))
if isinstance(user_id, basestring):
user_query = user_id
cms_user_record = CmsUserRecord(self.session)
user_id = cms_user_record.get_cms_user_id_from_name(user_id)
if user_id is None:
raise NotFound("user %s is not found" % user_query)
raise NotFound("user {} ".format(user_query))
# additional logic for delete_user only: check if the provided user id
# is associated with the customer and region in cms delete_user request
elif region_id > -1:
user_check = "SELECT DISTINCT user_id from user_role " \
"WHERE customer_id =%d AND region_id =%d " \
"AND user_id =%d" % (customer_id, region_id, user_id)
result = self.session.connection().execute(user_check)
if result.rowcount == 0:
raise NotFound("user {} ".format(user_query))
if region_id == -1:
delete_query = "DELETE ur FROM user_role ur,user_role u " \
@ -67,10 +78,12 @@ class UserRoleRecord:
"and ur.customer_id = u.customer_id and u.region_id =-1 " \
"and ur.customer_id = %d and ur.user_id=%d" % (customer_id, user_id)
else:
# modify query to correctly determine that the provided region user and its role(s)
# do NOT match that of the default user; otherwise delete user is NOT permitted
delete_query = "DELETE ur FROM user_role as ur LEFT JOIN user_role AS u " \
"ON ur.customer_id = u.customer_id and u.user_id=ur.user_id " \
"and u.region_id=-1 where ur.customer_id = %d and ur.region_id= %d " \
"and ur.user_id =%d and ur.role_id !=IFNULL(u.role_id,'')" \
"and u.region_id=-1 and ur.role_id = u.role_id where ur.customer_id = %d " \
"and ur.region_id= %d and ur.user_id =%d and u.role_id IS NULL" \
% (customer_id, region_id, user_id)
result = self.session.connection().execute(delete_query)

4
orm/services/customer_manager/config.py
View File

@ -39,10 +39,8 @@ quotas_default_values = {
}
# DB configurations
db_url = config.db_connect
database = {
'connection_string': db_url.endswith('/orm') and db_url.replace("/orm", "/orm_cms_db") or (db_url + 'orm_cms_db')
'connection_string': config.db_connect
}
api = {

13
orm/services/customer_manager/scripts/db_scripts/ranger_cms_create_db.sql
View File

@ -1,7 +1,6 @@
SET sql_notes=0;
create database if not exists orm_cms_db DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
use orm_cms_db;
use orm;
create table if not exists cms_role
(
@ -16,7 +15,7 @@ create table if not exists cms_user
primary key (id),
unique name_idx (name));
create table if not exists region
create table if not exists cms_region
(
id integer auto_increment not null,
name varchar(64) not null,
@ -52,7 +51,7 @@ create table if not exists customer_region
primary key (customer_id,region_id),
index region_id (region_id),
foreign key (customer_id) REFERENCES `customer` (`id`) ON DELETE CASCADE,
foreign key (region_id) REFERENCES `region` (`id`));
foreign key (region_id) REFERENCES `cms_region` (`id`));
create table if not exists quota
(
@ -60,7 +59,7 @@ create table if not exists quota
customer_id integer not null,
region_id integer not null,
quota_type varchar(64) not null,
foreign key (region_id) references region(id),
foreign key (region_id) references cms_region(id),
primary key (id),
unique quota_type (customer_id,region_id,quota_type),
foreign key (`customer_id`, `region_id`) REFERENCES `customer_region` (`customer_id`, `region_id`) ON DELETE CASCADE ON UPDATE NO ACTION
@ -85,7 +84,7 @@ create table if not exists user_role
primary key (customer_id,region_id,user_id,role_id),
foreign key (customer_id, region_id) REFERENCES customer_region (`customer_id`, `region_id`) ON DELETE CASCADE,
foreign key (customer_id) references customer(id) ON DELETE CASCADE,
foreign key (region_id) references region(id),
foreign key (region_id) references cms_region(id),
foreign key (user_id) references cms_user(id),
foreign key (role_id) references cms_role(id),
index region_id (region_id),
@ -94,4 +93,4 @@ create table if not exists user_role
create or replace view rds_resource_status_view AS
(
SELECT id, resource_id, region, status,
err_code, operation from orm_rds.resource_status);
err_code, operation from resource_status);

16
orm/services/customer_manager/scripts/db_scripts/ranger_cms_update_db.sql
View File

@ -1,6 +1,9 @@
SET sql_notes=0;
USE orm_cms_db;
USE orm;
insert ignore into cms_region(id,name,type) values(-1, "DEFAULT", "single");
DROP PROCEDURE IF EXISTS MoveKeyToQuota;
DELIMITER ;;
CREATE PROCEDURE `MoveKeyToQuota`(p_field_key varchar(64), p_quota varchar(64))
@ -86,14 +89,13 @@ DROP PROCEDURE IF EXISTS add_region_type ;;
CREATE PROCEDURE add_region_type()
BEGIN
UPDATE region set type = "single" where id = -1;
UPDATE cms_region set type = "single" where id = -1;
IF NOT EXISTS( SELECT * FROM region WHERE id=-1) THEN
insert ignore into region(id,name,type) values(-1, "DEFAULT", "single");
IF NOT EXISTS( SELECT * FROM cms_region WHERE id=-1) THEN
insert ignore into cms_region(id,name,type) values(-1, "DEFAULT", "single");
END IF;
END ;;
CALL add_region_type() ;;
DELIMITER ;
CALL add_region_type();

60
orm/services/db_cleanup.py
View File

@ -1,60 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2012 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from sqlalchemy import *
import sys
CONF = cfg.CONF
def main(argv=None):
if argv is None:
argv = sys.argv
cfg.CONF(argv[1:], project='ranger', validate_default_values=True)
orm_database_group = cfg.OptGroup(name='database',
title='Orm Database Options')
OrmDatabaseGroup = [
cfg.StrOpt('connection',
help='The SQLAlchemy connection string to use to connect to '
'the ORM database.',
secret=True),
cfg.IntOpt('max_retries',
default=-1,
help='The maximum number of retries for database connection.')
]
CONF.register_group(orm_database_group)
CONF.register_opts(OrmDatabaseGroup, orm_database_group)
drop_db_stmt = "SET sql_notes = 0;" \
"DROP database orm;" \
"DROP database orm_audit;" \
"DROP database orm_cms_db;" \
"DROP database orm_fms_db;" \
"DROP database orm_rds;" \
"DROP database orm_rms_db;" \
"DROP database orm_uuidgen;"
db_conn_url = CONF.database.connection
db_conn_url = db_conn_url and db_conn_url.replace("mysql+pymysql", "mysql") or ''
engine = create_engine(db_conn_url, echo=False)
conn = engine.connect()
exec_script = conn.execute(drop_db_stmt)
conn.close()

4
orm/services/flavor_manager/config.py
View File

@ -27,10 +27,8 @@ app_module = app['modules'][0]
logging = config.get_log_config(config.fms['log'], server['name'], app_module)
# DB configurations
db_url = config.db_connect
database = {
'connection_string': db_url.endswith('/orm') and db_url.replace("/orm", "/orm_fms_db") or (db_url + 'orm_fms_db')
'connection_string': config.db_connect
}
# this table is for calculating default extra specs needed

5
orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql
View File

@ -1,7 +1,6 @@
SET sql_notes=0;
create database if not exists orm_fms_db DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
use orm_fms_db;
use orm;
#*****
#* MySql script for Creating Table Flavor
@ -110,4 +109,4 @@ create table if not exists flavor_option
create or replace view rds_resource_status_view AS
(
SELECT ID, RESOURCE_ID, REGION,STATUS,
ERR_CODE,OPERATION from orm_rds.resource_status);
ERR_CODE,OPERATION from resource_status);

1
orm/services/id_generator/scripts/db_scripts/db_create.sql
View File

@ -1,6 +1,5 @@
SET sql_notes=0;
CREATE DATABASE if not exists orm;
USE orm;
CREATE TABLE if not exists `uuids` (

4
orm/services/image_manager/config.py
View File

@ -25,10 +25,8 @@ app_module = app['modules'][0]
logging = config.get_log_config(config.ims['log'], server['name'], app_module)
# DB configurations
db_url = config.db_connect
database = {
'connection_string': db_url.endswith('/orm') and db_url.replace("/orm", "/orm_ims_db") or (db_url + 'orm_ims_db')
'connection_string': config.db_connect
}

5
orm/services/image_manager/scripts/db_scripts/create_db.sql
View File

@ -1,7 +1,6 @@
SET sql_notes=0;
create database if not exists orm_ims_db DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
use orm_ims_db;
use orm;
#*****
#* MySql script for Creating Table image
@ -94,4 +93,4 @@ create table if not exists image_customer
create or replace view rds_resource_status_view AS
(
SELECT ID, RESOURCE_ID, REGION,STATUS,
ERR_CODE,OPERATION from orm_rds.resource_status);
ERR_CODE,OPERATION from resource_status);

4
orm/services/region_manager/config.py
View File

@ -40,10 +40,8 @@ region_options = {
}
# DB configurations
db_url = config.db_connect
database = {
'url': db_url.endswith('/orm') and db_url.replace("/orm", "/orm_rms_db") or (db_url + 'orm_rms_db'),
'url': config.db_connect,
'max_retries': 3,
'retries_interval': 10
}

3
orm/services/region_manager/scripts/db_scripts/create_db.sql
View File

@ -1,7 +1,6 @@
SET sql_notes=0;
create database if not exists orm_rms_db DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
use orm_rms_db;
use orm;
create table if not exists rms_groups
(

2
orm/services/region_manager/scripts/db_scripts/insert_test_values.sql
View File

@ -1,4 +1,4 @@
use orm_rms_db;
use orm;
insert into region (region_id,
name,

4
orm/services/resource_distributor/config.py
View File

@ -12,10 +12,8 @@ server = {
}
# DB configurations
db_url = config.db_connect
database = {
'url': db_url.endswith('/orm') and db_url.replace("/orm", "/orm_rds") or (db_url + 'orm_rds')
'url': config.db_connect
}
sot = {

2
orm/services/resource_distributor/rds/proxies/rms_proxy.py
View File

@ -15,7 +15,7 @@ def get_rms_region(region_name):
headers = {
'content-type': 'application/json',
}
rms_server_url = '%s%s/%s' % (
rms_server_url = '%s/%s/%s' % (
conf.rms.base_url, conf.rms.all_regions_path, region_name)
resp = requests.get(rms_server_url, headers=headers,
verify=conf.verify).json()

3
orm/services/resource_distributor/scripts/db_scripts/create_db.sql
View File

@ -1,7 +1,6 @@
SET sql_notes=0;
create database if not exists orm_rds DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
use orm_rds;
use orm;
#*****