Ranger: Merge multiple databases into one database

Update Ranger to merge multpiple databases into one.  Also fix the
delete_region and delete_user issues that were discovered .

Change-Id: I0642ee771b18caae81bdb6882caa21ef376c78b6

orm/cmd/db_drop.py

@@ -1,21 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2018 OpenStack Foundation
-# All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License"); you may
-#  not use this file except in compliance with the License. You may obtain
-#  a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#  License for the specific language governing permissions and limitations
-#  under the License.
-
-from orm.services import db_cleanup
-
-
-def main():
-    db_cleanup.main()

orm/services/audit_trail_manager/config.py

@@ -8,12 +8,8 @@ server = {
 }
 
 # DB configurations
-
-db_url = config.db_connect
-
 database = {
-    'url': db_url.endswith('/orm') and db_url.replace("/orm", "/orm_audit") or (db_url + 'orm_audit'),
+    'url': config.db_connect, 'echo_statements': False
-    'echo_statements': False
 }
 
 # Pecan Application Configurations

orm/services/audit_trail_manager/scripts/db_scripts/create_db.sql

@@ -1,8 +1,6 @@
 SET sql_notes=0;
 
-create database if not exists orm_audit;
+use orm;
-
-use orm_audit;
 
 create table if not exists transactions(
   id integer not null auto_increment,

orm/services/customer_manager/cms_rest/data/sql_alchemy/models.py

@@ -176,7 +176,7 @@ class CustomerRegion(Base, CMSBaseModel):
     __tablename__ = "customer_region"
 
     customer_id = Column(Integer, ForeignKey('customer.id'), primary_key=True, nullable=False)
-    region_id = Column(Integer, ForeignKey('region.id'), primary_key=True, nullable=False, index=True)
+    region_id = Column(Integer, ForeignKey('cms_region.id'), primary_key=True, nullable=False, index=True)
 
     customer_region_quotas = relationship("Quota",
                                           uselist=True,
@@ -369,7 +369,7 @@ class QuotaFieldDetail(Base, CMSBaseModel):
 
 
 class Region(Base, CMSBaseModel):
-    __tablename__ = "region"
+    __tablename__ = "cms_region"
 
     id = Column(Integer, primary_key=True)
     name = Column(String(64), nullable=False, unique=True)

orm/services/customer_manager/cms_rest/data/sql_alchemy/region_record.py

@@ -13,7 +13,7 @@ class RegionRecord:
         # self.setRecordData(self.region)
         # self.region.Clear()
 
-        self.__TableName = "region"
+        self.__TableName = "cms_region"
 
         if (session):
             self.session = session
@@ -37,7 +37,7 @@ class RegionRecord:
             raise
 
     def get_region_id_from_name(self, region_name):
-        result = self.session.connection().scalar("SELECT id from region WHERE name = \"{}\"".format(region_name))
+        result = self.session.connection().scalar("SELECT id from cms_region WHERE name = \"{}\"".format(region_name))
         if result is not None:
             return int(result)
         return result

orm/services/customer_manager/cms_rest/data/sql_alchemy/user_role_record.py

@@ -52,14 +52,25 @@ class UserRoleRecord:
             region_record = RegionRecord(self.session)
             region_id = region_record.get_region_id_from_name(region_id)
             if region_id is None:
-                raise NotFound("region %s is not found" % region_query)
+                raise NotFound("region {} ".format(region_query))
 
         if isinstance(user_id, basestring):
             user_query = user_id
             cms_user_record = CmsUserRecord(self.session)
             user_id = cms_user_record.get_cms_user_id_from_name(user_id)
             if user_id is None:
-                raise NotFound("user %s is not found" % user_query)
+                raise NotFound("user {} ".format(user_query))
+
+            # additional logic for delete_user only: check if the provided user id
+            # is associated with the customer and region in cms delete_user request
+            elif region_id > -1:
+                user_check = "SELECT DISTINCT user_id from user_role " \
+                             "WHERE customer_id =%d AND region_id =%d " \
+                             "AND user_id =%d" % (customer_id, region_id, user_id)
+
+                result = self.session.connection().execute(user_check)
+                if result.rowcount == 0:
+                    raise NotFound("user {} ".format(user_query))
 
         if region_id == -1:
             delete_query = "DELETE ur FROM user_role ur,user_role u " \
@@ -67,10 +78,12 @@ class UserRoleRecord:
                            "and ur.customer_id = u.customer_id and u.region_id =-1 " \
                            "and ur.customer_id = %d and ur.user_id=%d" % (customer_id, user_id)
         else:
+            # modify query to correctly determine that the provided region user and its role(s)
+            # do NOT match that of the default user; otherwise delete user is NOT permitted
             delete_query = "DELETE ur FROM user_role as ur LEFT JOIN user_role AS u " \
                            "ON ur.customer_id = u.customer_id and u.user_id=ur.user_id " \
-                           "and u.region_id=-1 where ur.customer_id = %d and ur.region_id= %d " \
+                           "and u.region_id=-1  and ur.role_id = u.role_id where ur.customer_id = %d " \
-                           "and ur.user_id =%d and ur.role_id !=IFNULL(u.role_id,'')" \
+                           "and ur.region_id= %d and ur.user_id =%d and u.role_id IS NULL" \
                            % (customer_id, region_id, user_id)
 
         result = self.session.connection().execute(delete_query)

orm/services/customer_manager/config.py

@@ -39,10 +39,8 @@ quotas_default_values = {
 }
 
 # DB configurations
-db_url = config.db_connect
-
 database = {
-    'connection_string': db_url.endswith('/orm') and db_url.replace("/orm", "/orm_cms_db") or (db_url + 'orm_cms_db')
+    'connection_string': config.db_connect
 }
 
 api = {

orm/services/customer_manager/scripts/db_scripts/ranger_cms_create_db.sql

@@ -1,7 +1,6 @@
 SET sql_notes=0;
 
-create database if not exists orm_cms_db DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
+use orm;
-use orm_cms_db;
 
 create table if not exists cms_role
    (
@@ -16,7 +15,7 @@ create table if not exists cms_user
 	 primary key (id),
 	 unique name_idx (name));
 
-create table if not exists region
+create table if not exists cms_region
    (
 	 id integer auto_increment not null,
 	 name varchar(64) not null,
@@ -52,7 +51,7 @@ create table if not exists customer_region
 	 primary key (customer_id,region_id),
 	 index region_id (region_id),
   	 foreign key (customer_id) REFERENCES `customer` (`id`) ON DELETE CASCADE,
-	 foreign key (region_id) REFERENCES `region` (`id`));
+	 foreign key (region_id) REFERENCES `cms_region` (`id`));
 
 create table if not exists quota
    (
@@ -60,7 +59,7 @@ create table if not exists quota
 	 customer_id integer not null,
 	 region_id integer not null,
 	 quota_type varchar(64) not null,
-	 foreign key (region_id) references region(id),
+	 foreign key (region_id) references cms_region(id),
 	 primary key (id),
 	 unique quota_type (customer_id,region_id,quota_type),
 	 foreign key (`customer_id`, `region_id`) REFERENCES `customer_region` (`customer_id`, `region_id`) ON DELETE CASCADE ON UPDATE NO ACTION
@@ -85,7 +84,7 @@ create table if not exists user_role
 	 primary key (customer_id,region_id,user_id,role_id),
 	 foreign key (customer_id, region_id) REFERENCES customer_region (`customer_id`, `region_id`) ON DELETE CASCADE,
 	 foreign key (customer_id) references customer(id) ON DELETE CASCADE,
-	 foreign key (region_id) references region(id),
+	 foreign key (region_id) references cms_region(id),
 	 foreign key (user_id) references cms_user(id),
 	 foreign key (role_id) references cms_role(id),
 	 index region_id (region_id),
@@ -94,4 +93,4 @@ create table if not exists user_role
 create or replace view rds_resource_status_view AS
     (
         SELECT id, resource_id, region, status,
-        err_code, operation from orm_rds.resource_status);
+        err_code, operation from resource_status);

orm/services/customer_manager/scripts/db_scripts/ranger_cms_update_db.sql

@@ -1,6 +1,9 @@
 SET sql_notes=0;
 
-USE orm_cms_db;
+USE orm;
+
+insert ignore into cms_region(id,name,type) values(-1, "DEFAULT", "single");
+
 DROP PROCEDURE IF EXISTS MoveKeyToQuota;
 DELIMITER ;;
 CREATE PROCEDURE `MoveKeyToQuota`(p_field_key varchar(64), p_quota varchar(64))
@@ -86,14 +89,13 @@ DROP PROCEDURE IF EXISTS add_region_type ;;
 CREATE PROCEDURE add_region_type()
 BEGIN
 
-  UPDATE region set type = "single" where id = -1;
+  UPDATE cms_region set type = "single" where id = -1;
 
-	IF NOT EXISTS( SELECT * FROM region WHERE id=-1) THEN
+	IF NOT EXISTS( SELECT * FROM cms_region WHERE id=-1) THEN
-  	insert ignore into region(id,name,type) values(-1, "DEFAULT", "single");
+  	insert ignore into cms_region(id,name,type) values(-1, "DEFAULT", "single");
 	END IF;
 
 END ;;
-
-CALL add_region_type() ;;
-
 DELIMITER ;
+
+CALL add_region_type();

orm/services/db_cleanup.py

@@ -1,60 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2012 OpenStack Foundation
-# All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License"); you may
-#  not use this file except in compliance with the License. You may obtain
-#  a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#  License for the specific language governing permissions and limitations
-#  under the License.
-
-from oslo_config import cfg
-from sqlalchemy import *
-import sys
-
-CONF = cfg.CONF
-
-
-def main(argv=None):
-
-    if argv is None:
-        argv = sys.argv
-    cfg.CONF(argv[1:], project='ranger', validate_default_values=True)
-
-    orm_database_group = cfg.OptGroup(name='database',
-                                      title='Orm Database Options')
-    OrmDatabaseGroup = [
-        cfg.StrOpt('connection',
-                   help='The SQLAlchemy connection string to use to connect to '
-                        'the ORM database.',
-                   secret=True),
-        cfg.IntOpt('max_retries',
-                   default=-1,
-                   help='The maximum number of retries for database connection.')
-    ]
-
-    CONF.register_group(orm_database_group)
-    CONF.register_opts(OrmDatabaseGroup, orm_database_group)
-
-    drop_db_stmt = "SET sql_notes = 0;" \
-                   "DROP database orm;" \
-                   "DROP database orm_audit;" \
-                   "DROP database orm_cms_db;" \
-                   "DROP database orm_fms_db;" \
-                   "DROP database orm_rds;" \
-                   "DROP database orm_rms_db;" \
-                   "DROP database orm_uuidgen;"
-
-    db_conn_url = CONF.database.connection
-    db_conn_url = db_conn_url and db_conn_url.replace("mysql+pymysql", "mysql") or ''
-    engine = create_engine(db_conn_url, echo=False)
-
-    conn = engine.connect()
-    exec_script = conn.execute(drop_db_stmt)
-    conn.close()

orm/services/flavor_manager/config.py

@@ -27,10 +27,8 @@ app_module = app['modules'][0]
 logging = config.get_log_config(config.fms['log'], server['name'], app_module)
 
 # DB configurations
-db_url = config.db_connect
-
 database = {
-    'connection_string': db_url.endswith('/orm') and db_url.replace("/orm", "/orm_fms_db") or (db_url + 'orm_fms_db')
+    'connection_string': config.db_connect
 }
 
 # this table is for calculating default extra specs needed

orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql

@@ -1,7 +1,6 @@
 SET sql_notes=0;
 
-create database if not exists orm_fms_db DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
+use orm;
-use orm_fms_db;
 
 #***** 
 #* MySql script for Creating Table Flavor
@@ -110,4 +109,4 @@ create table if not exists flavor_option
 create or replace view rds_resource_status_view AS
     (
       SELECT ID, RESOURCE_ID, REGION,STATUS,
-      ERR_CODE,OPERATION from orm_rds.resource_status);
+      ERR_CODE,OPERATION from resource_status);

orm/services/id_generator/scripts/db_scripts/db_create.sql

@@ -1,6 +1,5 @@
 SET sql_notes=0;
 
-CREATE DATABASE if not exists orm;
 USE orm;
 
 CREATE TABLE if not exists `uuids` (

orm/services/image_manager/config.py

@@ -25,10 +25,8 @@ app_module = app['modules'][0]
 logging = config.get_log_config(config.ims['log'], server['name'], app_module)
 
 # DB configurations
-db_url = config.db_connect
-
 database = {
-    'connection_string': db_url.endswith('/orm') and db_url.replace("/orm", "/orm_ims_db") or (db_url + 'orm_ims_db')
+    'connection_string': config.db_connect
 }
 
 

orm/services/image_manager/scripts/db_scripts/create_db.sql

@@ -1,7 +1,6 @@
 SET sql_notes=0;
 
-create database if not exists orm_ims_db DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
+use orm;
-use orm_ims_db;
 
 #***** 
 #* MySql script for Creating Table image
@@ -94,4 +93,4 @@ create table if not exists image_customer
 create or replace view rds_resource_status_view AS
     (
       SELECT ID, RESOURCE_ID, REGION,STATUS,
-      ERR_CODE,OPERATION from orm_rds.resource_status);
+      ERR_CODE,OPERATION from resource_status);

orm/services/region_manager/config.py

@@ -40,10 +40,8 @@ region_options = {
 }
 
 # DB configurations
-db_url = config.db_connect
-
 database = {
-    'url': db_url.endswith('/orm') and db_url.replace("/orm", "/orm_rms_db") or (db_url + 'orm_rms_db'),
+    'url': config.db_connect,
     'max_retries': 3,
     'retries_interval': 10
 }

orm/services/region_manager/scripts/db_scripts/create_db.sql

@@ -1,7 +1,6 @@
 SET sql_notes=0;
 
-create database if not exists orm_rms_db DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
+use orm;
-use orm_rms_db;
 
 create table if not exists rms_groups
    (

orm/services/region_manager/scripts/db_scripts/insert_test_values.sql

@@ -1,4 +1,4 @@
-use orm_rms_db;
+use orm;
 
 insert into region (region_id,
                     name,

orm/services/resource_distributor/config.py

@@ -12,10 +12,8 @@ server = {
 }
 
 # DB configurations
-db_url = config.db_connect
-
 database = {
-    'url': db_url.endswith('/orm') and db_url.replace("/orm", "/orm_rds") or (db_url + 'orm_rds')
+    'url': config.db_connect
 }
 
 sot = {

orm/services/resource_distributor/rds/proxies/rms_proxy.py

@@ -15,7 +15,7 @@ def get_rms_region(region_name):
         headers = {
             'content-type': 'application/json',
         }
-        rms_server_url = '%s%s/%s' % (
+        rms_server_url = '%s/%s/%s' % (
             conf.rms.base_url, conf.rms.all_regions_path, region_name)
         resp = requests.get(rms_server_url, headers=headers,
                             verify=conf.verify).json()

orm/services/resource_distributor/scripts/db_scripts/create_db.sql

@@ -1,7 +1,6 @@
 SET sql_notes=0;
 
-create database if not exists orm_rds DEFAULT CHARACTER SET utf8 COLLATE utf8_bin;
+use orm;
-use orm_rds;
 
 
 #*****