x/ranger
Code Issues Proposed changes

implement get list rms region authorization

Browse Source 
Update Ranger logic to pass the X-Auth-Token in the headers when
creating or updating a ranger resource (flavors, customers,
images, and groups).

Update rds logic to generate a token before making the delete
region request.

These changes are necessary as user authentication logic has been
implemented for RMS region creation and updates.

Change-Id: I6419dbbe440d915268d6b16ae166ab32731d9af2
This commit is contained in:
stewie925 2019-06-21 10:35:41 -07:00
parent 97b3de8bce
commit d5ee52f8a2
32 changed files with 365 additions and 828 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
orm/common/client/keystone/keystone_utils/tokens.py
View File

@ -81,7 +81,7 @@ def get_token_user(token, conf, lcp_id=None, keystone_ep=None):
message = 'Received None for both keystone_ep and lcp_id!'
logger.debug(message)
raise ValueError(message)
keystone_ep = _find_keystone_ep(conf.rms_url, lcp_id)
keystone_ep = _find_keystone_ep(conf.rms_url, lcp_id, token)
if keystone_ep is None:
message = 'Keystone EP of LCP %s not found in RMS' % (lcp_id,)
logger.debug(message)
@ -116,11 +116,12 @@ def get_token_user(token, conf, lcp_id=None, keystone_ep=None):
return None
def _find_keystone_ep(rms_url, lcp_name):
def _find_keystone_ep(rms_url, lcp_name, token):
"""Get the Keystone EP from RMS.
:param rms_url: RMS server URL
:param lcp_name: The LCP name
:param token: Token pass by user
:return: Keystone EP (string), None if it was not found
"""
if not rms_url:
@ -131,9 +132,13 @@ def _find_keystone_ep(rms_url, lcp_name):
logger.debug(
'Looking for Keystone EP of LCP {} using RMS URL {}'.format(
lcp_name, rms_url))
headers = {'content-type': 'application/json'}
headers['X-RANGER-Requester'] = "ranger-keystone-util"
headers['X-Auth-Region'] = lcp_name
headers['X-Auth-Token'] = token
response = requests.get('%s/v2/orm/regions?regionname=%s' % (
rms_url, lcp_name, ), verify=_verify)
rms_url, lcp_name, ), headers=headers, verify=_verify)
if response.status_code != OK_CODE:
# The LCP was not found in RMS
logger.debug('Received bad response code from RMS: {}'.format(

12
orm/common/orm_common/policy/policy.py
View File

@ -140,19 +140,19 @@ def authorize(action, request, app_conf, keystone_ep=None):
try:
# Set the service name for Nagios codes
dictator.soft_set('service_name', app_conf.server.name.upper())
user = tokens.get_token_user(token_to_validate, _TOKEN_CONF,
lcp_id, keystone_ep)
request.headers['X-RANGER-Client'] = user.user['name']
request.headers['X-RANGER-Owner'] = user.tenant['id']
request.headers['Keystone-Endpoint'] = user.auth_url
keystone_ep = user.auth_url
except Exception:
except Exception as ex:
user = None
request.headers['X-RANGER-Client'] = 'NA'
logger.exception(
"policy - Failed to get_token_user, using user={}".format(
user))
"policy - Failed to get_token, using endpoint={}".format(
keystone_ep))
raise err_utils.get_error('N/A', status_code=ex.code)
if token_to_validate is not None and lcp_id is not None and str(token_to_validate).strip() != '' and str(lcp_id).strip() != '':
logger.debug('Authorization: enforcing policy on token=[{}], lcp_id=[{}]'.format(token_to_validate, lcp_id))
@ -178,9 +178,7 @@ def authorize(action, request, app_conf, keystone_ep=None):
logger.error('The token is unauthorized according to the policy')
is_permitted = False
except Exception as e:
msg = 'Fail to validate request. due to {}.'.format(e.message)
logger.error(msg)
logger.exception(e)
logger.error('Fail to validate request. due to {}.'.format(e.message))
is_permitted = False
logger.info('Authorize...end')

111
orm/orm_client/ormcli/cli_common.py
View File

@ -13,39 +13,90 @@ class MissingArgumentError(Exception):
pass
def get_keystone_ep(rms_url, region_name):
"""Get the Keystone EP from RMS.
class ConnectionError(Exception):
pass
:param rms_url: RMS server URL
:param region_name: The region name
:return: Keystone EP (string), None if it was not found
"""
class ResponseError(Exception):
pass
def get_token(timeout, args):
headers = {
'Content-Type': 'application/json',
}
url = '%s/v3/auth/tokens'
data = '''
{
"auth":{
"identity":{
"methods":[
"password"
],
"password":{
"user":{
"domain":{
"name":"%s"
},
"name":"%s",
"password":"%s"
}
}
},
"scope":{
"project":{
"name":"%s",
"domain":{
"id":"%s"
}
}
}
}
}'''
for argument in ('tenant_name', 'username', 'password', 'auth_region', 'keystone_auth_url'):
argument_value = getattr(args, argument, None)
if argument_value is not None:
globals()[argument] = argument_value
else:
configuration_value = getattr(config, argument)
if configuration_value:
globals()[argument] = configuration_value
else:
message = ('ERROR: {} for token generation was not supplied. '
'Please use its command-line argument or '
'environment variable.'.format(argument))
print message
raise MissingArgumentError(message)
keystone_ep = args.keystone_auth_url if args.keystone_auth_url else None
if keystone_ep is None:
raise ConnectionError(
'Failed in get_token, keystone endpoint not define')
user_domain = args.user_domain if args.user_domain else 'default'
project_domain = args.project_domain if args.project_domain else 'default'
url = url % (keystone_ep,)
data = data % (user_domain,
username,
password,
tenant_name,
project_domain,)
if args.verbose:
print(
"Getting token:\ntimeout: %d\nheaders: %s\nurl: %s\n" % (
timeout, headers, url))
try:
response = requests.get('%s/v2/orm/regions?regionname=%s' % (
rms_url, region_name, ), verify=config.verify)
except requests.exceptions.ConnectionError as e:
print('Could not connect to RMS, URL: {}'.format(rms_url))
return None
resp = requests.post(url, timeout=timeout, data=data, headers=headers)
if resp.status_code != 201:
raise ResponseError(
'Failed to get token (Reason: {})'.format(
resp.status_code))
return resp.headers['x-subject-token']
if response.status_code != OK_CODE:
print('RMS returned status: {}, content: {}'.format(
response.status_code, response.content))
return None
# RMS returned 200
lcp = response.json()
try:
for endpoint in lcp['regions'][0]['endpoints']:
if endpoint['type'] == 'identity':
return endpoint['publicURL']
except KeyError:
print('Response from RMS came in an unsupported format. '
'Make sure that you are using RMS 3.5')
return None
# Keystone EP not found in the response
print('No identity endpoint was found in the response from RMS')
return None
except Exception as e:
print e.message
raise ConnectionError(e.message)
def pretty_print_json(json_to_print):

74
orm/orm_client/ormcli/cmscli.py
View File

@ -27,6 +27,9 @@ def add_to_parser(service_sub):
parser.add_argument('--auth-region', type=str,
help='Region used for authentication',
default=get_environment_variable('auth-region'))
parser.add_argument('--keystone-auth-url', type=str,
help='keystone-auth-url used for authentication',
default=get_environment_variable('keystone-auth-url'))
parser.add_argument('--tenant-name', type=str,
help='Keystone user tenant name',
default=get_environment_variable('tenant-name'))
@ -604,39 +607,8 @@ def cmd_details(args):
args.userdomain)
def get_token(timeout, args, host):
headers = {
'Content-Type': 'application/json',
}
url = '%s/v3/auth/tokens'
data = '''
{
"auth":{
"identity":{
"methods":[
"password"
],
"password":{
"user":{
"domain":{
"name":"%s"
},
"name":"%s",
"password":"%s"
}
}
},
"scope":{
"project":{
"name":"%s",
"domain":{
"id":"%s"
}
}
}
}
}'''
for argument in ('tenant_name', 'username', 'password', 'auth_region'):
def validate_args(args):
for argument in ('tenant_name', 'username', 'password', 'auth_region', 'keystone_auth_url'):
argument_value = getattr(args, argument, None)
if argument_value is not None:
globals()[argument] = argument_value
@ -651,37 +623,6 @@ def get_token(timeout, args, host):
print message
raise cli_common.MissingArgumentError(message)
keystone_ep = cli_common.get_keystone_ep('{}'.format(host), auth_region)
if keystone_ep is None:
raise ConnectionError(
'Failed in get_token, host: {}, region: {}'.format(host,
auth_region))
user_domain = args.user_domain if args.user_domain else 'default'
project_domain = args.project_domain if args.project_domain else 'default'
url = url % (keystone_ep,)
data = data % (user_domain,
username,
password,
tenant_name,
project_domain,)
if args.verbose:
print(
"Getting token:\ntimeout: %d\nheaders: %s\nurl: %s\n" % (
timeout, headers, url))
try:
resp = requests.post(url, timeout=timeout, data=data, headers=headers)
if resp.status_code != 201:
raise ResponseError(
'Failed to get token (Reason: {})'.format(
resp.status_code))
return resp.headers['x-subject-token']
except Exception as e:
print e.message
raise ConnectionError(e.message)
def get_environment_variable(argument):
# The rules are: all caps, underscores instead of dashes and prefixed
@ -693,8 +634,6 @@ def get_environment_variable(argument):
def run(args):
rms_url = args.rms_base_url if args.rms_base_url else \
base_config.rms['base_url']
host = args.cms_base_url if args.cms_base_url else \
base_config.cms['base_url']
port = args.port if args.port else base_config.cms['port']
@ -707,7 +646,8 @@ def run(args):
auth_token = auth_region = requester = client = ''
else:
try:
auth_token = get_token(timeout, args, rms_url)
validate_args(args)
auth_token = cli_common.get_token(timeout, args)
except Exception:
exit(1)
auth_region = globals()['auth_region']

1
orm/orm_client/ormcli/config.py
View File

@ -6,6 +6,7 @@ tenant_name = config.CONF.keystone_authtoken.project_name
username = config.CONF.keystone_authtoken.username
password = config.CONF.keystone_authtoken.password
auth_region = config.CONF.cli.base_region
keystone_auth_url = ''
rms_base_url = config.rms['base_url']
cms_base_url = config.cms['base_url']
fms_base_url = config.fms['base_url']

75
orm/orm_client/ormcli/fmscli.py
View File

@ -26,6 +26,9 @@ def add_to_parser(service_sub):
parser.add_argument('--auth-region', type=str,
help='Region used for authentication',
default=get_environment_variable('auth-region'))
parser.add_argument('--keystone-auth-url', type=str,
help='keystone-auth-url used for authentication',
default=get_environment_variable('keystone-auth-url'))
parser.add_argument('--tenant-name', type=str,
help='Keystone user tenant name',
default=get_environment_variable('tenant-name'))
@ -292,39 +295,8 @@ def cmd_details(args):
return requests.get, '/%s' % param
def get_token(timeout, args, host):
headers = {
'Content-Type': 'application/json',
}
url = '%s/v3/auth/tokens'
data = '''
{
"auth":{
"identity":{
"methods":[
"password"
],
"password":{
"user":{
"domain":{
"name":"%s"
},
"name":"%s",
"password":"%s"
}
}
},
"scope":{
"project":{
"name":"%s",
"domain":{
"id":"%s"
}
}
}
}
}'''
for argument in ('tenant_name', 'username', 'password', 'auth_region'):
def validate_args(args):
for argument in ('tenant_name', 'username', 'password', 'auth_region', 'keystone_auth_url'):
argument_value = getattr(args, argument, None)
if argument_value is not None:
globals()[argument] = argument_value
@ -339,40 +311,10 @@ def get_token(timeout, args, host):
print message
raise cli_common.MissingArgumentError(message)
keystone_ep = cli_common.get_keystone_ep('{}'.format(host), auth_region)
if keystone_ep is None:
raise ConnectionError(
'Failed in get_token, host: {}, region: {}'.format(host,
auth_region))
user_domain = args.user_domain if args.user_domain else 'default'
project_domain = args.project_domain if args.project_domain else 'default'
url = url % (keystone_ep,)
data = data % (user_domain,
username,
password,
tenant_name,
project_domain,)
if args.verbose:
print(
"Getting token:\ntimeout: %d\nheaders: %s\nurl: %s\n" % (
timeout, headers, url))
try:
resp = requests.post(url, timeout=timeout, data=data, headers=headers)
if resp.status_code != 201:
raise ResponseError(
'Failed to get token (Reason: {})'.format(
resp.status_code))
return resp.headers['x-subject-token']
except Exception as e:
print e.message
raise ConnectionError(e.message)
def get_environment_variable(argument):
# The rules are: all caps, underscores instead of dashes and prefixed
environment_variable = 'RANGER_{}'.format(
argument.replace('-', '_').upper())
@ -380,8 +322,6 @@ def get_environment_variable(argument):
def run(args):
rms_url = args.rms_base_url if args.rms_base_url else \
base_config.rms['base_url']
host = args.fms_base_url if args.fms_base_url else \
base_config.fms['base_url']
@ -395,7 +335,8 @@ def run(args):
auth_token = auth_region = requester = client = ''
else:
try:
auth_token = get_token(timeout, args, rms_url)
validate_args(args)
auth_token = cli_common.get_token(timeout, args)
except Exception:
exit(1)
auth_region = globals()['auth_region']

74
orm/orm_client/ormcli/imscli.py
View File

@ -27,6 +27,9 @@ def add_to_parser(service_sub):
parser.add_argument('--auth-region', type=str,
help='Region used for authentication',
default=get_environment_variable('auth-region'))
parser.add_argument('--keystone-auth-url', type=str,
help='keystone-auth-url used for authentication',
default=get_environment_variable('keystone-auth-url'))
parser.add_argument('--tenant-name', type=str,
help='Keystone user tenant name',
default=get_environment_variable('tenant-name'))
@ -185,39 +188,8 @@ def add_to_parser(service_sub):
parser_delete_customer.add_argument('customerid', type=str, help=h3)
def get_token(timeout, args, host):
headers = {
'Content-Type': 'application/json',
}
url = '%s/v3/auth/tokens'
data = '''
{
"auth":{
"identity":{
"methods":[
"password"
],
"password":{
"user":{
"domain":{
"name":"%s"
},
"name":"%s",
"password":"%s"
}
}
},
"scope":{
"project":{
"name":"%s",
"domain":{
"id":"%s"
}
}
}
}
}'''
for argument in ('tenant_name', 'username', 'password', 'auth_region'):
def validate_args(args):
for argument in ('tenant_name', 'username', 'password', 'auth_region', 'keystone_auth_url'):
argument_value = getattr(args, argument, None)
if argument_value is not None:
globals()[argument] = argument_value
@ -232,38 +204,6 @@ def get_token(timeout, args, host):
print message
raise cli_common.MissingArgumentError(message)
keystone_ep = cli_common.get_keystone_ep(
'{}'.format(host), auth_region)
if keystone_ep is None:
raise ConnectionError(
'Failed in get_token, host: {}, region: {}'.format(host,
auth_region))
user_domain = args.user_domain if args.user_domain else 'default'
project_domain = args.project_domain if args.project_domain else 'default'
url = url % (keystone_ep,)
data = data % (user_domain,
username,
password,
tenant_name,
project_domain,)
if args.verbose:
print(
"Getting token:\ntimeout: %d\nheaders: %s\nurl: %s\n" % (
timeout, headers, url))
try:
resp = requests.post(url, timeout=timeout, data=data, headers=headers)
if resp.status_code != 201:
raise ResponseError(
'Failed to get token (Reason: {})'.format(
resp.status_code))
return resp.headers['x-subject-token']
except Exception as e:
print e.message
raise ConnectionError(e.message)
def preparm(p):
return ('' if len(p) else '?') + ('&' if len(p) else '')
@ -337,7 +277,6 @@ def get_environment_variable(argument):
def run(args):
rms_url = args.rms_base_url if args.rms_base_url else base_config.rms['base_url']
host = args.ims_base_url if args.ims_base_url else base_config.ims['base_url']
port = args.port if args.port else base_config.ims['port']
data = args.datafile.read() if 'datafile' in args else '{}'
@ -349,7 +288,8 @@ def run(args):
auth_token = auth_region = requester = client = ''
else:
try:
auth_token = get_token(timeout, args, rms_url)
validate_args(args)
auth_token = cli_common.get_token(timeout, args)
except Exception:
exit(1)
auth_region = globals()['auth_region']

80
orm/orm_client/ormcli/rmscli.py
View File

@ -25,6 +25,9 @@ def add_to_parser(service_sub):
parser.add_argument('--auth-region', type=str,
help='Region used for authentication',
default=get_environment_variable('auth-region'))
parser.add_argument('--keystone-auth-url', type=str,
help='keystone-auth-url used for authentication',
default=get_environment_variable('keystone-auth-url'))
parser.add_argument('--tenant-name', type=str,
help='Keystone user tenant name',
default=get_environment_variable('tenant-name'))
@ -207,45 +210,12 @@ def add_to_parser(service_sub):
parser_update_status.add_argument('status', type=str, help=h2)
def get_token(timeout, args, host):
headers = {
'Content-Type': 'application/json',
}
url = '%s/v3/auth/tokens'
data = '''
{
"auth":{
"identity":{
"methods":[
"password"
],
"password":{
"user":{
"domain":{
"name":"%s"
},
"name":"%s",
"password":"%s"
}
}
},
"scope":{
"project":{
"name":"%s",
"domain":{
"id":"%s"
}
}
}
}
}'''
for argument in ('tenant_name', 'username', 'password', 'auth_region'):
def validate_args(args):
for argument in ('tenant_name', 'username', 'password', 'auth_region', 'keystone_auth_url'):
argument_value = getattr(args, argument, None)
if argument_value is not None:
globals()[argument] = argument_value
else:
# If it does not exist in the configuration, we would like the
# exception to be raised
configuration_value = getattr(config, argument)
if configuration_value:
globals()[argument] = configuration_value
@ -256,37 +226,6 @@ def get_token(timeout, args, host):
print message
raise cli_common.MissingArgumentError(message)
keystone_ep = cli_common.get_keystone_ep('{}'.format(host), auth_region)
if keystone_ep is None:
raise ConnectionError(
'Failed in get_token, host: {}, region: {}'.format(host,
auth_region))
user_domain = args.user_domain if args.user_domain else 'default'
project_domain = args.project_domain if args.project_domain else 'default'
url = url % (keystone_ep,)
data = data % (user_domain,
username,
password,
tenant_name,
project_domain,)
if args.verbose:
print(
"Getting token:\ntimeout: %d\nheaders: %s\nurl: %s\n" % (
timeout, headers, url))
try:
resp = requests.post(url, timeout=timeout, data=data, headers=headers)
if resp.status_code != 201:
raise ResponseError(
'Failed to get token (Reason: {})'.format(
resp.status_code))
return resp.headers['x-subject-token']
except Exception as e:
print e.message
raise ConnectionError(e.message)
def preparm(p):
return ('' if len(p) else '?') + ('&' if len(p) else '')
@ -381,15 +320,12 @@ def run(args):
timeout = args.timeout if args.timeout else 10
rest_cmd, cmd_url = cmd_details(args)
url = '%s/%s' % (rms_base_url, url_path) + cmd_url
if args.faceless or \
args.subcmd == 'get_region' or \
args.subcmd == 'list_regions' or \
args.subcmd == 'list_groups' or \
args.subcmd == 'get_group':
if args.faceless:
auth_token = auth_region = requester = client = ''
else:
try:
auth_token = get_token(timeout, args, rms_base_url)
validate_args(args)
auth_token = cli_common.get_token(timeout, args)
except Exception:
exit(1)
auth_region = globals()['auth_region']

12
orm/services/customer_manager/cms_rest/rds_proxy.py
View File

@ -83,6 +83,12 @@ class RdsProxy(object):
headers['X-RANGER-Requester'] = request.headers[
'X-RANGER-Requester'] if 'X-RANGER-Requester' in \
request.headers else ''
headers['X-Auth-Region'] = request.headers[
'X-Auth-Region'] if 'X-Auth-Region' in \
request.headers else ''
headers['X-Auth-Token'] = request.headers[
'X-Auth-Token'] if 'X-Auth-Token' in \
request.headers else ''
LOG.debug("Wrapper JSON before sending action: {0} to Rds "
"Proxy\n{1}".format(method, pretty_text))
@ -157,6 +163,12 @@ class RdsProxy(object):
headers['X-RANGER-Requester'] = request.headers[
'X-RANGER-Requester'] if 'X-RANGER-Requester' in \
request.headers else ''
headers['X-Auth-Region'] = request.headers[
'X-Auth-Region'] if 'X-Auth-Region' in \
request.headers else ''
headers['X-Auth-Token'] = request.headers[
'X-Auth-Token'] if 'X-Auth-Token' in \
request.headers else ''
LOG.debug("Wrapper JSON before sending action: {0} to Rds "
"Proxy\n{1}".format(method, pretty_text))

6
orm/services/flavor_manager/fms_rest/proxies/rds_proxy.py
View File

@ -57,6 +57,12 @@ def send_flavor(flavor_dict, transaction_id, action="put"):
headers['X-RANGER-Requester'] = request.headers[
'X-RANGER-Requester'] if 'X-RANGER-Requester' in request.headers else \
''
headers['X-Auth-Region'] = request.headers[
'X-Auth-Region'] if 'X-Auth-Region' in \
request.headers else ''
headers['X-Auth-Token'] = request.headers[
'X-Auth-Token'] if 'X-Auth-Token' in \
request.headers else ''
LOG.debug("Wrapper JSON before sending action: {0} to Rds Proxy {1}".format(action, pretty_text))
LOG.info("Sending to RDS Server: " + conf.api.rds_server.base + conf.api.rds_server.resources)

6
orm/services/image_manager/ims/proxies/rds_proxy.py
View File

@ -57,6 +57,12 @@ def send_image(image_dict, transaction_id, action="put"):
headers['X-RANGER-Requester'] = request.headers[
'X-RANGER-Requester'] if 'X-RANGER-Requester' in request.headers else \
''
headers['X-Auth-Region'] = request.headers[
'X-Auth-Region'] if 'X-Auth-Region' in \
request.headers else ''
headers['X-Auth-Token'] = request.headers[
'X-Auth-Token'] if 'X-Auth-Token' in \
request.headers else ''
LOG.debug("Wrapper JSON before sending action: {0} to Rds Proxy {1}".format(action, pretty_text))
LOG.info("Sending to RDS Server: " + conf.api.rds_server.base + conf.api.rds_server.resources)

4
orm/services/region_manager/rms/controllers/v2/orm/resources/regions.py
View File

@ -249,7 +249,7 @@ class RegionsController(rest.RestController):
:exception: EntityNotFoundError 404
"""
logger.info("Entered Get Regions")
authentication.authorize(request, 'region:get_all', skip_auth=True)
authentication.authorize(request, 'region:get_all')
url_args = {'type': type, 'status': status, 'metadata': metadata,
'ranger_agent_version': ranger_agent_version, 'clli': clli,
@ -285,7 +285,7 @@ class RegionsController(rest.RestController):
def get_one(self, id_or_name):
logger.info(
"API: Entered get region by id or name: {}".format(id_or_name))
authentication.authorize(request, 'region:get_one', skip_auth=True)
authentication.authorize(request, 'region:get_one')
try:
result = RegionService.get_region_by_id_or_name(id_or_name)

4
orm/services/region_manager/rms/storage/my_sql/data_models.py
View File

@ -1,10 +1,8 @@
# coding: utf-8
import datetime
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String
from orm.services.region_manager.rms.model.model import Address, EndPoint, RegionData
from sqlalchemy import Column, ForeignKey, Integer, String
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship

8
orm/services/resource_distributor/rds/ordupdate/ord_notifier.py
View File

@ -35,7 +35,7 @@ class ConfigFileError(Exception):
pass
def _find_correct_ord(url, lcp_name):
def _find_correct_ord(url, lcp_name, headers):
"""Use the Discover API to get the ORD URL.
:param url: Discovery server URL
@ -46,6 +46,7 @@ def _find_correct_ord(url, lcp_name):
# Get the LCP record from RMS
response = requests.get('%s/v2/orm/regions?regionname=%s' % (url,
lcp_name,),
headers=headers,
verify=conf.verify)
if response.status_code != OK_CODE:
return None
@ -203,7 +204,8 @@ def notify_ord(transaction_id,
application_id,
user_id,
external_id=None,
error=False):
error=False,
headers={}):
"""Notify ORD of the changes.
This function should be called after a resource has changed in SoT
@ -253,7 +255,7 @@ def notify_ord(transaction_id,
# Discover the correct ORD
discover_url = '%s:%d' % (conf.ordupdate.discovery_url,
conf.ordupdate.discovery_port,)
ord_to_update = _find_correct_ord(discover_url, region_id)
ord_to_update = _find_correct_ord(discover_url, region_id, headers)
if ord_to_update is None:
message = 'ORD of LCP %s not found' % (region_id, )

25
orm/services/resource_distributor/rds/proxies/rms_proxy.py
View File

@ -1,25 +0,0 @@
"""python module."""
import logging
from pecan import conf
import requests
logger = logging.getLogger(__name__)
def get_rms_region(region_name):
""" function to call rms api for region info
returns 200 for ok and None for error
"""
try:
headers = {
'content-type': 'application/json',
}
rms_server_url = '%s/%s/%s' % (
conf.rms.base_url, conf.rms.all_regions_path, region_name)
resp = requests.get(rms_server_url, headers=headers,
verify=conf.verify).json()
return resp
except Exception as e:
logger.log_exception('Failed in get_rms_region', e)
return None

15
orm/services/resource_distributor/rds/services/model/region_resource_id_status.py
View File

@ -67,3 +67,18 @@ class StatusModel(object):
# If self.regions is empty, the result will still be 'Success' but the
# server returns 404 Not Found
return 'Success'
class RegionEndPointData(object):
"""class method endpoints data"""
def __init__(self, region_id=None, public_url=None, end_point_type=None):
"""init function.
:param public_url: field
:param type: field
:return:
"""
self.region = region_id
self.type = end_point_type
self.public_url = public_url

10
orm/services/resource_distributor/rds/services/resource.py
View File

@ -121,11 +121,19 @@ def _upload_to_sot(uuid, tranid, targetslist):
'X-RANGER-Requester'] if 'X-RANGER-Requester' in request.headers else \
''
sot = sot_factory.get_sot()
headers = {}
headers['X-Auth-Region'] = request.headers[
'X-Auth-Region'] if 'X-Auth-Region' in \
request.headers else ''
headers['X-Auth-Token'] = request.headers[
'X-Auth-Token'] if 'X-Auth-Token' in \
request.headers else ''
sot.save_resource_to_sot(uuid,
tranid,
targetslist,
application_id,
user_id)
user_id,
headers)
def _check_resource_status(input_data):

3
orm/services/resource_distributor/rds/sot/base_sot.py
View File

@ -7,7 +7,8 @@ class BaseSoT(object):
def save_resource_to_sot(self,
tracking_id,
transaction_id,
resource_list):
resource_list,
headers={}):
raise NotImplementedError("Please Implement this method")
def validate_sot_state(self):

11
orm/services/resource_distributor/rds/sot/git_sot/git_sot.py
View File

@ -29,7 +29,8 @@ class GitSoT(base_sot.BaseSoT):
self.git_impl = git_factory.get_git_impl(GitSoT.git_type)
def save_resource_to_sot(self, tracking_id, transaction_id,
resource_list, application_id, user_id):
resource_list, application_id, user_id,
headers={}):
thread = threading.Thread(target=update_sot,
args=(self.git_impl,
lock,
@ -37,7 +38,8 @@ class GitSoT(base_sot.BaseSoT):
transaction_id,
resource_list,
application_id,
user_id))
user_id,
headers))
thread.start()
def validate_sot_state(self):
@ -48,7 +50,7 @@ class GitSoT(base_sot.BaseSoT):
def update_sot(git_impl, my_lock, tracking_id, transaction_id, resource_list,
application_id, user_id):
application_id, user_id, headers={}):
logger.info("Save resource to SoT. start ...")
commit_id = ""
result = False
@ -103,7 +105,8 @@ def update_sot(git_impl, my_lock, tracking_id, transaction_id, resource_list,
application_id, # application_id is not available
user_id, # user_id is not available
"NA", # external_id is not available
not result)
not result,
headers)
except Exception as e:
logger.error("Error in updating ORD! Error: {}".format(
e.message

57
orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py
View File

@ -1,14 +1,17 @@
import logging
import oslo_db
import time
import oslo_db
from oslo_db.sqlalchemy import session as db_session
from pecan import conf
from orm.services.resource_distributor.rds.services.model.region_resource_id_status import Model, StatusModel
from orm.services.resource_distributor.rds.services.model.region_resource_id_status \
import Model, StatusModel, RegionEndPointData
from orm.services.resource_distributor.rds.storage import region_resource_id_status
from sqlalchemy import BigInteger, Column, ForeignKey, Integer, Text
from sqlalchemy import BigInteger, Column, ForeignKey, Integer, String, Text
from sqlalchemy.ext.declarative.api import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy.sql import and_
Base = declarative_base()
logger = logging.getLogger(__name__)
@ -41,6 +44,26 @@ class ImageMetadData(Base):
size = Column(Text, primary_key=False)
class RegionEndPoint(Base):
__tablename__ = 'region_end_point'
region_id = Column(ForeignKey(u'region.region_id'), primary_key=True)
end_point_type = Column(String(64), primary_key=True)
public_url = Column(String(64), nullable=False)
def __json__(self):
return dict(
end_point_type=self.end_point_type,
public_url=self.public_url
)
def to_wsme(self):
region = self.region_id
url = self.public_url
atype = self.end_point_type
return RegionEndPointData(region, url, atype)
class Connection(region_resource_id_status.Base):
""" Implements mysql DB """
@ -127,7 +150,7 @@ class Connection(region_resource_id_status.Base):
def get_records_by_filter_args(self, **filter_args):
logger.debug("Get records filtered by [{}]".format(filter_args))
(timestamp, ref_timestamp) = self.get_timstamp_pair()
(timestamp, ref_timestamp) = self.get_timestamp_pair()
logger.debug("timestamp=%s, ref_timestamp=%s" % (timestamp, ref_timestamp))
records_model = []
session = self._engine_facade.get_session()
@ -168,7 +191,7 @@ class Connection(region_resource_id_status.Base):
logger.debug("Get records filtered by resource_id={} "
"and status={}".format(resource_id,
status))
(timestamp, ref_timestamp) = self.get_timstamp_pair()
(timestamp, ref_timestamp) = self.get_timestamp_pair()
logger.debug("timestamp=%s, ref_timestamp=%s" % (timestamp, ref_timestamp))
session = self._engine_facade.get_session()
records_model = []
@ -200,9 +223,31 @@ class Connection(region_resource_id_status.Base):
logger.debug("No records found")
return None
def get_timstamp_pair(self):
def get_timestamp_pair(self):
timestamp = int(time.time()) * 1000
# assume same time period for all resource types
max_interval_time_in_seconds = conf.region_resource_id_status.max_interval_time.default * 60
ref_timestamp = (int(time.time()) - max_interval_time_in_seconds) * 1000
return timestamp, ref_timestamp
def get_region_keystone_ep(self, region_name):
"""get keystone url from region record """
logger.debug("Get region keystone endpoint: {}".format(region_name))
try:
key_ep = 'identity'
session = self._engine_facade.get_session()
with session.begin():
record = session.query(RegionEndPoint)
record = record.filter(
and_(RegionEndPoint.region_id == region_name,
RegionEndPoint.end_point_type == key_ep))
if record.first():
# return record.first().to_wsme()
return record.first().public_url
return None
except Exception as exp:
logger.exception("DB error RegionEndPoint filtering by region name")
raise

22
orm/services/resource_distributor/rds/utils/authentication.py
View File

@ -3,7 +3,7 @@ import logging
import requests
from orm.common.client.keystone.keystone_utils import tokens
from orm.services.resource_distributor.rds.proxies import rms_proxy as RmsService
from orm.services.resource_distributor.rds.storage import factory
from pecan import conf
@ -34,19 +34,15 @@ def _get_token_conf():
def get_keystone_ep_region_name(region):
# get end point of a region
region_data = RmsService.get_rms_region(region)
if not region_data:
logger.error("fail to get region from rms")
return None
logger.debug("got rms region {} for region name {} ".format(
region, region_data))
keystone_ep = None
for endpoint in region_data['endpoints']:
if endpoint['type'] == 'identity':
keystone_ep = endpoint['publicURL']
break
"""get keystone endpoint of the region """
logger.debug("get data for region %s " % region)
conn = factory.get_region_resource_id_status_connection()
keystone_ep = conn.get_region_keystone_ep(region)
if not keystone_ep:
logger.error("failed to get region from rms")
return None
logger.debug("Got keystone_ep {} for region name {}".format(keystone_ep,
region))
return keystone_ep

13
orm/services/resource_distributor/rds/utils/utils.py
View File

@ -6,7 +6,7 @@ import requests
from orm.services.resource_distributor.rds.proxies import rds_resource_service_proxy
from orm.services.resource_distributor.rds.services.base import ErrorMessage
from pecan import conf
from pecan import conf, request
logger = logging.getLogger(__name__)
@ -37,9 +37,16 @@ def _get_all_rms_regions():
discover_url = '%s:%d' % (conf.ordupdate.discovery_url,
conf.ordupdate.discovery_port,)
# get all regions
headers = {}
headers['X-Auth-Region'] = request.headers[
'X-Auth-Region'] if 'X-Auth-Region' in \
request.headers else ''
headers['X-Auth-Token'] = request.headers[
'X-Auth-Token'] if 'X-Auth-Token' in \
request.headers else ''
# get all regions
response = requests.get('%s/v2/orm/regions' % (discover_url),
verify=conf.verify)
headers=headers, verify=conf.verify)
if response.status_code != 200:
# fail to get regions
error = "got bad response from rms {}".format(response)

7
orm/tests/unit/common/test_policy.py
View File

@ -88,7 +88,9 @@ class TestPolicy(unittest.TestCase):
policy.authorize('a', request, app_conf)
@mock.patch.object(policy, 'enforce')
def test_authorize_no_token(self, mock_enforce):
@mock.patch.object(policy.tokens, 'get_token_user')
def test_authorize_no_token(self, mock_enforce,
mock_get_token_user):
request = mock.MagicMock()
request.headers.get.return_value = None
app_conf = mock.MagicMock()
@ -119,7 +121,8 @@ class TestPolicy(unittest.TestCase):
app_conf)
@mock.patch.object(policy, 'enforce')
def test_authorize_success(self, mock_enforce):
@mock.patch.object(policy.tokens, 'get_token_user')
def test_authorize_success(self, mock_enforce, mock_get_token_user):
request = mock.MagicMock()
request.headers.get.return_value = 'test'
app_conf = mock.MagicMock()

8
orm/tests/unit/common/test_tokens.py
View File

@ -42,27 +42,27 @@ class TokensTest(unittest.TestCase):
tokens.OK_CODE, {'regions': [{'endpoints': [{'publicURL': 'test',
'type': 'identity'}]}]}))
def test_find_keystone_ep_sanity(self, mock_get):
result = tokens._find_keystone_ep('a', 'b')
result = tokens._find_keystone_ep('a', 'b', 'c')
self.assertEqual(result, 'test')
@mock.patch.object(tokens.requests, 'get', return_value=MyResponse(
tokens.OK_CODE + 1, {'regions': [{'endpoints': [
{'publicURL': 'test', 'type': 'identity'}]}]}))
def test_find_keystone_ep_bad_return_code(self, mock_get):
result = tokens._find_keystone_ep('a', 'b')
result = tokens._find_keystone_ep('a', 'b', 'c')
self.assertIsNone(result)
@mock.patch.object(tokens.requests, 'get', return_value=MyResponse(
tokens.OK_CODE, {}))
def test_find_keystone_ep_no_keystone_ep_in_response(self, mock_get):
result = tokens._find_keystone_ep('a', 'b')
result = tokens._find_keystone_ep('a', 'b', 'c')
self.assertIsNone(result)
@mock.patch.object(tokens.requests, 'get', return_value=MyResponse(
tokens.OK_CODE, {'regions': [{'endpoints': [{'publicURL': 'test',
'type': 'test'}]}]}))
def test_find_keystone_ep_no_identity_in_response(self, mock_get):
result = tokens._find_keystone_ep('a', 'b')
result = tokens._find_keystone_ep('a', 'b', 'c')
self.assertIsNone(result)
# @mock.patch.object(tokens.requests, 'get', return_value=MyResponse(

75
orm/tests/unit/ormcli/test_cli_common.py
View File

@ -4,50 +4,41 @@ from unittest import TestCase
class CmsTests(TestCase):
@mock.patch.object(cli_common.requests, 'get')
def test_get_keystone_ep_sanity(self, mock_get):
def respond(self, value, code, headers={}):
response = mock.Mock()
response.json.return_value = value
response.status_code = code
response.headers = headers
return response
@mock.patch.object(cli_common.requests, 'post')
def test_get_token_errors(self, mock_post):
# Bad status code
my_response = mock.MagicMock()
my_response.status_code = cli_common.OK_CODE
my_response.json.return_value = {
'regions': [{'endpoints': [{
'type': 'identity', 'publicURL': 'test'}]}]}
mock_get.return_value = my_response
my_response.status_code = 200
mock_post.return_value = my_response
self.assertRaises(cli_common.ConnectionError, cli_common.get_token,
3, mock.MagicMock())
self.assertEqual(cli_common.get_keystone_ep('a', 'b'), 'test')
# Post fails
mock_post.side_effect = ValueError('test')
self.assertRaises(cli_common.ConnectionError, cli_common.get_token,
3, mock.MagicMock(),)
@mock.patch.object(cli_common.requests, 'get',
side_effect=cli_common.requests.exceptions
.ConnectionError())
def test_get_keystone_ep_connection_failed(self, mock_get):
self.assertIsNone(cli_common.get_keystone_ep('a', 'b'))
@mock.patch.object(cli_common.requests, 'post')
def test_get_token_success(self, mock_post):
args = mock.MagicMock()
@mock.patch.object(cli_common.requests, 'get')
def test_get_keystone_ep_bad_status_code(self, mock_get):
my_response = mock.MagicMock()
my_response.status_code = cli_common.OK_CODE + 1
my_response.json.return_value = {
'regions': [{'endpoints': [{
'type': 'identity', 'publicURL': 'test'}]}]}
mock_get.return_value = my_response
self.assertIsNone(cli_common.get_keystone_ep('a', 'b'))
@mock.patch.object(cli_common.requests, 'get')
def test_get_keystone_ep_bad_response(self, mock_get):
my_response = mock.MagicMock()
my_response.status_code = cli_common.OK_CODE
my_response.json.return_value = {
'regions': [{'endpoinqs': [{
'type': 'identity', 'publicURL': 'test'}]}]}
mock_get.return_value = my_response
self.assertIsNone(cli_common.get_keystone_ep('a', 'b'))
my_response.json.return_value = {
'regions': [{'endpoints': [{
'type': 'identiqy', 'publicURL': 'test'}]}]}
mock_get.return_value = my_response
self.assertIsNone(cli_common.get_keystone_ep('a', 'b'))
args.tenant_name = 'tenant_name'
args.username = 'username'
args.password = 'password'
args.auth_region = 'auth_region'
args.keystone_auth_url = 'url'
# Bad status code
my_response = mock.MagicMock()
my_response.status_code = 201
mock_post.return_value = self.respond(
{"access": {"token": {"id": 989}}}, 201, {"x-subject-token": 989})
self.assertTrue(989, cli_common.get_token(3, args))

140
orm/tests/unit/ormcli/test_cmscli.py
View File

@ -149,61 +149,24 @@ class CmsTests(TestCase):
self.assertEqual(cmd_to_result,
cmscli.cmd_details(args))
@mock.patch.object(cmscli.cli_common, 'get_keystone_ep',
return_value=None)
def test_get_token_keystone_ep_not_found(self, mock_get_keystone_ep):
args = mock.MagicMock()
args.username = 'test'
self.assertRaises(cmscli.ConnectionError, cmscli.get_token,
'a', args, 'c')
@mock.patch.object(cmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(cmscli.requests, 'post')
def test_get_token_errors(self, mock_post, mock_get_keystone_ep):
# Bad status code
my_response = mock.MagicMock()
my_response.status_code = 200
mock_post.return_value = my_response
self.assertRaises(cmscli.ConnectionError, cmscli.get_token,
3, mock.MagicMock(), 'c')
# Post fails
mock_post.side_effect = ValueError('test')
self.assertRaises(cmscli.ConnectionError, cmscli.get_token,
3, mock.MagicMock(), 'c')
@mock.patch.object(cmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(cmscli, 'validate_args')
@mock.patch.object(cmscli.requests, 'post')
@mock.patch.object(cmscli.requests, 'get')
@mock.patch.object(cmscli, 'get_token')
@mock.patch.object(cmscli.cli_common, 'get_token')
@mock.patch.object(cmscli, 'globals')
def test_list_customers(self, mock_globals, mock_get_token,
mock_get, mock_post, mock_get_keystone_ep):
mock_post.return_value = self.respond(TJ, 200)
mock_get, mock_post,
mock_validate_args):
mock_post.return_value = self.respond(TJ, 201)
mock_get.return_value = self.mock_response
args = ormcli.main('orm cms list_customers t'.split())
sys.stdout.seek(0)
output = sys.stdout.read()
self.assertIn(json.dumps(TJ), output)
# @mock.patch.object(cmscli.cli_common, 'get_keystone_ep')
# @mock.patch.object(cmscli.requests, 'post')
# @mock.patch.object(cmscli.requests, 'get')
# @mock.patch.object(cmscli, 'get_token')
# def test_list_customers_a(self, mock_get_token,
# mock_get, mock_post, mock_get_keystone_ep):
# mock_post.return_value = self.respond(TJ, 200)
# mock_get.return_value = self.mock_response
# mock_get.__name__ = 'a'
# args = ormcli.main('orm cms --verbose list_customers t'.split())
# sys.stdout.seek(0)
# output = sys.stdout.read()
# self.assertIn(json.dumps(TJ), output)
@mock.patch.object(cmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(cmscli.requests, 'post')
@mock.patch.object(cmscli.requests, 'get')
def test_list_customers_e(self, mock_get, mock_post, mock_get_keystone_ep):
def test_list_customers_e(self, mock_get, mock_post):
mock_post.return_value = self.respond(TJ, 200)
mock_get.side_effect = Exception('e')
with self.assertRaises(SystemExit) as cm:
@ -213,14 +176,14 @@ class CmsTests(TestCase):
output = sys.stdout.read()
self.assertIn('e', output)
@mock.patch.object(cmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(cmscli, 'validate_args')
@mock.patch.object(cmscli.requests, 'post')
@mock.patch.object(cmscli.requests, 'get')
@mock.patch.object(cmscli, 'get_token')
@mock.patch.object(cmscli.cli_common, 'get_token')
@mock.patch.object(cmscli, 'globals')
def test_list_customers_errors(self, mock_globals, mock_get_token,
mock_get, mock_post,
mock_get_keystone_ep):
mock_validate_args):
mock_post.return_value = self.respond(TJ, 200)
mock_get.return_value = self.respond(TJ, 204, oy=True)
with self.assertRaises(SystemExit) as cm:
@ -230,69 +193,6 @@ class CmsTests(TestCase):
output = sys.stdout.read()
self.assertEqual('', output)
# def test_parsing(self):
# cli = ormcli.Cli()
# cli.create_parser()
# cli.parse(
# 'orm cms --orm-base-url 12.11.10.9 --port 8832 --timeout 150 '
# 'add_user '
# 'client1 customer1 region1 '
# 'ormcli/tests/data/cms-add-cust.json'.split())
# args = cli.args
# self.assertEqual(args.orm_base_url, '12.11.10.9')
# self.assertEqual(args.port, 8832)
# self.assertEqual(args.timeout, 150)
#
# @mock.patch('requests.post')
# def test_timeout(self, mock_post):
# mock_post.side_effect = Exception("timeout boom")
# cli = ormcli.Cli()
# cli.create_parser()
# cli.parse(
# 'orm cms --faceless add_user client1 customer1 region1 '
# 'ormcli/tests/data/cms-add-cust.json'.split())
# with self.assertRaises(SystemExit) as cm:
# cli.logic()
# self.assertEqual(cm.exception.code, 1)
# sys.stdout.seek(0)
# output = sys.stdout.read()
# self.assertIn('timeout boom', output)
#
# @mock.patch('requests.post')
# @mock.patch.object(cmscli, 'get_token')
# def test_no_keystone(self, mock_get_token, mock_post):
# mock_post.side_effect = Exception("timeout boom")
# cli = ormcli.Cli()
# cli.create_parser()
# cli.parse(
# 'orm cms add_user client1 customer1 region1 '
# 'ormcli/tests/data/cms-add-cust.json'.split())
# with self.assertRaises(SystemExit) as cm:
# cli.logic()
#
# @mock.patch('requests.post')
# @mock.patch.object(cmscli, 'get_token')
# def test_response_code(self, mock_get_token, mock_post):
# cli = ormcli.Cli()
# cli.create_parser()
# cli.parse(
# 'orm cms create_customer client1 '
# 'ormcli/tests/data/cms-add-cust.json'.split())
# resp = self.respond({"access": {"token": {"id": 989}}}, 400)
# mock_post.return_value = resp
# with self.assertRaises(SystemExit) as cm:
# cli.logic()
#
# @mock.patch('requests.post')
# def test_ok(self, mock_post):
# cli = ormcli.Cli()
# cli.create_parser()
# cli.parse(
# 'orm cms create_customer client1 '
# 'ormcli/tests/data/cms-add-cust.json'.split())
# mock_post.return_value = self.respond(
# {"access": {"token": {"id": 989}}}, 200)
@mock.patch('requests.get')
@mock.patch('requests.post')
def test_list_customers_with_filters(self, mock_post, mock_get):
@ -304,13 +204,13 @@ class CmsTests(TestCase):
mock_post.return_value = self.respond(
{"access": {"token": {"id": 989}}}, 200)
@mock.patch.object(cmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(cmscli, 'validate_args')
@mock.patch.object(cmscli.requests, 'post')
@mock.patch.object(cmscli.requests, 'get')
@mock.patch.object(cmscli, 'get_token')
@mock.patch.object(cmscli.cli_common, 'get_token')
@mock.patch.object(cmscli, 'globals')
def test_list_groups(self, mock_globals, mock_get_token,
mock_get, mock_post, mock_get_keystone_ep):
mock_get, mock_post, mock_validate_args):
mock_post.return_value = self.respond(TJ, 200)
mock_get.return_value = self.mock_response
args = ormcli.main('orm cms list_groups t'.split())
@ -318,13 +218,13 @@ class CmsTests(TestCase):
output = sys.stdout.read()
self.assertIn(json.dumps(TJ), output)
@mock.patch.object(cmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(cmscli, 'validate_args')
@mock.patch.object(cmscli.requests, 'post')
@mock.patch.object(cmscli.requests, 'get')
@mock.patch.object(cmscli, 'get_token')
@mock.patch.object(cmscli.cli_common, 'get_token')
@mock.patch.object(cmscli, 'globals')
def test_list_groups_a(self, mock_globals, mock_get_token,
mock_get, mock_post, mock_get_keystone_ep):
mock_get, mock_post, mock_validate_args):
mock_post.return_value = self.respond(TJ, 200)
mock_get.return_value = self.mock_response
mock_get.__name__ = 'a'
@ -333,10 +233,10 @@ class CmsTests(TestCase):
output = sys.stdout.read()
self.assertIn(json.dumps(TJ), output)
@mock.patch.object(cmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(cmscli, 'validate_args')
@mock.patch.object(cmscli.requests, 'post')
@mock.patch.object(cmscli.requests, 'get')
def test_list_groups_e(self, mock_get, mock_post, mock_get_keystone_ep):
def test_list_groups_e(self, mock_get, mock_post, mock_validate_args):
mock_post.return_value = self.respond(TJ, 200)
mock_get.side_effect = Exception('e')
with self.assertRaises(SystemExit) as cm:
@ -346,14 +246,14 @@ class CmsTests(TestCase):
output = sys.stdout.read()
self.assertIn('e', output)
@mock.patch.object(cmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(cmscli, 'validate_args')
@mock.patch.object(cmscli.requests, 'post')
@mock.patch.object(cmscli.requests, 'get')
@mock.patch.object(cmscli, 'get_token')
@mock.patch.object(cmscli.cli_common, 'get_token')
@mock.patch.object(cmscli, 'globals')
def test_list_groups_errors(self, mock_globals, mock_get_token,
mock_get, mock_post,
mock_get_keystone_ep):
mock_validate_args):
mock_post.return_value = self.respond(TJ, 200)
mock_get.return_value = self.respond(TJ, 204, oy=True)
with self.assertRaises(SystemExit) as cm:

45
orm/tests/unit/ormcli/test_fmscli.py
View File

@ -89,36 +89,13 @@ class FmsTests(TestCase):
self.assertEqual(subcmd_to_result[subcmd],
fmscli.cmd_details(args))
@mock.patch.object(fmscli.cli_common, 'get_keystone_ep',
return_value=None)
def test_get_token_keystone_ep_not_found(self, mock_get_keystone_ep):
args = mock.MagicMock()
args.username = 'test'
self.assertRaises(fmscli.ConnectionError, fmscli.get_token,
'a', args, 'c')
@mock.patch.object(fmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(fmscli.requests, 'post')
def test_get_token_errors(self, mock_post, mock_get_keystone_ep):
# Bad status code
my_response = mock.MagicMock()
my_response.status_code = 200
mock_post.return_value = my_response
self.assertRaises(fmscli.ConnectionError, fmscli.get_token,
3, mock.MagicMock(), 'c')
# Post fails
mock_post.side_effect = ValueError('test')
self.assertRaises(fmscli.ConnectionError, fmscli.get_token,
3, mock.MagicMock(), 'c')
@mock.patch.object(fmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(fmscli, 'validate_args')
@mock.patch.object(fmscli.requests, 'post')
@mock.patch.object(fmscli.requests, 'get')
@mock.patch.object(fmscli, 'get_token')
@mock.patch.object(fmscli.cli_common, 'get_token')
@mock.patch.object(fmscli, 'globals')
def test_list_flavors(self, mock_globals, mock_get_token,
mock_get, mock_post, mock_get_keystone_ep):
mock_get, mock_post, mock_validate_args):
mock_post.return_value = self.respond(TJ, 200)
mock_get.return_value = self.mock_response
args = ormcli.main('orm fms list_flavors t'.split())
@ -126,13 +103,13 @@ class FmsTests(TestCase):
output = sys.stdout.read()
self.assertIn(json.dumps(TJ), output)
@mock.patch.object(fmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(fmscli, 'validate_args')
@mock.patch.object(fmscli.requests, 'post')
@mock.patch.object(fmscli.requests, 'get')
@mock.patch.object(fmscli, 'get_token')
@mock.patch.object(fmscli.cli_common, 'get_token')
@mock.patch.object(fmscli, 'globals')
def test_list_flavors_a(self, mock_globals, mock_get_token,
mock_get, mock_post, mock_get_keystone_ep):
mock_get, mock_post, mock_validate_args):
mock_post.return_value = self.respond(TJ, 200)
mock_get.return_value = self.mock_response
mock_get.__name__ = 'a'
@ -141,10 +118,10 @@ class FmsTests(TestCase):
output = sys.stdout.read()
self.assertIn(json.dumps(TJ), output)
@mock.patch.object(fmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(fmscli, 'validate_args')
@mock.patch.object(fmscli.requests, 'post')
@mock.patch.object(fmscli.requests, 'get')
def test_list_flavors_e(self, mock_get, mock_post, mock_get_keystone_ep):
def test_list_flavors_e(self, mock_get, mock_post, mock_validate_args):
mock_post.return_value = self.respond(TJ, 200)
mock_get.side_effect = Exception('e')
with self.assertRaises(SystemExit) as cm:
@ -154,14 +131,14 @@ class FmsTests(TestCase):
output = sys.stdout.read()
self.assertIn('e', output)
@mock.patch.object(fmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(fmscli, 'validate_args')
@mock.patch.object(fmscli.requests, 'post')
@mock.patch.object(fmscli.requests, 'get')
@mock.patch.object(fmscli, 'get_token')
@mock.patch.object(fmscli.cli_common, 'get_token')
@mock.patch.object(fmscli, 'globals')
def test_list_flavors_errors(self, mock_globals, mock_get_token,
mock_get, mock_post,
mock_get_keystone_ep):
mock_validate_args):
mock_post.return_value = self.respond(TJ, 200)
mock_get.return_value = self.respond(TJ, 204, oy=True)
with self.assertRaises(SystemExit) as cm:

75
orm/tests/unit/ormcli/test_imscli.py
View File

@ -90,76 +90,6 @@ class ImsTests(TestCase):
self.assertEqual(subcmd_to_result[subcmd],
imscli.cmd_details(args))
@mock.patch.object(imscli.cli_common, 'get_keystone_ep',
return_value=None)
def test_get_token_keystone_ep_not_found(self, mock_get_keystone_ep):
args = mock.MagicMock()
args.username = 'test'
self.assertRaises(imscli.ConnectionError, imscli.get_token,
'a', args, 'c')
@mock.patch.object(imscli.cli_common, 'get_keystone_ep')
@mock.patch.object(imscli.requests, 'post')
def test_get_token_errors(self, mock_post, mock_get_keystone_ep):
# Bad status code
my_response = mock.MagicMock()
my_response.status_code = 200
mock_post.return_value = my_response
self.assertRaises(imscli.ConnectionError, imscli.get_token,
3, mock.MagicMock(), 'c')
# Post fails
mock_post.side_effect = ValueError('test')
self.assertRaises(imscli.ConnectionError, imscli.get_token,
3, mock.MagicMock(), 'c')
# @mock.patch.object(imscli, 'cli_common')
# @mock.patch('requests.put')
# @mock.patch('requests.post')
# @mock.patch.object(imscli, 'get_token')
# @mock.patch.object(imscli, 'globals')
# def test_timeout(self, mock_globals, mock_get_token,
# mock_post, mock_put, mock_common):
# mock_post.side_effect = Exception("timeout boom")
# cli = ormcli.Cli()
# cli.create_parser()
# cli.parse(
# 'orm ims create_image client1 '
# 'ormcli/tests/data/ims-create-image.json'.split())
# with self.assertRaises(SystemExit) as cm:
# cli.logic()
# self.assertEqual(cm.exception.code, 1)
# sys.stdout.seek(0)
# output = sys.stdout.read()
# self.assertIn('timeout boom', output)
#
# @mock.patch('requests.post')
# @mock.patch.object(imscli, 'get_token')
# @mock.patch.object(imscli, 'globals')
# def test_no_keystone(self, mock_globals, mock_get_token, mock_post):
# mock_post.side_effect = Exception("timeout boom")
# cli = ormcli.Cli()
# cli.create_parser()
# globals()['auth_region'] = 'test'
# cli.parse(
# 'orm ims create_image client1 '
# 'ormcli/tests/data/ims-create-image.json'.split())
# with self.assertRaises(SystemExit) as cm:
# cli.logic()
#
# @mock.patch.object(imscli, 'cli_common')
# @mock.patch('requests.post')
# def test_response_code(self, mock_post, mock_common):
# cli = ormcli.Cli()
# cli.create_parser()
# cli.parse(
# 'orm ims create_image client1 '
# 'ormcli/tests/data/ims-create-image.json'.split())
# resp = self.respond({"access": {"token": {"id": 989}}}, 400)
# mock_post.return_value = resp
# with self.assertRaises(SystemExit) as cm:
# cli.logic()
@mock.patch.object(imscli, 'cli_common')
@mock.patch('requests.get')
@mock.patch('requests.post')
@ -176,12 +106,13 @@ class ImsTests(TestCase):
{"access": {"token": {"id": 989}}}, 200)
@mock.patch.object(imscli, 'cli_common')
@mock.patch.object(imscli, 'validate_args')
@mock.patch('requests.get')
@mock.patch('requests.post')
@mock.patch.object(imscli, 'get_token')
@mock.patch.object(imscli.cli_common, 'get_token')
@mock.patch.object(imscli, 'globals')
def test_list_images_bad_request(self, mock_get_token, mock_globals,
mock_post, mock_get, mock_common):
mock_post, mock_get, mock_validate_args, mock_common):
mock_post.return_value = self.respond(
{"access": {"token": {"id": 989}}}, 201)
cli = ormcli.Cli()

148
orm/tests/unit/ormcli/test_rmscli.py
View File

@ -70,36 +70,13 @@ class RmsTests(TestCase):
self.assertEqual(subcmd_to_result[subcmd],
rmscli.cmd_details(args))
@mock.patch.object(rmscli.cli_common, 'get_keystone_ep',
return_value=None)
def test_get_token_keystone_ep_not_found(self, mock_get_keystone_ep):
args = mock.MagicMock()
args.username = 'test'
self.assertRaises(rmscli.ConnectionError, rmscli.get_token,
'a', args, 'c')
@mock.patch.object(rmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(rmscli.requests, 'post')
def test_get_token_errors(self, mock_post, mock_get_keystone_ep):
# Bad status code
my_response = mock.MagicMock()
my_response.status_code = 200
mock_post.return_value = my_response
self.assertRaises(rmscli.ConnectionError, rmscli.get_token,
3, mock.MagicMock(), 'c')
# Post fails
mock_post.side_effect = ValueError('test')
self.assertRaises(rmscli.ConnectionError, rmscli.get_token,
3, mock.MagicMock(), 'c')
@mock.patch.object(rmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(rmscli, 'validate_args')
@mock.patch.object(rmscli.requests, 'post')
@mock.patch.object(rmscli.requests, 'get')
@mock.patch.object(rmscli, 'get_token')
@mock.patch.object(rmscli.cli_common, 'get_token')
@mock.patch.object(rmscli, 'globals')
def test_list_regions(self, mock_globals, mock_get_token,
mock_get, mock_post, mock_get_keystone_ep):
mock_get, mock_post, mock_validate_args):
mock_post.return_value = self.respond(TJ, 200)
mock_get.return_value = self.mock_response
args = ormcli.main('orm rms list_regions t'.split())
@ -107,12 +84,13 @@ class RmsTests(TestCase):
output = sys.stdout.read()
self.assertIn(json.dumps(TJ), output)
@mock.patch.object(rmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(rmscli, 'validate_args')
@mock.patch.object(rmscli.requests, 'post')
@mock.patch.object(rmscli.requests, 'get')
@mock.patch.object(rmscli, 'get_token')
def test_list_regions_a(self, mock_get_token, mock_get,
mock_post, mock_get_keystone_ep):
@mock.patch.object(rmscli.cli_common, 'get_token')
@mock.patch.object(rmscli, 'globals')
def test_list_regions_a(self, mock_globals, mock_get_token, mock_get,
mock_post, mock_validate_args):
mock_post.return_value = self.respond(TJ, 200)
mock_get.return_value = self.mock_response
mock_get.__name__ = 'a'
@ -121,10 +99,10 @@ class RmsTests(TestCase):
output = sys.stdout.read()
self.assertIn(json.dumps(TJ), output)
@mock.patch.object(rmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(rmscli, 'validate_args')
@mock.patch.object(rmscli.requests, 'post')
@mock.patch.object(rmscli.requests, 'get')
def test_list_regions_e(self, mock_get, mock_post, mock_get_keystone_ep):
def test_list_regions_e(self, mock_get, mock_post, mock_validate_args):
mock_post.return_value = self.respond(TJ, 200)
mock_get.side_effect = Exception('e')
with self.assertRaises(SystemExit) as cm:
@ -145,14 +123,14 @@ class RmsTests(TestCase):
mock_post.return_value = self.respond(
{"access": {"token": {"id": 989}}}, 200)
@mock.patch.object(rmscli.cli_common, 'get_keystone_ep')
@mock.patch.object(rmscli, 'validate_args')
@mock.patch.object(rmscli.requests, 'post')
@mock.patch.object(rmscli.requests, 'get')
@mock.patch.object(rmscli, 'get_token')
@mock.patch.object(rmscli.cli_common, 'get_token')
@mock.patch.object(rmscli, 'globals')
def test_list_regions_errors(self, mock_globals, mock_get_token,
mock_get, mock_post,
mock_get_keystone_ep):
mock_validate_args):
mock_post.return_value = self.respond(TJ, 200)
mock_get.return_value = self.respond(TJ, 204)
with self.assertRaises(SystemExit) as cm:
@ -170,56 +148,6 @@ class RmsTests(TestCase):
output = sys.stdout.read()
self.assertIn('API error:', output)
# @mock.patch('requests.post')
# @mock.patch.object(rmscli, 'get_token')
# def test_response_code(self, mock_get_token, mock_post):
# cli = ormcli.Cli()
# cli.create_parser()
# cli.parse(
# 'orm rms create_region client1 '
# 'ormcli/tests/data/rms-create-region.json'.split())
# resp = self.respond({"access": {"token": {"id": 989}}}, 400)
# mock_post.return_value = resp
# with self.assertRaises(SystemExit) as cm:
# cli.logic()
#
# @mock.patch('requests.post')
# def test_ok(self, mock_post):
# cli = ormcli.Cli()
# cli.create_parser()
# cli.parse(
# 'orm rms create_region client1 '
# 'ormcli/tests/data/rms-create-region.json'.split())
# mock_post.return_value = self.respond(
# {"access": {"token": {"id": 989}}}, 200)
#
# def test_parsing(self):
# cli = ormcli.Cli()
# cli.create_parser()
# cli.parse(
# 'orm rms --orm-base-url 12.11.10.9 --port 8832 --timeout 150 '
# 'list_regions --type big '.split())
# args = cli.args
# self.assertEqual(args.orm_base_url, '12.11.10.9')
# self.assertEqual(args.port, 8832)
# self.assertEqual(args.type, 'big')
# self.assertEqual(args.timeout, 150)
#
# @mock.patch('requests.get')
# def test_timeout(self, mock_get):
# cli = ormcli.Cli()
# cli.create_parser()
# cli.parse(
# 'orm rms --faceless --orm-base-url 12.11.10.9 --port 8832'
# ' --timeout 1 get_region x'.split())
# mock_get.side_effect = Exception("timeout boom")
# with self.assertRaises(SystemExit) as cm:
# cli.logic()
# self.assertEqual(cm.exception.code, 1)
# sys.stdout.seek(0)
# output = sys.stdout.read()
# self.assertIn('timeout boom', output)
@mock.patch('requests.get')
def test_one_zone(self, mock_get):
cli = ormcli.Cli()
@ -257,53 +185,3 @@ class RmsTests(TestCase):
sys.stdout.seek(0)
output = sys.stdout.read()
self.assertIn('"ranger_agent_version": "aic3.0"', output)
# def test_error_with_wrong_port(self):
# args = self.parser.parse_args('--port 1111'.split())
# with self.assertRaises(SystemExit) as cm:
# rmscli.rmscli_logic(args)
# self.assertEqual(cm.exception.code, 1)
# sys.stdout.seek(0)
# output = sys.stdout.read()
# self.assertIn('Connection refused', output)
# def test_help_command(self):
# with self.assertRaises(SystemExit) as cm:
# args = self.parser.parse_args(['--help'])
# self.assertEqual(cm.exception.code, 0)
# sys.stdout.seek(0)
# output = sys.stdout.read()
# self.assertIn('usage:', output)
# self.assertIn('timeout', output)
# self.assertIn('optional arguments:', output)
# self.assertIn('--host', output)
# @mock.patch('requests.get')
# def test_timeout(self, mock_get):
# args = self.parser.parse_args('--host 1.1.1.1 --timeout
# 1000'.split())
# mock_get.side_effect = Exception("HTTPConnectionPool(
# host='1.1.1.1', port=8080): Max retries exceeded with url: /lcp (
# Caused by ConnectTimeoutError(
# <requests.packages.urllib3.connection.HTTPConnection object at
# 0x7f9469c1a310>, 'Connection to 1.1.1.1 timed out. (connect
# timeout=1.0)'))")
# with self.assertRaises(SystemExit) as cm:
# rmscli.rmscli_logic(args)
# self.assertEqual(cm.exception.code, 1)
# sys.stdout.seek(0)
# output = sys.stdout.read()
# self.assertIn('ConnectTimeoutError', output)
# learn how to mock 'real' request.get
# @mock.patch('rmscli.rmscli.rmscli.requests.get', autospec=True)
# def test_bad_status(self, mock_get):
# args = self.parser.parse_args([])
# mock_get.return_value = Response({},500)
# with self.assertRaises(SystemExit) as cm:
# rmscli.rmscli_logic(args)
# self.assertEqual(cm.exception.code, 1)
# sys.stdout.seek(0)
# output = sys.stdout.read()
# self.assertIn('GET', output)

8
orm/tests/unit/rds/ordupdate/test_ord_notifier.py
View File

@ -65,7 +65,7 @@ class MainTest(unittest.TestCase):
def test_find_correct_ord_get_failure(self, mock_conf):
ord_notifier.requests.get = mock.MagicMock(
return_value=MyResponse(404, 'test'))
result = ord_notifier._find_correct_ord(None, None)
result = ord_notifier._find_correct_ord(None, None, None)
self.assertIsNone(result)
@mock.patch.object(ord_notifier, 'conf')
@ -75,14 +75,14 @@ class MainTest(unittest.TestCase):
{'regions': [{'endpoints': [
{'publicurl': 'test',
'type': 'test'}]}]}))
result = ord_notifier._find_correct_ord(None, None)
result = ord_notifier._find_correct_ord(None, None, None)
self.assertIsNone(result)
ord_notifier.requests.get = mock.MagicMock(
return_value=MyResponse(ord_notifier.OK_CODE,
{'regions': [{'endqoints': [
{'publicurl': 'test',
'type': 'test'}]}]}))
result = ord_notifier._find_correct_ord(None, None)
result = ord_notifier._find_correct_ord(None, None, None)
self.assertIsNone(result)
@mock.patch.object(ord_notifier, 'conf')
@ -92,7 +92,7 @@ class MainTest(unittest.TestCase):
{'regions': [{'endpoints': [
{'publicURL': 'test',
'type': 'ord'}]}]}))
result = ord_notifier._find_correct_ord(None, 'gigi')
result = ord_notifier._find_correct_ord(None, 'gigi', None)
self.assertEqual('test', result)
@mock.patch.object(ord_notifier, 'conf')

16
orm/tests/unit/rds/storage/mysql/test_region_resource_id_status.py
View File

@ -119,7 +119,7 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase):
@mock.patch.object(region_resource_id_status, 'StatusModel')
@patch.object(region_resource_id_status.Connection,
'get_timstamp_pair',
'get_timestamp_pair',
return_value=(1, 2))
@mock.patch.object(region_resource_id_status, 'Model')
@mock.patch.object(region_resource_id_status.db_session, 'EngineFacade',
@ -134,7 +134,7 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase):
@mock.patch.object(region_resource_id_status, 'StatusModel')
@patch.object(region_resource_id_status.Connection,
'get_timstamp_pair',
'get_timestamp_pair',
return_value=(1, 2))
@mock.patch.object(region_resource_id_status, 'Model')
@mock.patch.object(region_resource_id_status.db_session, 'EngineFacade',
@ -150,7 +150,7 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase):
@mock.patch.object(region_resource_id_status, 'StatusModel')
@patch.object(region_resource_id_status.Connection,
'get_timstamp_pair',
'get_timestamp_pair',
return_value=(1, 2))
@mock.patch.object(region_resource_id_status, 'Model')
@mock.patch.object(region_resource_id_status.db_session, 'EngineFacade',
@ -167,16 +167,16 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase):
return_value=MyFacade())
@patch.object(time, 'time', return_value=80)
@mock.patch.object(region_resource_id_status, 'conf')
def test_get_timstamp_pair_sanity(self, db_session, time_mock, conf_mock):
def test_get_timestamp_pair_sanity(self, db_session, time_mock, conf_mock):
"""Test get_timestamp_pair"""
conf_mock.region_resource_id_status.max_interval_time.default = 1
my_connection = region_resource_id_status.Connection('url')
(timestamp, ref_timestamp) = my_connection.get_timstamp_pair()
(timestamp, ref_timestamp) = my_connection.get_timestamp_pair()
self.assertEqual(timestamp, 80000)
@mock.patch.object(region_resource_id_status, 'StatusModel')
@patch.object(region_resource_id_status.Connection,
'get_timstamp_pair',
'get_timestamp_pair',
return_value=(1, 2))
@mock.patch.object(region_resource_id_status, 'Model')
@mock.patch.object(region_resource_id_status.db_session, 'EngineFacade',
@ -190,7 +190,7 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase):
self.assertIsNone(my_connection.get_records_by_resource_id_and_status('1', '2'))
@mock.patch.object(region_resource_id_status, 'StatusModel')
@patch.object(region_resource_id_status.Connection, 'get_timstamp_pair',
@patch.object(region_resource_id_status.Connection, 'get_timestamp_pair',
return_value=(1, 2))
@mock.patch.object(region_resource_id_status, 'Model')
@mock.patch.object(region_resource_id_status.db_session, 'EngineFacade',
@ -203,7 +203,7 @@ class MysqlRegionResourceIdStatusTest(unittest.TestCase):
my_connection.get_records_by_resource_id_and_status('1', '2')
@mock.patch.object(region_resource_id_status, 'StatusModel')
@patch.object(region_resource_id_status.Connection, 'get_timstamp_pair',
@patch.object(region_resource_id_status.Connection, 'get_timestamp_pair',
return_value=(1, 0))
@mock.patch.object(region_resource_id_status, 'Model')
@mock.patch.object(region_resource_id_status.db_session, 'EngineFacade',

37
ranger-tempest-plugin/ranger_tempest_plugin/services/base_client.py
View File

@ -37,39 +37,10 @@ class RangerClientBase(rest_client.RestClient):
timeout = 10
# def get_keystone_ep(rms_url, region_name):
def get_keystone_ep(self, rms_url, region_name):
"""Get the Keystone EP from RMS.
:param rms_url: RMS server URL
:param region_name: The region name
:return: Keystone EP (string), None if it was not found
def get_keystone_ep(self):
"""Get the Keystone EP from tempest conf.
"""
try:
response = requests.get('%s/v2/orm/regions?regionname=%s' % (
rms_url, region_name, ), verify=CONF.ranger.verify)
except requests.exceptions.ConnectionError as e:
print('Could not connect to RMS, URL: {}'.format(rms_url))
return None
if response.status_code != 200:
print('RMS returned status: {}, content: {}'.format(
response.status_code, response.content))
return None
# get the identity URL info from the rms region record
lcp = response.json()
try:
for endpoint in lcp['regions'][0]['endpoints']:
if endpoint['type'] == 'identity':
return endpoint['publicURL']
except KeyError:
print('Key error while attempting to get keystone endpoint. '
'Please investigate.')
return None
# Keystone EP not found in the response
print('No identity endpoint was found in the response from RMS')
return None
return CONF.identity.uri_v3.strip('/v3')
def get_token(self, timeout, host):
headers = {
@ -108,7 +79,7 @@ class RangerClientBase(rest_client.RestClient):
region = self.auth_region
keystone_ep = self.get_keystone_ep('{}'.format(host), region)
keystone_ep = self.get_keystone_ep()
if keystone_ep is None:
raise ConnectionError(
'Failed in get_token, host: {}, region: {}'.format(host,