x/ranger
Code Issues Proposed changes

update policy.json

Browse Source 
policy.json will be updated to become
monolithic under orm/ directory, maintaining
all previous rules. Also update policy.json to
allow for ranger project as an authorized project
to use for authing through ranger. Also removes
unused git classes.

Change-Id: I547cb601318b507f759c264644c781743673be72
This commit is contained in:
jh629g 2020-01-29 14:23:03 -06:00
parent 7767e6bc7f
commit a41951c6f7
20 changed files with 131 additions and 1125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
etc/policy.json Normal file
View File

@ -0,0 +1,121 @@
{
"default": "!",
"admin": "role:admin and tenant:admin or role:admin and tenant:service or role:admin and tenant:ranger",
"admin_support": "role:admin_support and tenant:admin or role:admin_support and tenant:service or role:admin_support and tenant:ranger",
"admin_viewer": "role:admin_viewer and tenant:admin or role:admin_viewer and tenant:service or role:admin_viewer and tenant:ranger",
"flavor_creator": "role:ranger_flavor_creator and tenant:admin or role:ranger_flavor_creator and tenant:service or role:ranger_flavor_creator and tenant:ranger",
"customer_creator": "role:ranger_customer_creator and tenant:admin or role:ranger_customer_creator and tenant:service or role:ranger_flavor_creator and tenant:ranger",
"admin_or_flavor_creator": "rule:admin or rule:flavor_creator",
"admin_or_customer_creator": "rule:admin or rule:customer_creator",
"admin_or_support": "rule:admin or rule:admin_support",
"admin_or_support_or_flavor_creator": "rule:admin or rule:admin_support or rule:flavor_creator",
"admin_or_support_or_customer_creator": "rule:admin or rule:admin_support or rule:customer_creator",
"admin_or_support_or_viewer": "rule:admin or rule:admin_support or rule:admin_viewer",
"admin_or_support_or_viewer_or_flavor_creator": "rule:admin or rule:admin_support or rule:admin_viewer or rule:flavor_creator",
"admin_or_support_or_viewer_or_customer_creator": "rule:admin or rule:admin_support or rule:admin_viewer or rule:customer_creator",
"flavor:get_one": "rule:admin_or_support_or_viewer_or_flavor_creator",
"flavor:get_all": "rule:admin_or_support_or_viewer_or_flavor_creator",
"flavor:create": "rule:admin_or_support_or_flavor_creator",
"flavor:delete": "rule:admin",
"flavor:get_flavor_extra_specs": "rule:admin_or_support_or_viewer_or_flavor_creator",
"flavor:add_flavor_extra_specs": "rule:admin_or_support_or_flavor_creator",
"flavor:replace_flavor_extra_specs": "rule:admin_or_flavor_creator",
"flavor:delete_flavor_extra_specs": "rule:admin",
"flavor:add_flavor_regions": "rule:admin_or_support_or_flavor_creator",
"flavor:delete_flavor_region": "rule:admin",
"flavor:get_flavor_tags": "rule:admin_or_support_or_viewer_or_flavor_creator",
"flavor:add_flavor_tags": "rule:admin_or_support_or_flavor_creator",
"flavor:replace_flavor_tags": "rule:admin_or_flavor_creator",
"flavor:delete_flavor_tags": "rule:admin",
"flavor:add_flavor_tenants": "rule:admin_or_support_or_flavor_creator",
"flavor:delete_flavor_tenant": "rule:admin",
"lcp:get_one": "",
"lcp:get_all": "",
"region:get_one": "",
"region:get_all": "",
"region:create": "rule:admin_or_support",
"region:update": "rule:admin",
"region:delete": "rule:admin",
"group:get_one": "",
"group:get_all": "",
"group:create": "rule:admin_or_support",
"group:update": "rule:admin",
"group:delete": "rule:admin",
"configuration:get": "rule:admin_or_support_or_viewer",
"log:update": "rule:admin",
"metadata:get": "rule:admin_or_support_or_viewer",
"metadata:create": "rule:admin_or_support",
"metadata:update": "rule:admin",
"metadata:delete": "rule:admin",
"status:put": "rule:admin",
"customers:get_one": "rule:admin_or_support_or_viewer_or_customer_creator",
"customers:get_all": "rule:admin_or_support_or_viewer_or_customer_creator",
"customers:create": "rule:admin_or_support_or_customer_creator",
"customers:update": "rule:admin_or_customer_creator",
"customers:delete": "rule:admin",
"customers:add_region": "rule:admin_or_support_or_customer_creator",
"customers:update_region": "rule:admin_or_customer_creator",
"customers:delete_region": "rule:admin_or_customer_creator",
"customers:add_region_user": "rule:admin_or_support",
"customers:update_region_user": "rule:admin",
"customers:delete_region_user": "rule:admin",
"customers:add_default_user": "rule:admin_or_support",
"customers:update_default_user": "rule:admin",
"customers:delete_default_user": "rule:admin",
"customers:add_metadata": "rule:admin_or_support_or_customer_creator",
"customers:update_metadata": "rule:admin_or_customer_creator",
"customers:enable": "rule:admin_or_support_or_customer_creator",
"groups:get_one": "rule:admin_or_support_or_viewer_or_customer_creator",
"groups:get_all": "rule:admin_or_support_or_viewer_or_customer_creator",
"groups:create": "rule:admin_or_support_or_customer_creator",
"groups:update": "rule:admin_or_customer_creator",
"groups:delete": "rule:admin",
"groups:add_region": "rule:admin_or_support_or_customer_creator",
"groups:delete_region": "rule:admin_or_customer_creator",
"groups:assign_role": "rule:admin_or_support_or_customer_creator",
"groups:assign_region_role": "rule:admin_or_support_or_customer_creator",
"groups:unassign_role": "rule:admin_or_customer_creator",
"groups:add_group_default_users": "rule:admin_or_support",
"groups:delete_group_default_user": "rule:admin",
"groups:add_group_region_users": "rule:admin_or_support",
"groups:delete_group_region_user": "rule:admin",
"groups:get_all_roles": "rule:admin_or_support_or_viewer_or_customer_creator",
"image:create": "rule:admin_or_support",
"image:list": "rule:admin_or_support_or_viewer",
"image:get_one": "rule:admin_or_support_or_viewer",
"image:update": "rule:admin",
"image:delete": "rule:admin",
"image:enable": "rule:admin_or_support",
"region:create": "rule:admin_or_support",
"region:update": "rule:admin",
"region:delete": "rule:admin",
"tenant:create": "rule:admin_or_support",
"tenant:update": "rule:admin",
"tenant:delete": "rule:admin",
"metadata:create": "rule:admin_or_support"
}

43
orm/base_config.py
View File

@ -280,27 +280,9 @@ OrmRdsGroup = [
cfg.PortOpt('port',
default=8777,
help='Rds port.'),
cfg.StrOpt('repo_local_location',
default='/opt/stack/git_repo',
help='Path to repo location.'),
cfg.StrOpt('repo_remote_location',
default='git@127.0.0.1:/home/repo/ORM.git',
help='Repo remote location.'),
cfg.StrOpt('repo_user',
default='orm',
help='Repo user name.'),
cfg.StrOpt('repo_email',
default='orm@test.com',
help='Repo email address.'),
cfg.StrOpt('customer_domain',
default='Default',
help='Keystone project domain'),
cfg.IntOpt('git_retries',
default=5,
help='Maximum number of retries for git operation.'),
cfg.IntOpt('git_retries_interval',
default=10,
help='Wait time in seconds for git retries.'),
cfg.StrOpt('log',
default='rds.log',
help='Rds log name.')
@ -356,14 +338,10 @@ uuid = {'port': CONF.uuid.port,
cms = {'port': CONF.cms.port,
'base_url': '{}://{}:{}/'.format(protocol, orm_host, CONF.cms.port),
'policy_file': CONF.ranger_base +
'/orm/services/customer_manager/cms_rest/etc/policy.json',
'log': '{}/{}'.format(CONF.log_location, CONF.cms.log)}
fms = {'port': CONF.fms.port,
'base_url': '{}://{}:{}/'.format(protocol, orm_host, CONF.fms.port),
'policy_file': CONF.ranger_base +
'/orm/services/flavor_manager/fms_rest/etc/policy.json',
'log': '{}/{}'.format(CONF.log_location, CONF.fms.log)}
audit_server = {'port': CONF.audit.port,
@ -373,41 +351,26 @@ audit_server = {'port': CONF.audit.port,
ims = {'port': CONF.ims.port,
'base_url': '{}://{}:{}/'.format(protocol, orm_host, CONF.ims.port),
'policy_file': CONF.ranger_base +
'/orm/services/image_manager/ims/etc/policy.json',
'log': '{}/{}'.format(CONF.log_location, CONF.ims.log)}
# Had to remove trailing slash from port number in base_url so rms would work.
rms = {'port': CONF.rms.port,
'base_url': '{}://{}:{}'.format(protocol, orm_host, CONF.rms.port),
'policy_file': CONF.ranger_base +
'/orm/services/region_manager/rms/etc/policy.json',
'log': '{}/{}'.format(CONF.log_location, CONF.rms.log)}
rds = {'port': CONF.rds.port,
'token_version': CONF.keystone_authtoken.auth_version,
'repo_local_location': CONF.rds.repo_local_location,
'repo_remote_location': CONF.rds.repo_remote_location,
'repo_user': CONF.rds.repo_user,
'repo_email': CONF.rds.repo_email,
'customer_domain': CONF.rds.customer_domain,
'base_url': '{}://{}:{}/'.format(protocol, orm_host, CONF.rds.port),
'git_retries': CONF.rds.git_retries,
'git_retries_interval': CONF.rds.git_retries_interval,
'log': '{}/{}'.format(CONF.log_location, CONF.rds.log)}
cli = {'base_region': CONF.cli.base_region}
def server_request_auth(server_name):
if server_name == 'cms':
policy_name = cms['policy_file']
elif server_name == 'fms':
policy_name = fms['policy_file']
elif server_name == 'ims':
policy_name = ims['policy_file']
elif server_name == 'rms':
policy_name = rms['policy_file']
if server_name[1:] == 'ms':
# cms, fms, ims, rms
policy_name = CONF.ranger_base + '/etc/policy.json'
else:
# policy_N/A for services 'rds', 'audit', and 'uuid'
policy_name = None

21
orm/common/orm_common/policy/qolicy.py
View File

@ -220,7 +220,6 @@ import os
from . import _parser
from orm.common.orm_common.utils import api_error_utils as err_utils
from oslo_config import cfg
from oslo_policy import _checks
from oslo_policy._i18n import _
@ -463,26 +462,6 @@ class Enforcer(object):
for policy_file in [p for p in policy_files if not p.startswith('.')]:
func(os.path.join(path, policy_file), *args)
def _get_policy_path(self, path):
"""Locate the policy JSON data file/path.
:param path: It's value can be a full path or related path. When
full path specified, this function just returns the full
path. When related path specified, this function will
search configuration directories to find one that exists.
:returns: The policy path
:raises: ConfigFilesNotFoundError if the file/path couldn't
be located.
"""
policy_path = self.conf.find_file(path)
if policy_path:
return policy_path
raise cfg.ConfigFilesNotFoundError((path,))
def enforce(self, rule, target, creds, do_raise=False,
exc=None, *args, **kwargs):
"""Checks authorization of a rule against the target and credentials.

55
orm/services/customer_manager/cms_rest/etc/policy.json
View File

@ -1,55 +0,0 @@
{
"default": "!",
"admin": "role:admin and tenant:admin or role:admin and tenant:service",
"admin_support": "role:admin_support and tenant:admin or role:admin_support and tenant:service",
"admin_viewer": "role:admin_viewer and tenant:admin or role:admin_viewer and tenant:service",
"creator": "role:ranger_customer_creator and tenant:admin or role:ranger_customer_creator and tenant:service",
"admin_or_creator": "rule:admin or rule:creator",
"admin_or_support": "rule:admin or rule:admin_support",
"admin_or_support_or_creator": "rule:admin or rule:admin_support or rule:creator",
"admin_or_support_or_viewer": "rule:admin or rule:admin_support or rule:admin_viewer",
"admin_or_support_or_viewer_or_creator": "rule:admin or rule:admin_support or rule:admin_viewer or rule:creator",
"customers:get_one": "rule:admin_or_support_or_viewer_or_creator",
"customers:get_all": "rule:admin_or_support_or_viewer_or_creator",
"customers:create": "rule:admin_or_support_or_creator",
"customers:update": "rule:admin_or_creator",
"customers:delete": "rule:admin",
"customers:add_region": "rule:admin_or_support_or_creator",
"customers:update_region": "rule:admin_or_creator",
"customers:delete_region": "rule:admin_or_creator",
"customers:add_region_user": "rule:admin_or_support",
"customers:update_region_user": "rule:admin",
"customers:delete_region_user": "rule:admin",
"customers:add_default_user": "rule:admin_or_support",
"customers:update_default_user": "rule:admin",
"customers:delete_default_user": "rule:admin",
"customers:add_metadata": "rule:admin_or_support_or_creator",
"customers:update_metadata": "rule:admin_or_creator",
"customers:enable": "rule:admin_or_support_or_creator",
"groups:get_one": "rule:admin_or_support_or_viewer_or_creator",
"groups:get_all": "rule:admin_or_support_or_viewer_or_creator",
"groups:create": "rule:admin_or_support_or_creator",
"groups:update": "rule:admin_or_creator",
"groups:delete": "rule:admin",
"groups:add_region": "rule:admin_or_support_or_creator",
"groups:delete_region": "rule:admin_or_creator",
"groups:assign_role": "rule:admin_or_support_or_creator",
"groups:assign_region_role": "rule:admin_or_support_or_creator",
"groups:unassign_role": "rule:admin_or_creator",
"groups:add_group_default_users": "rule:admin_or_support",
"groups:delete_group_default_user": "rule:admin",
"groups:add_group_region_users": "rule:admin_or_support",
"groups:delete_group_region_user": "rule:admin",
"groups:get_all_roles": "rule:admin_or_support_or_viewer_or_creator"
}

38
orm/services/flavor_manager/fms_rest/etc/policy.json
View File

@ -1,38 +0,0 @@
{
"default": "!",
"admin": "role:admin and tenant:admin or role:admin and tenant:service",
"admin_support": "role:admin_support and tenant:admin or role:admin_support and tenant:service",
"admin_viewer": "role:admin_viewer and tenant:admin or role:admin_viewer and tenant:service",
"creator": "role:ranger_flavor_creator and tenant:admin or role:ranger_flavor_creator and tenant:service",
"admin_or_creator": "rule:admin or rule:creator",
"admin_or_support": "rule:admin or rule:admin_support",
"admin_or_support_or_creator": "rule:admin or rule:admin_support or rule:creator",
"admin_or_support_or_viewer": "rule:admin or rule:admin_support or rule:admin_viewer",
"admin_or_support_or_viewer_or_creator": "rule:admin or rule:admin_support or rule:admin_viewer or rule:creator",
"flavor:get_one": "rule:admin_or_support_or_viewer_or_creator",
"flavor:get_all": "rule:admin_or_support_or_viewer_or_creator",
"flavor:create": "rule:admin_or_support_or_creator",
"flavor:delete": "rule:admin",
"flavor:get_flavor_extra_specs": "rule:admin_or_support_or_viewer_or_creator",
"flavor:add_flavor_extra_specs": "rule:admin_or_support_or_creator",
"flavor:replace_flavor_extra_specs": "rule:admin_or_creator",
"flavor:delete_flavor_extra_specs": "rule:admin",
"flavor:add_flavor_regions": "rule:admin_or_support_or_creator",
"flavor:delete_flavor_region": "rule:admin",
"flavor:get_flavor_tags": "rule:admin_or_support_or_viewer_or_creator",
"flavor:add_flavor_tags": "rule:admin_or_support_or_creator",
"flavor:replace_flavor_tags": "rule:admin_or_creator",
"flavor:delete_flavor_tags": "rule:admin",
"flavor:add_flavor_tenants": "rule:admin_or_support_or_creator",
"flavor:delete_flavor_tenant": "rule:admin"
}

27
orm/services/image_manager/ims/etc/policy.json
View File

@ -1,27 +0,0 @@
{
"default": "!",
"admin": "role:admin and tenant:admin or role:admin and tenant:service or role:admin_image and tenant:admin or role:admin_image and tenant:service",
"admin_support": "role:admin_support and tenant:admin or role:admin_support and tenant:service",
"admin_viewer": "role:admin_viewer and tenant:admin or role:admin_viewer and tenant:service",
"admin_or_support": "rule:admin or rule:admin_support",
"admin_or_support_or_viewer": "rule:admin or rule:admin_support or rule:admin_viewer",
"image:create": "rule:admin_or_support",
"image:list": "rule:admin_or_support_or_viewer",
"image:get_one": "rule:admin_or_support_or_viewer",
"image:update": "rule:admin",
"image:delete": "rule:admin",
"image:enable": "rule:admin_or_support",
"region:create": "rule:admin_or_support",
"region:update": "rule:admin",
"region:delete": "rule:admin",
"tenant:create": "rule:admin_or_support",
"tenant:update": "rule:admin",
"tenant:delete": "rule:admin",
"metadata:create": "rule:admin_or_support"
}

12
orm/services/resource_distributor/rds/sot/git_sot/git_factory.py
View File

@ -1,12 +0,0 @@
from .git_native import GitNative
def get_git_impl(git_type):
"""Return the correct Git implementation according to git_type"""
git_impl = None
if git_type == 'native':
git_impl = GitNative()
else:
raise RuntimeError("Invalid Git implementation!!")
return git_impl

281
orm/services/resource_distributor/rds/sot/git_sot/git_native.py
View File

@ -1,281 +0,0 @@
"""Native (bash commands) Git module."""
import logging
import shlex
import subprocess
from subprocess import TimeoutExpired as ProcessTimeout
import time
from pecan import conf
logger = logging.getLogger(__name__)
class GitNative(object):
"""The native Git implementation."""
def git_init(self):
"""Initialize Git."""
try:
logger.info("Local repository path:{}, "
"Git server url: {}, "
"Git command timeout: {} seconds, "
"Git retries: {}, "
"Git retries interval: "
"{} seconds".format(conf.git.local_repository_path,
conf.git.git_server_url,
conf.git.git_cmd_timeout,
conf.git.git_retries,
conf.git.git_retries_interval))
out, error = self._git_pull(conf.git.local_repository_path)
if self._is_conflict(out) or self._is_error(error):
logger.error("Git pull result:\nerror:"
" {}\nout: {}".format(error, out))
self.git_reset_changes("Reset all changes due "
"to git pull conflict or error.")
except GitResetError as exc:
msg = "Failed to initialize git repository. " \
"Reason: {}".format(str(exc))
logger.error(msg)
raise GitInitError(msg)
def git_upload_changes(self):
"""Upload (commit and push) the changes to Git."""
commit_id = ""
try:
logger.info("Upload changes in progress ..")
self._git_add(conf.git.local_repository_path)
commit_message = conf.git.commit_message_format.format("")
logger.info("Committing with the following parameters: "
"user: {}, email: {}, message: {}".
format(conf.git.commit_user,
conf.git.commit_email,
commit_message))
self._git_commit(conf.git.commit_user,
conf.git.commit_email,
commit_message,
conf.git.local_repository_path)
commit_id, error = self._git_get_commit_id(
conf.git.local_repository_path)
logger.info("Commit id : {}".format(commit_id))
out, error = self._git_pull(conf.git.local_repository_path)
# This check is needed only for Pull before Push.
if self._is_error(error):
raise GitNativeError("Git pull error: [{}]".format(error))
# Push to repository
self._git_push(conf.git.local_repository_path)
except GitNativeError as exc:
msg = "Failed to upload file to git. " \
"Reason: {}".format(str(exc))
logger.error(msg)
raise GitUploadError(msg)
return commit_id
def validate_git(self):
logger.info("Git repository validation started...\n"
"Git commands timeout: {} "
"seconds".format(conf.git.git_cmd_timeout))
try:
self._git_config(conf.git.local_repository_path)
out, error = self._git_pull(conf.git.local_repository_path)
if self._is_conflict(out) or self._is_error(error):
logger.info("Git pull error, reset ...")
self._git_commit(conf.git.commit_user,
conf.git.commit_email,
"Git pull error found.",
conf.git.local_repository_path)
self._git_reset(conf.git.local_repository_path)
logger.info("Git was reset to server's state.")
else:
self._git_add(conf.git.local_repository_path)
self._git_commit(conf.git.commit_user,
conf.git.commit_email,
"Git validation commit",
conf.git.local_repository_path)
out, error = self._git_pull(conf.git.local_repository_path)
# This check is needed only for pull before push.
if self._is_error(error):
raise GitNativeError("Git pull error: [{}]".format(error))
self._git_push(conf.git.local_repository_path)
logger.info("Git repository state validation check done.")
except GitNativeError as exc:
logger.error("Git state invalid. Reason: [{}]".format(
str(exc)))
raise GitValidateError("Git state invalid.")
def git_reset_changes(self, msg):
logger.info("Reset local repository to Git server started.")
try:
self._git_commit(conf.git.commit_user,
conf.git.commit_email,
msg,
conf.git.local_repository_path)
self._git_reset(conf.git.local_repository_path)
logger.info("Local repository is now up to date "
"with Git server.")
except GitNativeError as exc:
msg = "Git reset changes failed. " \
"Reason: {}".format(str(exc))
logger.error(msg)
raise GitResetError(msg)
def _git_config(self, repo_dir):
logger.info("Set git configuration params started.")
cmds = ['git config --global user.name {}'.format(
conf.git.commit_user),
'git config --global user.email {}'.format(
conf.git.commit_email)]
for cmd in cmds:
self._execute_git_cmd(cmd, repo_dir)
logger.info("Set git configuration params done.")
def _git_add(self, repo_dir):
logger.info("Git add started.")
cmd = 'git add -A'
out, error = self._execute_git_cmd(cmd, repo_dir)
logger.info("Git add done.")
return out, error
def _git_commit(self, user, email, commit_message, repo_dir):
logger.info("Git commit started.")
cmd = 'git commit --author="%s <%s>" -am "%s"' % (user,
email,
commit_message)
out, error = self._execute_git_cmd(cmd, repo_dir)
logger.info("Git commit done.")
return out, error
def _git_get_commit_id(self, repo_dir):
logger.info("Git get commit id started.")
cmd = 'git rev-parse HEAD'
out, error = self._execute_git_cmd(cmd, repo_dir)
# we need to clean \n and whitespaces before returning the commit id
out = out.strip().split('\n')[0]
logger.info("Git get commit id done. commit id: {}".format(out))
return out, error
def _git_reset(self, repo_dir):
logger.info("Git reset started.")
cmd = 'git reset --hard origin/master '
out, error = self._execute_git_cmd(cmd, repo_dir)
logger.info("Git reset done.")
return out, error
def _git_push(self, repo_dir):
logger.info("Git push started.")
cmd = 'git push '
start_time = time.time()
out, error = self._execute_git_cmd(cmd, repo_dir)
logger.info("Git push done "
"(%.3f seconds)" % (time.time() - start_time))
return out, error
def _execute_git_cmd(self, cmd, repo_dir):
error = ""
proc = subprocess.Popen(shlex.split(cmd), cwd=repo_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
try:
(out, error) = proc.communicate(timeout=conf.git.git_cmd_timeout)
logger.debug("Cmd proc id: {}".format(proc.pid))
proc.wait()
except ProcessTimeout:
msg = "Git command '{}' timed out.".format(cmd)
logger.error(msg)
# the word error must be in message
error = "error:" + msg
if self._is_error(error):
raise GitNativeError("Git error: [{}]".format(error))
return out, error
def _git_pull(self, repo_dir):
logger.info("Git pull started.")
cmd = 'git pull '
error = ""
start_time = time.time()
proc = subprocess.Popen(shlex.split(cmd), cwd=repo_dir,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True)
try:
(out, error) = proc.communicate(timeout=conf.git.git_cmd_timeout)
logger.debug("Cmd proc id: {}".format(proc.pid))
proc.wait()
except TimeoutError:
msg = "Git command '{}' timed out.".format(cmd)
logger.error(msg)
# the word error must be in message
error = "error:" + msg
# Special case for pull caller method will check the output
if not self._is_error(error):
logger.info("Git pull done "
"(%.3f seconds)" % (time.time() - start_time))
return out, error
def _is_error(self, error):
if error is not None:
l_error = error.lower()
if 'error' in l_error or 'fatal' in l_error:
logger.error("Git operation returned with "
"error:\n{}".format(error))
return True
return False
def _is_conflict(self, out):
if out is not None:
l_out = out.lower()
if 'conflict' in l_out:
logger.info("Git operation returned with "
"conflict:\n{}".format(out))
return True
return False
def on_subprocess_timeout(cmd, proc):
logger.error("Subprocess for command: {}, timed out.".format(cmd))
logger.info("Terminating subprocess id: {}".format(proc.pid))
proc.kill()
class GitNativeError(Exception):
"""Describes a generic error in a Git operation."""
pass
class GitInitError(Exception):
pass
class GitUploadError(Exception):
pass
class GitResetError(Exception):
pass
class GitValidateError(Exception):
pass

231
orm/services/resource_distributor/rds/sot/git_sot/git_sot.py
View File

@ -1,231 +0,0 @@
import logging
import os
import threading
from . import git_factory
from .git_native import (GitInitError, GitResetError, GitUploadError,
GitValidateError)
from orm.services.resource_distributor.rds.ordupdate.ord_notifier import notify_ord
from orm.services.resource_distributor.rds.sot import base_sot
from orm.services.resource_distributor.rds.sot.base_sot import SoTError
logger = logging.getLogger(__name__)
lock = threading.Lock()
class GitSoT(base_sot.BaseSoT):
local_repository_path = ""
relative_path_format = ""
file_name_format = ""
commit_message_format = ""
commit_user = ""
commit_email = ""
git_server_url = ""
git_type = ""
def __init__(self):
logger.debug("In Git based SoT")
self.git_impl = git_factory.get_git_impl(GitSoT.git_type)
def save_resource_to_sot(self, tracking_id, transaction_id,
resource_list, application_id, user_id,
headers={}):
thread = threading.Thread(target=update_sot,
args=(self.git_impl,
lock,
tracking_id,
transaction_id,
resource_list,
application_id,
user_id,
headers))
thread.start()
def validate_sot_state(self):
thread = threading.Thread(target=validate_git,
args=(self.git_impl, lock))
thread.start()
def update_sot(git_impl, my_lock, tracking_id, transaction_id, resource_list,
application_id, user_id, headers={}):
logger.info("Save resource to SoT. start ...")
commit_id = ""
result = False
logger.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
logger.info("Acquiring Git lock...")
# Lock the entire git operations, so that no other threads change local
# files.
my_lock.acquire()
logger.info("Git lock acquired.")
try:
init_git(git_impl)
handle_file_operations(resource_list)
commit_id = update_git(git_impl)
logger.info("All files were successfully updated in Git server")
result = True
except SoTError as exc:
logger.error("Saving resource to SoT Git repository failed. "
"Reason: {}.".
format(str(exc)))
except GitInitError as init_exc:
logger.error("Initializing Git repository Failed. Reason: {}.".
format(str(init_exc)))
except GitUploadError as upload_exc:
logger.error("Uploading to Git repository Failed. Reason: {}.".
format(str(upload_exc)))
cleanup(git_impl)
finally:
logger.info("Releasing Git lock...")
my_lock.release()
logger.info("Git lock released.")
logger.info("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
# This method is called also in case exception raised.
# Notification to ords will not be sent but status db and audit
# will be updated.
for resource in resource_list:
try:
notify_ord(transaction_id,
tracking_id,
resource["resource_type"],
commit_id, # This is the resource-template-version
GitSoT.file_name_format.format(
resource["resource_name"]),
resource["resource_name"], # This is the resource_id
resource["operation"],
resource["region_id"],
application_id, # application_id is not available
user_id, # user_id is not available
"NA", # external_id is not available
not result,
headers)
except Exception as e:
logger.error("Error in updating ORD. Error: {}".format(
str(e)
))
def handle_file_operations(resource_list):
for resource in resource_list:
file_path = get_resource_file_path(resource)
operation = resource["operation"]
logger.debug("Operation: {}".format(operation))
if operation == "delete":
logger.info("Deleting file: {}".format(file_path))
if os.path.exists(file_path):
try:
os.remove(file_path)
logger.info("File successfully deleted!")
except OSError as ex:
msg = "Could not delete file. " \
"Reason: {}".format(str(ex))
logger.error(msg)
raise SoTError(msg)
else:
logger.info("File does not exist, nothing to delete.")
else: # for all other operations "modify", "create"
logger.info("Adding file: {}".format(file_path))
create_file_in_path(file_path, resource["template_data"])
logger.info("File was successfully added!")
def get_resource_file_path(resource):
file_name = GitSoT.file_name_format.format(resource["resource_name"])
relative_path = GitSoT.relative_path_format. \
format(resource["region_id"],
resource["resource_type"],
file_name)
file_path = GitSoT.local_repository_path + relative_path
return file_path
def create_file_in_path(file_path, file_data):
logger.info("Creating file : {}".format(file_path))
create_dir(file_path)
logger.debug("Directory path created.")
write_data_to_file(file_path, file_data)
logger.info("Data written to file.")
def create_dir(file_path):
# Create actual directory path if not exist
f_path = os.path.dirname(file_path)
if not os.path.exists(f_path):
try:
os.makedirs(f_path)
except OSError as ex:
msg = "Failed to create directory path. " \
"Reason: {}".format(str(ex))
logger.error(msg)
raise SoTError(msg)
def write_data_to_file(file_path, file_data):
# Create and write data to file (If file exists it is overwritten)
try:
with open(file_path, 'w') as fo:
fo.write(file_data)
except IOError as ex:
msg = "Could not write data to file. " \
"Reason: {}".format(str(ex))
logger.error(msg)
raise SoTError(msg)
else:
fo.close()
def init_git(git_impl):
try:
git_impl.git_init()
except GitInitError as exc:
msg = "Failed to initialize Git. " \
"Reason: {}".format(str(exc))
logger.error(msg)
raise GitInitError(msg)
def update_git(git_impl):
commit_id = ""
try:
commit_id = git_impl.git_upload_changes()
except GitUploadError as exc:
logger.error(str(exc))
raise GitUploadError(str(exc))
return commit_id
def validate_git(git_impl, my_lock):
logger.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
logger.info("Acquiring Git lock...")
my_lock.acquire()
logger.info("Git lock acquired.")
try:
git_impl.validate_git()
except GitValidateError as exc:
logger.error("Git validation error. Reason: {}.".
format(str(exc)))
finally:
logger.info("Releasing Git lock...")
my_lock.release()
logger.info("Git lock released.")
logger.info("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<")
def cleanup(git_impl):
logger.info("Cleanup started...")
try:
git_impl.git_reset_changes("Cleaning up changes due to an upload error.")
except GitResetError as exc:
logger.error(str(exc))
raise SoTError(str(exc))

2
orm/tests/config.py
View File

@ -101,7 +101,7 @@ authentication = {
"rms_url": "http://127.0.0.1:8080",
"tenant_name": "admin",
"keystone_version": "3",
"policy_file": "orm/services/flavor_manager/fms_rest/etc/policy.json",
"policy_file": "etc/policy.json",
"user_domain_name": "default",
"project_domain_name": "default"
}

2
orm/tests/unit/cms/config.py
View File

@ -126,7 +126,7 @@ authentication = {
"token_role": "admin",
"role_location": {"tenant": "admin"},
"keystone_version": "3",
"policy_file": "orm/services/customer_manager/cms_rest/etc/policy.json",
"policy_file": "etc/policy.json",
"user_domain_name": "default",
"project_domain_name": "default"
}

2
orm/tests/unit/fms/config.py
View File

@ -92,7 +92,7 @@ authentication = {
"rms_url": "http://127.0.0.1:8080",
"tenant_name": "admin",
"keystone_version": "3",
"policy_file": "orm/services/flavor_manager/fms_rest/etc/policy.json",
"policy_file": "etc/policy.json",
"user_domain_name": "default",
"project_domain_name": "default"
}

2
orm/tests/unit/ims/config.py
View File

@ -101,7 +101,7 @@ authentication = {
"tenant_name": "admin",
"keystone_version": "3",
"token_role": "admin",
"policy_file": "orm/services/image_manager/ims/etc/policy.json",
"policy_file": "etc/policy.json",
"user_domain_name": "default",
"project_domain_name": "default"
}

15
orm/tests/unit/rds/config.py
View File

@ -15,21 +15,6 @@ database = {
'url': 'mysql://root:stack@127.0.0.1/orm_rds?charset=utf8'
}
sot = {
'type': 'git'
}
git = {
'type': 'native',
'local_repository_path': '/home/orm/SoT/ORM',
'file_name_format': 's_{}.yml',
'relative_path_format': '/Document_Store/LCP/{}/{}/{}',
'commit_message_format': 'File was added to repository: {}',
'commit_user': 'orm_rds',
'commit_email': 'orm_rds@att.com',
'git_server_url': 'orm_rds@127.0.0.1:~/SoT/ORM.git',
'git_cmd_timeout': 4
}
audit = {
'audit_server_url': 'http://127.0.0.1:8776/v1/audit/transaction',
'num_of_send_retries': 3,

19
orm/tests/unit/rds/sot/git_sot/test_git_factory.py
View File

@ -1,19 +0,0 @@
import unittest
from orm.services.resource_distributor.rds.sot.git_sot import git_factory
from orm.services.resource_distributor.rds.sot.git_sot.git_native import GitNative
class GitFactoryTests(unittest.TestCase):
def setUp(self):
super(GitFactoryTests, self).setUp()
def test_get_git_impl_with_native(self):
"""Test that when given native the GitNative instance returned"""
obj = git_factory.get_git_impl("native")
self.assertIsInstance(obj, GitNative)
def test_get_sot_no_sot_type(self):
"""Test that when given unknown type, exception raised"""
with self.assertRaises(RuntimeError):
git_factory.get_git_impl("unknown")

69
orm/tests/unit/rds/sot/git_sot/test_git_native.py
View File

@ -1,69 +0,0 @@
"""Unittest module for git_native."""
import unittest
import mock
from mock import patch
from orm.services.resource_distributor.rds.sot.git_sot import git_native
from orm.services.resource_distributor.rds.sot.git_sot.git_native import GitNativeError, GitValidateError
class GitNativeTest(unittest.TestCase):
"""The test case of GitNative."""
@patch.object(git_native, 'conf')
@patch.object(git_native.subprocess, 'Popen')
def test_git_init_sanity(self, mock_popen, mock_conf):
"""Test that no exception is raised when calling git_init."""
mock_conf.git.git_cmd_timeout = 4
my_pipe = mock.MagicMock()
my_pipe.communicate = mock.MagicMock(return_value=('1', '2'))
mock_popen.return_value = my_pipe
test_git = git_native.GitNative()
test_git.git_init()
@patch.object(git_native, 'conf')
@patch.object(git_native.subprocess, 'Popen')
def test_git_upload_changes_sanity(self, mock_popen, mock_conf):
"""Test that no exception is raised when calling git_upload_changes."""
mock_conf.git.git_cmd_timeout = 4
my_pipe = mock.MagicMock()
my_pipe.communicate = mock.MagicMock(return_value=('1', '2'))
mock_popen.return_value = my_pipe
test_git = git_native.GitNative()
test_git.git_upload_changes()
@patch.object(git_native, 'conf')
@patch.object(git_native.subprocess, 'Popen')
def test_git_upload_changes_error(self, mock_popen, mock_conf):
"""Test that an exception is raised when stderror returns error."""
mock_conf.git.git_cmd_timeout = 4
my_pipe = mock.MagicMock()
my_pipe.communicate = mock.MagicMock(return_value=('1', 'error'))
mock_popen.return_value = my_pipe
test_git = git_native.GitNative()
self.assertRaises(git_native.GitUploadError,
test_git.git_upload_changes)
@patch.object(git_native, 'conf')
@patch.object(git_native.subprocess, 'Popen')
def test_git_validate_git_sanity(self, mock_popen, mock_conf):
"""Test that no exception is raised when calling validate_git."""
mock_conf.git.git_cmd_timeout = 4
my_pipe = mock.MagicMock()
my_pipe.communicate = mock.MagicMock(return_value=('1', '2'))
mock_popen.return_value = my_pipe
test_git = git_native.GitNative()
test_git.validate_git()
@patch.object(git_native, 'conf')
@patch.object(git_native.subprocess, 'Popen')
@patch.object(git_native.GitNative, '_git_config',
side_effect=GitNativeError("Could not write to file"))
def test_git_native_validate_git_config_fail(self, mock_conf, mock_popen, result):
"""Test that no exception is raised when calling git_init.aein"""
mock_conf.git.git_cmd_timeout = 4
my_pipe = mock.MagicMock()
my_pipe.communicate = mock.MagicMock(return_value=('1', '2'))
mock_popen.return_value = my_pipe
test_git = git_native.GitNative()
with self.assertRaises(GitValidateError):
test_git.validate_git()

264
orm/tests/unit/rds/sot/git_sot/test_git_sot.py
View File

@ -1,264 +0,0 @@
import os
import unittest
from orm.services.resource_distributor.rds.sot.base_sot import SoTError
from orm.services.resource_distributor.rds.sot.git_sot.git_native import (GitInitError, GitResetError,
GitUploadError, GitValidateError)
from orm.services.resource_distributor.rds.sot.git_sot import git_sot as sot
from orm.services.resource_distributor.rds.sot import sot_factory
from orm.tests.unit.rds import config as conf
from unittest import mock
from unittest.mock import patch
lock = mock.MagicMock()
resource = {
"operation": "create",
"region_id": '1',
"resource_type": '2',
"resource_name": '3',
"template_data": '4'
}
resource_delete = {
"operation": "delete",
"region_id": '1',
"resource_type": '2',
"resource_name": '3',
"template_data": '4'
}
def dummy_thread_method():
pass
class GitSoTTest(unittest.TestCase):
def setUp(self):
super(GitSoTTest, self).setUp()
self.addCleanup(mock.patch.stopall)
git_factory = mock.MagicMock()
git_factory.get_git_impl = mock.MagicMock()
# update_sot
@patch.object(sot, 'init_git', side_effect=GitInitError("Failed to initialize Git"))
def test_git_sot_update_sot_init_git_fail(self, result):
"""" init_git fails and raise exception"""
try:
sot.update_sot("", lock, '1', '2', ['3', '5'], '4', '6')
except SoTError:
self.fail("Exception should have been handled inside method")
@patch.object(sot, 'handle_file_operations', side_effect=SoTError("Failed to create file path"))
@patch.object(sot, 'init_git')
def test_git_sot_update_sot_create_files_fail(self, git_repo, result):
"""" create_files fails and raise exception"""
try:
sot.update_sot("", lock, "a", "b", ['c', 'd'], 'e', 'f')
except SoTError:
self.fail("Exception should have been handled inside method")
@patch.object(sot, 'update_git', side_effect=GitUploadError("Failed to upload file to Git"))
@patch.object(sot, 'init_git')
@patch.object(sot, 'handle_file_operations')
@patch.object(sot, 'cleanup')
def test_git_sot_update_sot_update_git_fail(self, git_repo, files_created, result, clean_result):
"""" update git fails and raise exception"""
try:
sot.update_sot("", lock, 'a', 'b', ['c', 'd'], 'e', 'f')
except GitUploadError:
self.fail("Exception should have been handled inside method")
@patch.object(sot, 'update_git')
@patch.object(sot, 'init_git')
@patch.object(sot, 'handle_file_operations')
@patch.object(sot, 'cleanup')
def test_git_sot_update_sot_success(self, git_repo, files_created, result, cleanup_result):
""""no exceptions raised"""
try:
sot.update_sot("", lock, "a", "b", ['c', 'd'], 'e', 'f')
except SoTError:
self.fail("Exception should have been handled inside method")
#######################
# create_dir #
#######################
@patch.object(os.path, 'dirname', return_value="File path")
@patch.object(os.path, 'exists', return_value=False)
@patch.object(os, 'makedirs')
def test_git_sot_create_dir_path_not_exist_success(self, dir_name, exists, makedirs):
"""create directory path when path not exist and makedir success"""
sot.create_dir("my_file")
@patch.object(os.path, 'dirname', return_value="File path")
@patch.object(os.path, 'exists', return_value=True)
def test_git_sot_create_dir_path_exists_success(self, dir_name, exists):
"""create directory path when path not exist and makedir success"""
sot.create_dir("my_file")
@patch.object(os.path, 'dirname', return_value="File path")
@patch.object(os.path, 'exists', return_value=False)
@patch.object(os, 'makedirs', side_effect=OSError("Could not make dir"))
def test_git_sot_create_dir_makedir_fails(self, dir_name, exists, makedirs):
"""create direcory path makedir throws exception and the methos throws SoTError"""
with self.assertRaises(SoTError):
sot.create_dir("my_file")
################################
# create_file_in_path #
################################
@patch.object(sot, 'create_dir', return_value="File path")
@patch.object(sot, 'write_data_to_file', return_value=True)
def test_git_sot_create_file_in_path_success(self, aDir, aFile):
"""create file in path success scenario no exception raised"""
try:
sot.create_file_in_path("path", "data")
except SoTError:
self.fail("No exceptions should be thrown in this case")
@patch.object(sot, 'create_dir', side_effect=SoTError("Failed to create file path"))
def test_git_sot_create_file_in_path_create_dir_failed(self, aDir):
"""create file in path fail, create file raise exception """
with self.assertRaises(SoTError):
sot.create_file_in_path("path", "data")
@patch.object(sot, 'create_dir', return_value="File path")
@patch.object(sot, 'write_data_to_file', side_effect=SoTError("Could not write to file"))
def test_git_sot_create_file_in_path_create_file_failed(self, aDir, aFile):
"""create file in path fail,create dir success, writing data to file failed """
with self.assertRaises(SoTError):
sot.create_file_in_path("path", "data")
#############################
# get_resource_file_path #
#############################
def test_git_sot_get_resource_file_path_failed(self):
"""get_resource_file_path """
sot_factory.sot_type = "git"
sot_factory.git_type = "native"
sot_factory.local_repository_path = conf.git["local_repository_path"]
sot_factory.relative_path_format = conf.git["relative_path_format"]
sot_factory.file_name_format = conf.git["file_name_format"]
sot_factory.get_sot()
name = conf.git["file_name_format"].format(resource["resource_name"])
path = conf.git["local_repository_path"] + \
conf.git["relative_path_format"].format(resource["region_id"],
resource["resource_type"],
name)
result_path = sot.get_resource_file_path(resource)
self.assertEqual(path, result_path)
#############################
# handle_file_operations #
#############################
# @patch.object(sot, 'create_file_in_path', return_value=True)
# def test_git_sot_handle_file_operations_success(self, result):
# """create files """
# roll_list = []
# sot.handle_file_operations([resource, ])
# self.assertEqual(len(roll_list), 1)
#
#
# @patch.object(os, 'remove')
# @patch.object(os.path, 'exists', return_value=True)
# def test_git_sot_handle_file_operations_delete_success(self,
# result,
# result2):
# """delete files """
# roll_list = []
# sot.handle_file_operations([resource_delete, ], roll_list)
# self.assertEqual(len(roll_list), 1)
#############################
# write_data_to_file #
#############################
@patch('builtins.open')
@patch.object(os, 'close')
@patch.object(os, 'write', return_value=True)
def test_git_sot_write_data_to_file_success(self, result1, result2, result3):
"""Check create_file success """
try:
sot.write_data_to_file("file_path", "data11")
except SoTError:
self.fail("No exceptions should be thrown in this case")
@patch('builtins.open', side_effect=IOError("Failed writing data to file"))
def test_git_sot_write_data_to_file_failed(self, result1):
"""Check create file failed , could not write to file """
with self.assertRaises(SoTError):
sot.write_data_to_file("file_path", "data")
#############################
# cleanup #
#############################
def test_git_sot_cleanup_files_success(self):
"""Check cleanup success """
git_impl = mock.MagicMock()
git_impl.git_reset_changes = mock.MagicMock()
try:
sot.cleanup(git_impl)
except SoTError:
self.fail("No exceptions should be thrown in this case")
def test_git_sot_cleanup_files_remove_failed(self):
"""Check cleanup fail because reset git failed """
git_impl = mock.MagicMock()
git_impl.git_reset_changes = mock.MagicMock(side_effect=GitResetError("failed to reset"))
with self.assertRaises(SoTError):
sot.cleanup(git_impl)
#############################
# init_git #
#############################
def test_git_sot_init_git_success(self):
"""Check init_git success """
git_impl = mock.MagicMock()
git_impl.git_init = mock.MagicMock()
try:
sot.init_git(git_impl)
except GitInitError:
self.fail("No exceptions should be thrown in this case")
#############################
# update_git #
#############################
def test_git_sot_update_git_success(self):
"""Check update_git success"""
git_impl = mock.MagicMock()
git_impl.git_upload_changes = mock.MagicMock(return_value="123")
commit_id = sot.update_git(git_impl)
self.assertEqual(commit_id, "123")
def test_git_sot_update_git_commit_faild(self):
"""Check update_git commit failed"""
git_impl = mock.MagicMock()
git_impl.git_upload_changes = mock.MagicMock(
side_effect=GitUploadError("Failed in upload"))
with self.assertRaises(GitUploadError):
sot.update_git(git_impl)
#############################
# validate_git #
#############################
def test_git_sot_validate_git_faild(self):
"""Check validate_git failed"""
git_impl = mock.MagicMock()
git_impl.validate_git = mock.MagicMock(
side_effect=GitValidateError("Failed in upload"))
try:
sot.validate_git(git_impl, lock)
except GitInitError:
self.fail("No exceptions should be thrown in this case")

48
orm/tests/unit/rds/sot/test_sot_factory.py
View File

@ -1,48 +0,0 @@
import unittest
from orm.services.resource_distributor.rds.sot.git_sot.git_sot import GitSoT
from orm.services.resource_distributor.rds.sot import sot_factory
from unittest import mock
class SoTFactoryTests(unittest.TestCase):
def setUp(self):
super(SoTFactoryTests, self).setUp()
self.addCleanup(mock.patch.stopall)
git_factory = mock.MagicMock()
git_factory.get_git_impl = mock.MagicMock()
def test_get_sot_no_sot_type(self):
"""Check that a runtime error is raised if no git type
is available from config
"""
sot_factory.sot_type = ""
with self.assertRaises(RuntimeError):
sot_factory.get_sot()
# def test_get_sot_git_type(self):
# """ Check that when 'git' type is provided the returned object
# is instance of GiTSoT
# """
# sot_factory.sot_type = "git"
# obj = sot_factory.get_sot()
# self.assertIsInstance(obj, GitSoT)
def test_get_sot_git_sot_params(self):
sot_factory.sot_type = "git"
sot_factory.local_repository_path = "2"
sot_factory.relative_path_format = "3"
sot_factory.commit_message_format = "4"
sot_factory.commit_user = "5"
sot_factory.commit_email = "6"
sot_factory.git_server_url = "7"
sot_factory.git_type = "native"
obj = sot_factory.get_sot()
self.assertEqual(GitSoT.local_repository_path, "2", "local_repository_path not match")
self.assertEqual(GitSoT.relative_path_format, "3", "relative_path_format not match")
self.assertEqual(GitSoT.commit_message_format, "4", "commit_message_format not match")
self.assertEqual(GitSoT.commit_user, "5", "commit_user not match")
self.assertEqual(GitSoT.commit_email, "6", "commit_email not match")
self.assertEqual(GitSoT.git_server_url, "7", "git_server_url not match")
self.assertEqual(GitSoT.git_type, "native", "git_type not match")

2
orm/tests/unit/rms/config.py
View File

@ -42,7 +42,7 @@ authentication = {
"tenant_name": "admin",
# The Keystone version currently in use. Can be either "2.0" or "3"
"keystone_version": "3",
"policy_file": "/opt/app/orm/rms/rms/etc/policy.json",
"policy_file": "etc/policy.json",
"user_domain_name": "default",
"project_domain_name": "default"
}

2
setup.cfg
View File

@ -26,6 +26,8 @@ packages = ranger
data_files =
/etc/ranger =
etc/ranger/ranger.conf
/etc/policy =
etc/policy.json
[entry_points]
console_scripts=