airship/armada
Code Issues Proposed changes

feat(api): policy enforcement and api standard

Browse Source 
- enhanced logging
- created base structure
- updated docs
- PasteDeploy auth
- Oslo Policy

Closes #107

Change-Id: I805863c57f17fcfb26dac5d03efb165e4be49a4e
This commit is contained in:
gardlt 2017-09-12 03:11:57 +00:00
parent d5f4378731
commit bb26131ce2
33 changed files with 883 additions and 365 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Dockerfile
View File

@ -28,8 +28,7 @@ RUN apt-get update && \
\
apt-get purge --auto-remove -y \
build-essential \
curl \
python-all-dev && \
curl && \
apt-get clean -y && \
rm -rf \
/root/.cache \

138
armada/api/__init__.py
View File

@ -1,4 +1,4 @@
# Copyright 2017 The Armada Authors.
# Copyright 2017 The Armada Authors. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -11,3 +11,139 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import uuid
import logging as log
import falcon
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
class BaseResource(object):
def __init__(self):
self.logger = LOG
def on_options(self, req, resp):
self_attrs = dir(self)
methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH']
allowed_methods = []
for m in methods:
if 'on_' + m.lower() in self_attrs:
allowed_methods.append(m)
resp.headers['Allow'] = ','.join(allowed_methods)
resp.status = falcon.HTTP_200
def req_json(self, req):
if req.content_length is None or req.content_length == 0:
return None
if req.content_type is not None and req.content_type.lower(
) == 'application/json':
raw_body = req.stream.read(req.content_length or 0)
if raw_body is None:
return None
try:
# json_body = json.loads(raw_body.decode('utf-8'))
# return json_body
return raw_body
except json.JSONDecodeError as jex:
self.error(
req.context,
"Invalid JSON in request: \n%s" % raw_body.decode('utf-8'))
raise json.JSONDecodeError("%s: Invalid JSON in body: %s" %
(req.path, jex))
else:
raise json.JSONDecodeError("Requires application/json payload")
def return_error(self, resp, status_code, message="", retry=False):
resp.body = json.dumps({
'type': 'error',
'message': message,
'retry': retry
})
resp.status = status_code
def log_error(self, ctx, level, msg):
extra = {'user': 'N/A', 'req_id': 'N/A', 'external_ctx': 'N/A'}
if ctx is not None:
extra = {
'user': ctx.user,
'req_id': ctx.request_id,
'external_ctx': ctx.external_marker,
}
self.logger.log(level, msg, extra=extra)
def debug(self, ctx, msg):
self.log_error(ctx, log.DEBUG, msg)
def info(self, ctx, msg):
self.log_error(ctx, log.INFO, msg)
def warn(self, ctx, msg):
self.log_error(ctx, log.WARN, msg)
def error(self, ctx, msg):
self.log_error(ctx, log.ERROR, msg)
class ArmadaRequestContext(object):
def __init__(self):
self.log_level = 'ERROR'
self.user = None # Username
self.user_id = None # User ID (UUID)
self.user_domain_id = None # Domain owning user
self.roles = ['anyone']
self.project_id = None
self.project_domain_id = None # Domain owning project
self.is_admin_project = False
self.authenticated = False
self.request_id = str(uuid.uuid4())
self.external_marker = ''
def set_log_level(self, level):
if level in ['error', 'info', 'debug']:
self.log_level = level
def set_user(self, user):
self.user = user
def set_project(self, project):
self.project = project
def add_role(self, role):
self.roles.append(role)
def add_roles(self, roles):
self.roles.extend(roles)
def remove_role(self, role):
self.roles = [x for x in self.roles if x != role]
def set_external_marker(self, marker):
self.external_marker = marker
def to_policy_view(self):
policy_dict = {}
policy_dict['user_id'] = self.user_id
policy_dict['user_domain_id'] = self.user_domain_id
policy_dict['project_id'] = self.project_id
policy_dict['project_domain_id'] = self.project_domain_id
policy_dict['roles'] = self.roles
policy_dict['is_admin_project'] = self.is_admin_project
return policy_dict
class ArmadaRequest(falcon.request.Request):
context_type = ArmadaRequestContext

50
armada/api/armada_controller.py
View File

@ -13,41 +13,47 @@
# limitations under the License.
import json
from falcon import HTTP_200
from oslo_config import cfg
import falcon
from oslo_log import log as logging
from armada.handlers.armada import Armada as Handler
from armada import api
from armada.handlers.armada import Armada
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class Apply(object):
class Apply(api.BaseResource):
'''
apply armada endpoint service
'''
def on_post(self, req, resp):
try:
# Load data from request and get options
data = json.load(req.stream)
opts = data['options']
# Load data from request and get options
data = self.req_json(req)
opts = {}
# opts = data['options']
# Encode filename
data['file'] = data['file'].encode('utf-8')
# Encode filename
# data['file'] = data['file'].encode('utf-8')
armada = Armada(
data,
disable_update_pre=opts.get('disable_update_pre', False),
disable_update_post=opts.get('disable_update_post', False),
enable_chart_cleanup=opts.get('enable_chart_cleanup', False),
dry_run=opts.get('dry_run', False),
wait=opts.get('wait', False),
timeout=opts.get('timeout', False))
armada = Handler(open('../../' + data['file']),
disable_update_pre=opts['disable_update_pre'],
disable_update_post=opts['disable_update_post'],
enable_chart_cleanup=opts['enable_chart_cleanup'],
dry_run=opts['dry_run'],
wait=opts['wait'],
timeout=opts['timeout'])
msg = armada.sync()
armada.sync()
resp.data = json.dumps({'message': msg})
resp.data = json.dumps({'message': 'Success'})
resp.content_type = 'application/json'
resp.status = HTTP_200
resp.content_type = 'application/json'
resp.status = falcon.HTTP_200
except Exception as e:
self.error(req.context, "Failed to apply manifest")
self.return_error(
resp, falcon.HTTP_500,
message="Failed to install manifest: {} {}".format(e, data))

117
armada/api/middleware.py
View File

@ -12,10 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon
from uuid import UUID
from keystoneauth1 import session
from keystoneauth1.identity import v3
from oslo_config import cfg
from oslo_log import log as logging
@ -25,75 +23,80 @@ CONF = cfg.CONF
class AuthMiddleware(object):
# Authentication
def process_request(self, req, resp):
ctx = req.context
# Validate token and get user session
token = req.get_header('X-Auth-Token')
req.context['session'] = self._get_user_session(token)
for k, v in req.headers.items():
LOG.debug("Request with header %s: %s" % (k, v))
# Add token roles to request context
req.context['roles'] = self._get_roles(req.context['session'])
auth_status = req.get_header('X-SERVICE-IDENTITY-STATUS')
service = True
def _get_roles(self, session):
if auth_status is None:
auth_status = req.get_header('X-IDENTITY-STATUS')
service = False
# Get roles IDs associated with user
request_url = CONF.auth_url + '/role_assignments'
resp = self._session_request(session=session, request_url=request_url)
if auth_status == 'Confirmed':
# Process account and roles
ctx.authenticated = True
ctx.user = req.get_header(
'X-SERVICE-USER-NAME') if service else req.get_header(
'X-USER-NAME')
ctx.user_id = req.get_header(
'X-SERVICE-USER-ID') if service else req.get_header(
'X-USER-ID')
ctx.user_domain_id = req.get_header(
'X-SERVICE-USER-DOMAIN-ID') if service else req.get_header(
'X-USER-DOMAIN-ID')
ctx.project_id = req.get_header(
'X-SERVICE-PROJECT-ID') if service else req.get_header(
'X-PROJECT-ID')
ctx.project_domain_id = req.get_header(
'X-SERVICE-PROJECT-DOMAIN-ID') if service else req.get_header(
'X-PROJECT-DOMAIN-NAME')
if service:
ctx.add_roles(req.get_header('X-SERVICE-ROLES').split(','))
else:
ctx.add_roles(req.get_header('X-ROLES').split(','))
json_resp = resp.json()['role_assignments']
role_ids = [r['role']['id'].encode('utf-8') for r in json_resp]
if req.get_header('X-IS-ADMIN-PROJECT') == 'True':
ctx.is_admin_project = True
else:
ctx.is_admin_project = False
# Get role names associated with role IDs
roles = []
for role_id in role_ids:
request_url = CONF.auth_url + '/roles/' + role_id
resp = self._session_request(session=session,
request_url=request_url)
LOG.debug('Request from authenticated user %s with roles %s' %
(ctx.user, ','.join(ctx.roles)))
else:
ctx.authenticated = False
role = resp.json()['role']['name'].encode('utf-8')
roles.append(role)
return roles
class ContextMiddleware(object):
def _get_user_session(self, token):
def process_request(self, req, resp):
ctx = req.context
# Get user session from token
auth = v3.Token(auth_url=CONF.auth_url,
project_name=CONF.project_name,
project_domain_name=CONF.project_domain_name,
token=token)
ext_marker = req.get_header('X-Context-Marker')
return session.Session(auth=auth)
if ext_marker is not None and self.is_valid_uuid(ext_marker):
ctx.set_external_marker(ext_marker)
def _session_request(self, session, request_url):
def is_valid_uuid(self, id, version=4):
try:
return session.get(request_url)
uuid_obj = UUID(id, version=version)
except:
raise falcon.HTTPUnauthorized('Authentication required',
('Authentication token is invalid.'))
return False
class RoleMiddleware(object):
return str(uuid_obj) == id
def process_request(self, req, resp):
endpoint = req.path
roles = req.context['roles']
# Verify roles have sufficient permissions for request endpoint
if not (self._verify_roles(endpoint, roles)):
raise falcon.HTTPUnauthorized('Insufficient permissions',
('Token role insufficient.'))
def _verify_roles(self, endpoint, roles):
# Compare the verified roles listed in the config with the user's
# associated roles
if endpoint == '/armada/apply':
approved_roles = CONF.armada_apply_roles
elif endpoint == '/tiller/releases':
approved_roles = CONF.tiller_release_roles
elif endpoint == '/tiller/status':
approved_roles = CONF.tiller_status_roles
verified_roles = set(roles).intersection(approved_roles)
return bool(verified_roles)
class LoggingMiddleware(object):
def process_response(self, req, resp, resource, req_succeeded):
ctx = req.context
extra = {
'user': ctx.user,
'req_id': ctx.request_id,
'external_ctx': ctx.external_marker,
}
resp.append_header('X-Armada-Req', ctx.request_id)
LOG.info("%s - %s" % (req.uri, resp.status), extra=extra)

51
armada/api/server.py
View File

@ -13,42 +13,63 @@
# limitations under the License.
import falcon
import os
from oslo_config import cfg
from oslo_log import log as logging
import armada.conf as configs
from armada.common import policy
from armada import conf
from armada.api import ArmadaRequest
from armada_controller import Apply
from middleware import AuthMiddleware
from middleware import RoleMiddleware
from middleware import ContextMiddleware
from middleware import LoggingMiddleware
from tiller_controller import Release
from tiller_controller import Status
from validation_controller import Validate
LOG = logging.getLogger(__name__)
configs.set_app_default_configs()
conf.set_app_default_configs()
CONF = cfg.CONF
# Build API
def create(middleware=CONF.middleware):
logging.register_options(CONF)
logging.set_defaults(default_log_levels=CONF.default_log_levels)
logging.setup(CONF, 'armada')
if not (os.path.exists('etc/armada/armada.conf')):
logging.register_options(CONF)
logging.set_defaults(default_log_levels=CONF.default_log_levels)
logging.setup(CONF, 'armada')
policy.setup_policy()
if middleware:
api = falcon.API(middleware=[AuthMiddleware(), RoleMiddleware()])
api = falcon.API(
request_type=ArmadaRequest,
middleware=[
AuthMiddleware(),
LoggingMiddleware(),
ContextMiddleware()
])
else:
api = falcon.API()
api = falcon.API(request_type=ArmadaRequest)
# Configure API routing
url_routes = (
('/tiller/status', Status()),
('/tiller/releases', Release()),
('/armada/apply/', Apply())
)
url_routes_v1 = (('apply', Apply()),
('releases', Release()),
('status', Status()),
('validate', Validate()))
for route, service in url_routes:
api.add_route(route, service)
for route, service in url_routes_v1:
api.add_route("/v1.0/{}".format(route), service)
return api
def paste_start_armada(global_conf, **kwargs):
# At this time just ignore everything in the paste configuration
# and rely on olso_config
return api

63
armada/api/tiller_controller.py
View File

@ -13,45 +13,66 @@
# limitations under the License.
import json
from falcon import HTTP_200
import falcon
from oslo_config import cfg
from oslo_log import log as logging
from armada.handlers.tiller import Tiller as tillerHandler
from armada import api
from armada.common import policy
from armada.handlers.tiller import Tiller
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class Status(object):
class Status(api.BaseResource):
@policy.enforce('tiller:get_status')
def on_get(self, req, resp):
'''
get tiller status
'''
message = "Tiller Server is {}"
if tillerHandler().tiller_status():
resp.data = json.dumps({'message': message.format('Active')})
LOG.info('Tiller Server is Active.')
else:
resp.data = json.dumps({'message': message.format('Not Present')})
LOG.info('Tiller Server is Not Present.')
try:
message = {'tiller': Tiller().tiller_status()}
resp.content_type = 'application/json'
resp.status = HTTP_200
if message.get('tiller', False):
resp.status = falcon.HTTP_200
else:
resp.status = falcon.HTTP_503
class Release(object):
resp.data = json.dumps(message)
resp.content_type = 'application/json'
except Exception as e:
self.error(req.context, "Unable to find resources")
self.return_error(
resp, falcon.HTTP_500,
message="Unable to get status: {}".format(e))
class Release(api.BaseResource):
@policy.enforce('tiller:get_release')
def on_get(self, req, resp):
'''
get tiller releases
'''
# Get tiller releases
handler = tillerHandler()
try:
# Get tiller releases
handler = Tiller()
releases = {}
for release in handler.list_releases():
releases[release.name] = release.namespace
releases = {}
for release in handler.list_releases():
if not releases.get(release.namespace, None):
releases[release.namespace] = []
resp.data = json.dumps({'releases': releases})
resp.content_type = 'application/json'
resp.status = HTTP_200
releases[release.namespace].append(release.name)
resp.data = json.dumps({'releases': releases})
resp.content_type = 'application/json'
resp.status = falcon.HTTP_200
except Exception as e:
self.error(req.context, "Unable to find resources")
self.return_error(
resp, falcon.HTTP_500,
message="Unable to find Releases: {}".format(e))

56
armada/api/validation_controller.py Normal file
View File

@ -0,0 +1,56 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import yaml
import falcon
from oslo_log import log as logging
from armada import api
from armada.common import policy
from armada.utils.lint import validate_armada_documents
LOG = logging.getLogger(__name__)
class Validate(api.BaseResource):
'''
apply armada endpoint service
'''
@policy.enforce('armada:validate_manifest')
def on_post(self, req, resp):
try:
message = {
'valid':
validate_armada_documents(
list(yaml.safe_load_all(self.req_json(req))))
}
if message.get('valid', False):
resp.status = falcon.HTTP_200
else:
resp.status = falcon.HTTP_400
resp.data = json.dumps(message)
resp.content_type = 'application/json'
except Exception:
self.error(req.context, "Failed: Invalid Armada Manifest")
self.return_error(
resp,
falcon.HTTP_400,
message="Failed: Invalid Armada Manifest")

0
armada/common/__init__.py Normal file
View File

19
armada/common/i18n.py Normal file
View File

@ -0,0 +1,19 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import oslo_i18n
_translators = oslo_i18n.TranslatorFactory(domain='armada')
# The primary translation function using the well-known name "_"
_ = _translators.primary

25
armada/common/policies/__init__.py Normal file
View File

@ -0,0 +1,25 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import itertools
from armada.common.policies import base
from armada.common.policies import service
from armada.common.policies import tiller
def list_rules():
return itertools.chain(
base.list_rules(),
service.list_rules(),
tiller.list_rules()
)

34
armada/common/policies/base.py Normal file
View File

@ -0,0 +1,34 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_policy import policy
ARMADA = 'armada:%s'
TILLER = 'tiller:%s'
RULE_ADMIN_REQUIRED = 'rule:admin_required'
RULE_ADMIN_OR_TARGET_PROJECT = (
'rule:admin_required or project_id:%(target.project.id)s')
RULE_SERVICE_OR_ADMIN = 'rule:service_or_admin'
rules = [
policy.RuleDefault(name='admin_required',
check_str='role:admin'),
policy.RuleDefault(name='service_or_admin',
check_str='rule:admin_required or rule:service_role'),
policy.RuleDefault(name='service_role',
check_str='role:service'),
]
def list_rules():
return rules

33
armada/common/policies/service.py Normal file
View File

@ -0,0 +1,33 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_policy import policy
from armada.common.policies import base
armada_policies = [
policy.DocumentedRuleDefault(
name=base.ARMADA % 'create_endpoints',
check_str=base.RULE_ADMIN_REQUIRED,
description='install manifest charts',
operations=[{'path': '/v1.0/apply/', 'method': 'POST'}]),
policy.DocumentedRuleDefault(
name=base.ARMADA % 'validate_manifest',
check_str=base.RULE_ADMIN_REQUIRED,
description='validate install manifest',
operations=[{'path': '/v1.0/validate/', 'method': 'POST'}]),
]
def list_rules():
return armada_policies

36
armada/common/policies/tiller.py Normal file
View File

@ -0,0 +1,36 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_policy import policy
from armada.common.policies import base
tiller_policies = [
policy.DocumentedRuleDefault(
name=base.TILLER % 'get_status',
check_str=base.RULE_ADMIN_REQUIRED,
description='Get tiller status',
operations=[{'path': '/v1.0/status/',
'method': 'GET'}]),
policy.DocumentedRuleDefault(
name=base.TILLER % 'get_release',
check_str=base.RULE_ADMIN_REQUIRED,
description='Get tiller release',
operations=[{'path': '/v1.0/releases/',
'method': 'GET'}]),
]
def list_rules():
return tiller_policies

57
armada/common/policy.py Normal file
View File

@ -0,0 +1,57 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import functools
from oslo_config import cfg
from oslo_policy import policy
from armada.common import policies
from armada.exceptions import base_exception as exc
CONF = cfg.CONF
_ENFORCER = None
def setup_policy():
global _ENFORCER
if not _ENFORCER:
_ENFORCER = policy.Enforcer(CONF)
register_rules(_ENFORCER)
def enforce_policy(action, target, credentials, do_raise=True):
extras = {}
if do_raise:
extras.update(exc=exc.ActionForbidden, do_raise=do_raise)
_ENFORCER.enforce(action, target, credentials.to_policy_view(), **extras)
def enforce(rule):
setup_policy()
def decorator(func):
@functools.wraps(func)
def handler(*args, **kwargs):
context = args[1].context
enforce_policy(rule, {}, context, do_raise=True)
return func(*args, **kwargs)
return handler
return decorator
def register_rules(enforcer):
enforcer.register_defaults(policies.list_rules())

32
armada/exceptions/api_exceptions.py Normal file
View File

@ -0,0 +1,32 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import base_exception as base
class ApiException(base.ArmadaBaseException):
'''Base class for API exceptions and error handling.'''
message = 'An unknown API error occur.'
class ApiBaseException(ApiException):
'''Exception that occurs during chart cleanup.'''
message = 'There was an error listing the helm chart releases.'
class ApiJsonException(ApiException):
'''Exception that occurs during chart cleanup.'''
message = 'There was an error listing the helm chart releases.'

21
armada/exceptions/base_exception.py
View File

@ -12,9 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon
from oslo_config import cfg
from oslo_log import log as logging
from armada.common.i18n import _
LOG = logging.getLogger(__name__)
DEFAULT_TIMEOUT = 3600
@ -27,3 +30,21 @@ class ArmadaBaseException(Exception):
def __init__(self, message=None):
self.message = message or self.message
super(ArmadaBaseException, self).__init__(self.message)
class ArmadaAPIException(falcon.HTTPError):
status = falcon.HTTP_500
message = "unknown error"
title = "Internal Server Error"
def __init__(self, message=None, **kwargs):
self.message = message or self.message
super(ArmadaAPIException, self).__init__(
self.status, self.title, self.message, **kwargs
)
class ActionForbidden(ArmadaAPIException):
status = falcon.HTTP_403
message = _("Insufficient privilege to perform action.")
title = _("Action Forbidden")

13
armada/handlers/__init__.py
View File

@ -1,13 +0,0 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

41
armada/handlers/armada.py
View File

@ -15,7 +15,6 @@
import difflib
import yaml
from oslo_config import cfg
from oslo_log import log as logging
from supermutes.dot import dotify
@ -37,7 +36,6 @@ from ..const import KEYWORD_ARMADA, KEYWORD_GROUPS, KEYWORD_CHARTS,\
LOG = logging.getLogger(__name__)
DEFAULT_TIMEOUT = 3600
CONF = cfg.CONF
class Armada(object):
@ -183,6 +181,12 @@ class Armada(object):
Syncronize Helm with the Armada Config(s)
'''
msg = {
'installed': [],
'upgraded': [],
'diff': []
}
# TODO: (gardlt) we need to break up this func into
# a more cleaner format
LOG.info("Performing Pre-Flight Operations")
@ -268,9 +272,9 @@ class Armada(object):
# TODO(alanmeadows) account for .files differences
# once we support those
upgrade_diff = self.show_diff(chart, apply_chart,
apply_values,
chartbuilder.dump(), values)
upgrade_diff = self.show_diff(
chart, apply_chart, apply_values, chartbuilder.dump(),
values, msg)
if not upgrade_diff:
LOG.info("There are no updates found in this chart")
@ -290,6 +294,8 @@ class Armada(object):
wait=chart_wait,
timeout=chart_timeout)
msg['upgraded'].append(prefix_chart)
# process install
else:
LOG.info("Installing release %s", chart.release)
@ -301,6 +307,8 @@ class Armada(object):
wait=chart_wait,
timeout=chart_timeout)
msg['installed'].append(prefix_chart)
LOG.debug("Cleaning up chart source in %s",
chartbuilder.source_directory)
@ -322,6 +330,8 @@ class Armada(object):
self.tiller.chart_cleanup(
prefix, self.config[KEYWORD_ARMADA][KEYWORD_GROUPS])
return msg
def post_flight_ops(self):
'''
Operations to run after deployment process has terminated
@ -333,7 +343,7 @@ class Armada(object):
source.source_cleanup(ch.get('chart').get('source_dir')[0])
def show_diff(self, chart, installed_chart, installed_values, target_chart,
target_values):
target_values, msg):
'''
Produce a unified diff of the installed chart vs our intention
@ -342,19 +352,32 @@ class Armada(object):
'''
chart_diff = list(
difflib.unified_diff(installed_chart.SerializeToString()
.split('\n'), target_chart.split('\n')))
difflib.unified_diff(
installed_chart.SerializeToString().split('\n'),
target_chart.split('\n')))
if len(chart_diff) > 0:
LOG.info("Chart Unified Diff (%s)", chart.release)
diff_msg = []
for line in chart_diff:
diff_msg.append(line)
LOG.debug(line)
msg['diff'].append({'chart': diff_msg})
values_diff = list(
difflib.unified_diff(
installed_values.split('\n'),
yaml.safe_dump(target_values).split('\n')))
if len(values_diff) > 0:
LOG.info("Values Unified Diff (%s)", chart.release)
diff_msg = []
for line in values_diff:
diff_msg.append(line)
LOG.debug(line)
msg['diff'].append({'values': diff_msg})
return (len(chart_diff) > 0) or (len(values_diff) > 0)
result = (len(chart_diff) > 0) or (len(values_diff) > 0)
return result

37
armada/tests/unit/api/test_api.py
View File

@ -16,16 +16,21 @@ import json
import mock
import unittest
import falcon
from falcon import testing
from armada import conf as cfg
from armada.api import server
CONF = cfg.CONF
class APITestCase(testing.TestCase):
def setUp(self):
super(APITestCase, self).setUp()
self.app = server.create(middleware=False)
class TestAPI(APITestCase):
@unittest.skip('this is incorrectly tested')
@mock.patch('armada.api.armada_controller.Handler')
@ -35,7 +40,7 @@ class TestAPI(APITestCase):
'''
mock_armada.sync.return_value = None
body = json.dumps({'file': '../examples/openstack-helm.yaml',
body = json.dumps({'file': '',
'options': {'debug': 'true',
'disable_update_pre': 'false',
'disable_update_post': 'false',
@ -50,10 +55,10 @@ class TestAPI(APITestCase):
result = self.simulate_post(path='/armada/apply', body=body)
self.assertEqual(result.json, doc)
@mock.patch('armada.api.tiller_controller.tillerHandler')
@mock.patch('armada.api.tiller_controller.Tiller')
def test_tiller_status(self, mock_tiller):
'''
Test /tiller/status endpoint
Test /status endpoint
'''
# Mock tiller status value
@ -61,10 +66,17 @@ class TestAPI(APITestCase):
doc = {u'message': u'Tiller Server is Active'}
result = self.simulate_get('/tiller/status')
self.assertEqual(result.json, doc)
result = self.simulate_get('/v1.0/status')
@mock.patch('armada.api.tiller_controller.tillerHandler')
# TODO(lamt) This should be HTTP_401 if no auth is happening, but auth
# is not implemented currently, so it falls back to a policy check
# failure, thus a 403. Change this once it is completed
self.assertEqual(falcon.HTTP_403, result.status)
# FIXME(lamt) Need authentication - mock, fixture
# self.assertEqual(result.json, doc)
@mock.patch('armada.api.tiller_controller.Tiller')
def test_tiller_releases(self, mock_tiller):
'''
Test /tiller/releases endpoint
@ -75,5 +87,12 @@ class TestAPI(APITestCase):
doc = {u'releases': {}}
result = self.simulate_get('/tiller/releases')
self.assertEqual(result.json, doc)
result = self.simulate_get('/v1.0/releases')
# TODO(lamt) This should be HTTP_401 if no auth is happening, but auth
# is not implemented currently, so it falls back to a policy check
# failure, thus a 403. Change this once it is completed
self.assertEqual(falcon.HTTP_403, result.status)
# FIXME(lamt) Need authentication - mock, fixture
# self.assertEqual(result.json, doc)

65
armada/tests/unit/test_policy.py Normal file
View File

@ -0,0 +1,65 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import testtools
from oslo_policy import policy as common_policy
from armada.common import policy
from armada import conf as cfg
from armada.exceptions import base_exception as exc
import mock
CONF = cfg.CONF
class PolicyTestCase(testtools.TestCase):
def setUp(self):
super(PolicyTestCase, self).setUp()
self.rules = {
"true": [],
"example:allowed": [],
"example:disallowed": [["false:false"]]
}
self._set_rules()
self.credentials = {}
self.target = {}
def _set_rules(self):
curr_rules = common_policy.Rules.from_dict(self.rules)
policy._ENFORCER.set_rules(curr_rules)
@mock.patch('armada.api.ArmadaRequestContext')
def test_enforce_nonexistent_action(self, mock_ctx):
action = "example:nope"
mock_ctx.to_policy_view.return_value = self.credentials
self.assertRaises(
exc.ActionForbidden, policy.enforce_policy, action,
self.target, mock_ctx)
@mock.patch('armada.api.ArmadaRequestContext')
def test_enforce_good_action(self, mock_ctx):
action = "example:allowed"
mock_ctx.to_policy_view.return_value = self.credentials
policy.enforce_policy(action, self.target, mock_ctx)
@mock.patch('armada.api.ArmadaRequestContext')
def test_enforce_bad_action(self, mock_ctx):
action = "example:disallowed"
mock_ctx.to_policy_view.return_value = self.credentials
self.assertRaises(exc.ActionForbidden, policy.enforce_policy,
action, self.target, mock_ctx)

30
docs/source/development/getting-started.rst
View File

@ -13,9 +13,17 @@ To use the docker containter to develop:
.. code-block:: bash
git clone http://github.com/att-comdev/armada.git
cd armada
pip install tox
tox -e genconfig
tox -e genpolicy
docker build . -t armada/latest
docker run -d --name armada -v ~/.kube/config:/armada/.kube/config -v $(pwd)/examples/:/examples armada/latest
docker run -d --name armada -v ~/.kube/config:/armada/.kube/config -v $(pwd)/etc:/armada/etc armada:local
.. note::
@ -25,17 +33,21 @@ To use the docker containter to develop:
Virtualenv
##########
To use VirtualEnv:
How to set up armada in your local using virtualenv:
1. virtualenv venv
2. source ./venv/bin/activate
.. note::
Suggest that you use a Ubuntu 16.04 VM
From the directory of the forked repository:
.. code-block:: bash
pip install -r requirements.txt
pip install -r test-requirements.txt
git clone http://github.com/att-comdev/armada.git && cd armada
virtualenv venv
pip install -r requirements.txt -r test-requirements.txt
pip install .
@ -48,6 +60,12 @@ From the directory of the forked repository:
tox -e bandit
tox -e cover
# policy and config are used in order to use and configure Armada API
tox -e genconfig
tox -e genpolicy
.. note::
If building from source, Armada requires that git be installed on

52
docs/source/operations/guide-configure.rst Normal file
View File

@ -0,0 +1,52 @@
==================
Configuring Armada
==================
Armada uses an INI-like standard oslo_config file. A sample
file can be generated via tox
.. code-block:: bash
$ tox -e genconfig
Customize your configuration based on the information below
Keystone Integration
====================
Armada requires a service account to use for validating API
tokens
.. note::
If you do not have a keystone already deploy, then armada can deploy a keystone service.
armada apply keystone-manifest.yaml
.. code-block:: bash
$ openstack domain create 'ucp'
$ openstack project create --domain 'ucp' 'service'
$ openstack user create --domain ucp --project service --project-domain 'ucp' --password armada armada
$ openstack role add --project-domain ucp --user-domain ucp --user armada --project service admin
# OR
$ ./tools/keystone-account.sh
The service account must then be included in the armada.conf
.. code-block:: ini
[keystone_authtoken]
auth_uri = https://<keystone-api>:5000/
auth_version = 3
delay_auth_decision = true
auth_type = password
auth_url = https://<keystone-api>:35357/
project_name = service
project_domain_name = ucp
user_name = armada
user_domain_name = ucp
password = armada

7
docs/source/operations/index.rst
View File

@ -10,7 +10,8 @@ Operations Guide
:maxdepth: 2
:caption: Contents:
guide-troubleshooting.rst
guide-build-armada-yaml.rst
guide-use-armada.rst
guide-api.rst
guide-build-armada-yaml.rst
guide-configure.rst
guide-troubleshooting.rst
guide-use-armada.rst

2
entrypoint.sh
View File

@ -6,7 +6,7 @@ PORT="8000"
set -e
if [ "$1" = 'server' ]; then
exec gunicorn server:api -b :$PORT --chdir armada/api
exec uwsgi --http 0.0.0.0:${PORT} --paste config:$(pwd)/etc/armada/api-paste.ini --enable-threads -L --pyargv " --config-file $(pwd)/etc/armada/armada.conf"
fi
if [ "$1" = 'tiller' ] || [ "$1" = 'apply' ]; then

8
etc/armada/api-paste.ini Normal file
View File

@ -0,0 +1,8 @@
[app:armada-api]
paste.app_factory = armada.api.server:paste_start_armada
[pipeline:main]
pipeline = authtoken armada-api
[filter:authtoken]
paste.filter_factory = keystonemiddleware.auth_token:filter_factory

140
etc/armada/armada.conf.sample
View File

@ -1,140 +0,0 @@
[DEFAULT]
#
# From armada.conf
#
# IDs of approved API access roles. (list value)
#armada_apply_roles = admin
# The default Keystone authentication url. (string value)
#auth_url = http://0.0.0.0/v3
# Path to Kubernetes configurations. (string value)
#kubernetes_config_path = /home/user/.kube/
# Enables or disables Keystone authentication middleware. (boolean value)
#middleware = true
# The Keystone project domain name used for authentication. (string value)
#project_domain_name = default
# The Keystone project name used for authentication. (string value)
#project_name = admin
# Path to SSH private key. (string value)
#ssh_key_path = /home/user/.ssh/
# IDs of approved API access roles. (list value)
#tiller_release_roles = admin
# IDs of approved API access roles. (list value)
#tiller_status_roles = admin
#
# From oslo.log
#
# If set to true, the logging level will be set to DEBUG instead of the default
# INFO level. (boolean value)
# Note: This option can be changed without restarting.
#debug = false
# The name of a logging configuration file. This file is appended to any
# existing logging configuration files. For details about logging configuration
# files, see the Python logging module documentation. Note that when logging
# configuration files are used then all logging configuration is set in the
# configuration file and other logging configuration options are ignored (for
# example, logging_context_format_string). (string value)
# Note: This option can be changed without restarting.
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
# Defines the format string for %%(asctime)s in log records. Default:
# %(default)s . This option is ignored if log_config_append is set. (string
# value)
#log_date_format = %Y-%m-%d %H:%M:%S
# (Optional) Name of log file to send logging output to. If no default is set,
# logging will go to stderr as defined by use_stderr. This option is ignored if
# log_config_append is set. (string value)
# Deprecated group/name - [DEFAULT]/logfile
#log_file = <None>
# (Optional) The base directory used for relative log_file paths. This option
# is ignored if log_config_append is set. (string value)
# Deprecated group/name - [DEFAULT]/logdir
#log_dir = <None>
# Uses logging handler designed to watch file system. When log file is moved or
# removed this handler will open a new log file with specified path
# instantaneously. It makes sense only if log_file option is specified and Linux
# platform is used. This option is ignored if log_config_append is set. (boolean
# value)
#watch_log_file = false
# Use syslog for logging. Existing syslog format is DEPRECATED and will be
# changed later to honor RFC5424. This option is ignored if log_config_append is
# set. (boolean value)
#use_syslog = false
# Enable journald for logging. If running in a systemd environment you may wish
# to enable journal support. Doing so will use the journal native protocol which
# includes structured metadata in addition to log messages.This option is
# ignored if log_config_append is set. (boolean value)
#use_journal = false
# Syslog facility to receive log lines. This option is ignored if
# log_config_append is set. (string value)
#syslog_log_facility = LOG_USER
# Log output to standard error. This option is ignored if log_config_append is
# set. (boolean value)
#use_stderr = false
# Format string to use for log messages with context. (string value)
#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
# Format string to use for log messages when context is undefined. (string
# value)
#logging_default_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s
# Additional data to append to log message when logging level for the message is
# DEBUG. (string value)
#logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d
# Prefix each line of exception output with this format. (string value)
#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d ERROR %(name)s %(instance)s
# Defines the format string for %(user_identity)s that is used in
# logging_context_format_string. (string value)
#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s
# List of package logging levels in logger=LEVEL pairs. This option is ignored
# if log_config_append is set. (list value)
#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,oslo_messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN,oslo.cache=INFO,dogpile.core.dogpile=INFO
# Enables or disables publication of error events. (boolean value)
#publish_errors = false
# The format for an instance that is passed with the log message. (string value)
#instance_format = "[instance: %(uuid)s] "
# The format for an instance UUID that is passed with the log message. (string
# value)
#instance_uuid_format = "[instance: %(uuid)s] "
# Interval, number of seconds, of log rate limiting. (integer value)
#rate_limit_interval = 0
# Maximum number of logged messages per rate_limit_interval. (integer value)
#rate_limit_burst = 0
# Log level name used by rate limiting: CRITICAL, ERROR, INFO, WARNING, DEBUG or
# empty string. Logs with level greater or equal to rate_limit_except_level are
# not filtered. An empty string means that all levels are filtered. (string
# value)
#rate_limit_except_level = CRITICAL
# Enables or disables fatal status of deprecations. (boolean value)
#fatal_deprecations = false

5
etc/armada/config-generator.conf
View File

@ -1,5 +1,8 @@
[DEFAULT]
output_file = etc/armada/armada.conf.sample
wrap_width = 80
wrap_width = 79
namespace = armada.conf
namespace = oslo.log
namespace = oslo.policy
namespace = oslo.middleware
namespace = keystonemiddleware.auth_token

5
etc/armada/policy-generator.conf Normal file
View File

@ -0,0 +1,5 @@
[DEFAULT]
output_file = etc/armada/policy.yaml.sample
wrap_width = 79
namespace = armada

59
examples/armada-manifest-v1.yaml → examples/keystone-manifest.yaml
View File

@ -22,7 +22,7 @@ metadata:
data:
chart_name: mariadb
release: mariadb
namespace: undercloud
namespace: openstack
timeout: 3600
install:
no_hooks: false
@ -44,7 +44,7 @@ metadata:
data:
chart_name: memcached
release: memcached
namespace: undercloud
namespace: openstack
timeout: 100
install:
no_hooks: false
@ -60,50 +60,6 @@ data:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: etcd
data:
chart_name: etcd
release: etcd
namespace: undercloud
timeout: 3600
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: etcd
reference: master
dependencies:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: rabbitmq
data:
chart_name: rabbitmq
release: rabbitmq
namespace: undercloud
timeout: 100
install:
no_hooks: false
upgrade:
no_hooks: false
values: {}
source:
type: git
location: git://github.com/openstack/openstack-helm
subpath: rabbitmq
reference: master
dependencies:
- helm-toolkit
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: keystone
@ -111,8 +67,8 @@ data:
chart_name: keystone
test: true
release: keystone
namespace: undercloud
timeout: 3600
namespace: openstack
timeout: 100
install:
no_hooks: false
upgrade:
@ -130,13 +86,12 @@ data:
schema: armada/ChartGroup/v1
metadata:
schema: metadata/Document/v1
name: openstack-infra-services
name: keystone-infra-services
data:
description: "OpenStack Infra Services"
description: "Keystone Infra Services"
sequenced: True
chart_group:
- mariadb
- etcd
- memcached
---
schema: armada/ChartGroup/v1
@ -157,5 +112,5 @@ metadata:
data:
release_prefix: armada
chart_groups:
- openstack-infra-services
- keystone-infra-services
- openstack-keystone

19
requirements.txt
View File

@ -2,15 +2,17 @@ gitpython==2.1.5
grpcio==1.6.0rc1
grpcio-tools==1.6.0rc1
keystoneauth1==2.21.0
keystonemiddleware==4.9.1
kubernetes>=1.0.0
oslo.log==3.28.0
oslo.messaging==5.28.0
protobuf==3.2.0
PyYAML==3.12
requests==2.17.3
sphinx_rtd_theme
supermutes==0.2.5
urllib3==1.21.1
uwsgi>=2.0.15
Paste>=2.0.3
PasteDeploy>=1.5.2
# API
falcon==1.1.0
@ -20,6 +22,15 @@ gunicorn==19.7.1
cliff==2.7.0
# Oslo
oslo.log==3.28.0
oslo.config>=3.22.0 # Apache-2.0
oslo.cache>=1.5.0 # Apache-2.0
oslo.concurrency>=3.8.0 # Apache-2.0
oslo.config!=4.3.0,!=4.4.0,>=4.0.0 # Apache-2.0
oslo.context>=2.14.0 # Apache-2.0
oslo.messaging!=5.25.0,>=5.24.2 # Apache-2.0
oslo.db>=4.24.0 # Apache-2.0
oslo.i18n!=3.15.2,>=2.1.0 # Apache-2.0
oslo.log>=3.22.0 # Apache-2.0
oslo.middleware>=3.27.0 # Apache-2.0
oslo.policy>=1.23.0 # Apache-2.0
oslo.serialization!=2.19.1,>=1.10.0 # Apache-2.0
oslo.utils>=3.20.0 # Apache-2.0

2
setup.cfg
View File

@ -43,6 +43,8 @@ armada =
test = armada.cli.test:TestServerCommand
oslo.config.opts =
armada.conf = armada.conf.opts:list_opts
oslo.policy.policies =
armada = armada.common.policies:list_rules
[pbr]
warnerrors = True

6
tools/keystone-account.sh Executable file
View File

@ -0,0 +1,6 @@
#!/usr/bin/env bash
openstack domain create 'ucp'
openstack project create --domain 'ucp' 'service'
openstack user create --domain ucp --project service --project-domain 'ucp' --password armada armada
openstack role add --project-domain ucp --user-domain ucp --user armada --project service admin

22
tox.ini
View File

@ -10,33 +10,37 @@ deps=
setenv=
VIRTUAL_ENV={envdir}
usedevelop = True
install_command = pip install {opts} {packages}
commands =
find . -type f -name "*.pyc" -delete
python -V
py.test -vvv -s --ignore=hapi
find . -type f -name "*.pyc" -delete
python -V
py.test -vvv -s --ignore=hapi
[testenv:docs]
commands =
python setup.py build_sphinx
python setup.py build_sphinx
[testenv:genconfig]
commands =
oslo-config-generator --config-file=etc/armada/config-generator.conf
oslo-config-generator --config-file=etc/armada/config-generator.conf
[testenv:genpolicy]
commands =
oslopolicy-sample-generator --config-file=etc/armada/policy-generator.conf
[testenv:pep8]
commands =
flake8 {posargs}
flake8 {posargs}
[testenv:bandit]
commands =
bandit -r armada -x armada/tests -n 5
bandit -r armada -x armada/tests -n 5
[testenv:coverage]
commands =
nosetests -w armada/tests/unit --cover-package=armada --with-coverage --cover-tests --exclude=.tox
nosetests -w armada/tests/unit --cover-package=armada --with-coverage --cover-tests --exclude=.tox
[flake8]
filename= *.py
ignore = W503,E302
exclude= .git, .idea, .tox, *.egg-info, *.eggs, bin, dist, hapi