feat(yaml): Support document references

This allows the user to apply and/or validate a manifest using
either a filepath (as before) or URL.

Addition by sh8121att:

- Create a general document resolver class to handle local paths
  and URIs
- Allow multiple filenames and combine them into a single document
  set
- Change API to allow for passing document reference URIs to be
  resolved server-side rather
- Update validation API to conform to UCP specification
- Dockerfile updates to speed up build
- Fix unit tests

Closes #96

Change-Id: I5a57779f10d1b63ffc161a14afec851a34ae9efe

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:16.04
+FROM python:3.5
 
 MAINTAINER Armada Team
 
@@ -6,33 +6,20 @@ ENV DEBIAN_FRONTEND noninteractive
 ENV LANG=C.UTF-8
 ENV LC_ALL=C.UTF-8
 
-COPY . /armada
+COPY requirements.txt /tmp/
+RUN pip3 install -r /tmp/requirements.txt
 
+COPY . /armada
 RUN apt-get update && \
-    apt-get upgrade -y && \
     apt-get install -y --no-install-recommends \
       netbase \
-      python3-pip && \
-    apt-get install -y \
-      build-essential \
       curl \
-      git \
+      git && \
-      python3-minimal \
-      python3-setuptools \
-      python3-dev && \
     useradd -u 1000 -g users -d /armada armada && \
     chown -R armada:users /armada && \
     mv /armada/etc/armada /etc/ && \
-    \
     cd /armada && \
-    pip3 install --upgrade pip && \
-    pip3 install -r requirements.txt && \
     python3 setup.py install && \
-    \
-    apt-get purge --auto-remove -y \
-      build-essential \
-      curl && \
-    apt-get clean -y && \
     rm -rf \
       /root/.cache \
       /var/lib/apt/lists/*

armada/api/__init__.py

@@ -59,6 +59,24 @@ class BaseResource(object):
             raise Exception(
                 "%s: Invalid YAML in body: %s" % (req.path, jex))
 
+    def req_json(self, req):
+        if req.content_length is None or req.content_length == 0:
+            return None
+
+        raw_body = req.stream.read(req.content_length or 0)
+
+        if raw_body is None:
+            return None
+
+        try:
+            return json.loads(raw_body.decode())
+        except json.JSONDecodeError as jex:
+            self.error(
+                req.context,
+                "Invalid JSON in request: %s" % str(jex))
+            raise Exception(
+                "%s: Invalid JSON in body: %s" % (req.path, jex))
+
     def return_error(self, resp, status_code, message="", retry=False):
         resp.body = json.dumps({
             'type': 'error',

armada/api/controller/armada.py

@@ -13,12 +13,15 @@
 # limitations under the License.
 
 import json
+import yaml
 
 import falcon
 
 from armada import api
 from armada.common import policy
 from armada.handlers.armada import Armada
+from armada.handlers.document import ReferenceResolver
+from armada.handlers.override import Override
 
 
 class Apply(api.BaseResource):
@@ -30,17 +33,45 @@ class Apply(api.BaseResource):
         try:
 
             # Load data from request and get options
-
+            if req.content_type == 'application/x-yaml':
                 data = list(self.req_yaml(req))
-
                 if type(data[0]) is list:
-                data = list(data[0])
+                    documents = list(data[0])
+                else:
+                    documents = data
+            elif req.content_type == 'application/json':
+                self.logger.debug("Applying manifest based on reference.")
+                req_body = self.req_json(req)
+                doc_ref = req_body.get('hrefs', None)
+
+                if not doc_ref:
+                    self.logger.info("Request did not contain 'hrefs'.")
+                    resp.status = falcon.HTTP_400
+                    return
+
+                data = ReferenceResolver.resolve_reference(doc_ref)
+                documents = list()
+                for d in data:
+                    documents.extend(list(yaml.safe_load_all(d.decode())))
+
+                if req_body.get('overrides', None):
+                    overrides = Override(documents,
+                                         overrides=req_body.get('overrides'))
+                    documents = overrides.update_manifests()
+            else:
+                self.error(req.context, "Unknown content-type %s"
+                           % req.content_type)
+                self.return_error(
+                    resp,
+                    falcon.HTTP_415,
+                    message="Request must be in application/x-yaml"
+                            "or application/json")
 
             opts = req.params
 
             # Encode filename
             armada = Armada(
-                data,
+                documents,
                 disable_update_pre=req.get_param_as_bool(
                     'disable_update_pre'),
                 disable_update_post=req.get_param_as_bool(

armada/api/controller/validation.py

@@ -13,12 +13,13 @@
 # limitations under the License.
 
 import json
-
 import falcon
+import yaml
 
 from armada import api
 from armada.common import policy
 from armada.utils.lint import validate_armada_documents
+from armada.handlers.document import ReferenceResolver
 
 
 class Validate(api.BaseResource):
@@ -29,19 +30,58 @@ class Validate(api.BaseResource):
     @policy.enforce('armada:validate_manifest')
     def on_post(self, req, resp):
         try:
+            if req.content_type == 'application/json':
+                self.logger.debug("Validating manifest based on reference.")
+                json_body = self.req_json(req)
+                if json_body.get('href', None):
+                    self.logger.debug("Validating manifest from reference %s."
+                                      % json_body.get('href'))
+                    data = ReferenceResolver.resolve_reference(
+                        json_body.get('href'))
+                    documents = list()
+                    for d in data:
+                        documents.extend(list(yaml.safe_load_all(d.decode())))
+                else:
+                    resp.status = falcon.HTTP_400
+                    return
+            else:
                 manifest = self.req_yaml(req)
                 documents = list(manifest)
 
-            message = {
+            self.logger.debug("Validating set of %d documents."
-                'valid': validate_armada_documents(documents)
+                              % len(documents))
+
+            result = validate_armada_documents(documents)
+
+            resp.content_type = 'application/json'
+            resp_body = {
+                'kind': 'Status',
+                'apiVersion': 'v1.0',
+                'metadata': {},
+                'reason': 'Validation',
+                'details': {
+                    'errorCount': 0,
+                    'messageList': []
+                },
             }
 
+            if result:
                 resp.status = falcon.HTTP_200
-            resp.body = json.dumps(message)
+                resp_body['status'] = 'Success'
-            resp.content_type = 'application/json'
+                resp_body['message'] = 'Armada validations succeeded'
+                resp_body['code'] = 200
+            else:
+                resp.status = falcon.HTTP_400
+                resp_body['status'] = 'Failure'
+                resp_body['message'] = 'Armada validations failed'
+                resp_body['code'] = 400
+                resp_body['details']['errorCount'] = 1
+                resp_body['details']['messageList'].\
+                    append(dict(message='Validation failed.', error=True))
 
-        except Exception:
+            resp.body = json.dumps(resp_body)
+        except Exception as ex:
             err_message = 'Failed to validate Armada Manifest'
-            self.error(req.context, err_message)
+            self.logger.error(err_message, exc_info=ex)
             self.return_error(
                 resp, falcon.HTTP_400, message=err_message)

armada/api/server.py

@@ -15,6 +15,7 @@
 import falcon
 from oslo_config import cfg
 from oslo_policy import policy
+from oslo_log import log as logging
 
 from armada import conf
 from armada.api import ArmadaRequest
@@ -52,6 +53,9 @@ def create(enable_middleware=CONF.middleware):
     else:
         api = falcon.API(request_type=ArmadaRequest)
 
+    logging.set_defaults(default_log_levels=CONF.default_log_levels)
+    logging.setup(CONF, 'armada')
+
     # Configure API routing
     url_routes_v1 = (
         ('health', Health()),
@@ -61,6 +65,7 @@ def create(enable_middleware=CONF.middleware):
         ('tests', Tests()),
         ('test/{release}', Test()),
         ('validate', Validate()),
+        ('validatedesign', Validate()),
     )
 
     for route, service in url_routes_v1:

armada/cli/apply.py

@@ -18,7 +18,9 @@ import click
 from oslo_config import cfg
 
 from armada.cli import CliAction
+from armada.exceptions.source_exceptions import InvalidPathException
 from armada.handlers.armada import Armada
+from armada.handlers.document import ReferenceResolver
 
 CONF = cfg.CONF
 
@@ -64,7 +66,7 @@ SHORT_DESC = "command install manifest charts"
 
 
 @apply.command(name='apply', help=DESC, short_help=SHORT_DESC)
-@click.argument('filename')
+@click.argument('locations', nargs=-1)
 @click.option('--api', help="Contacts service endpoint", is_flag=True)
 @click.option(
     '--disable-update-post', help="run charts without install", is_flag=True)
@@ -78,66 +80,35 @@ SHORT_DESC = "command install manifest charts"
 @click.option(
     '--tiller-port', help="Tiller host port", type=int, default=44134)
 @click.option(
-    '--timeout', help="specifies time to wait for charts", type=int,
+    '--timeout',
+    help="specifies time to wait for charts",
+    type=int,
     default=3600)
 @click.option('--values', '-f', multiple=True, type=str, default=[])
-@click.option(
+@click.option('--wait', help="wait until all charts deployed", is_flag=True)
-    '--wait', help="wait until all charts deployed", is_flag=True)
 @click.option(
     '--debug/--no-debug', help='Enable or disable debugging', default=False)
 @click.pass_context
-def apply_create(ctx,
+def apply_create(ctx, locations, api, disable_update_post, disable_update_pre,
-                 filename,
+                 dry_run, enable_chart_cleanup, set, tiller_host, tiller_port,
-                 api,
+                 timeout, values, wait, debug):
-                 disable_update_post,
-                 disable_update_pre,
-                 dry_run,
-                 enable_chart_cleanup,
-                 set,
-                 tiller_host,
-                 tiller_port,
-                 timeout,
-                 values,
-                 wait,
-                 debug):
 
     if debug:
         CONF.debug = debug
 
-    ApplyManifest(
+    ApplyManifest(ctx, locations, api, disable_update_post, disable_update_pre,
-        ctx,
+                  dry_run, enable_chart_cleanup, set, tiller_host, tiller_port,
-        filename,
+                  timeout, values, wait).invoke()
-        api,
-        disable_update_post,
-        disable_update_pre,
-        dry_run,
-        enable_chart_cleanup,
-        set,
-        tiller_host,
-        tiller_port,
-        timeout,
-        values,
-        wait).invoke()
 
 
 class ApplyManifest(CliAction):
-    def __init__(self,
+    def __init__(self, ctx, locations, api, disable_update_post,
-                 ctx,
+                 disable_update_pre, dry_run, enable_chart_cleanup, set,
-                 filename,
+                 tiller_host, tiller_port, timeout, values, wait):
-                 api,
-                 disable_update_post,
-                 disable_update_pre,
-                 dry_run,
-                 enable_chart_cleanup,
-                 set,
-                 tiller_host,
-                 tiller_port,
-                 timeout,
-                 values,
-                 wait):
         super(ApplyManifest, self).__init__()
         self.ctx = ctx
-        self.filename = filename
+        # Filename can also be a URL reference
+        self.locations = locations
         self.api = api
         self.disable_update_post = disable_update_post
         self.disable_update_pre = disable_update_pre
@@ -153,8 +124,7 @@ class ApplyManifest(CliAction):
     def output(self, resp):
         for result in resp:
             if not resp[result] and not result == 'diff':
-                self.logger.info(
+                self.logger.info('Did not performed chart %s(s)', result)
-                    'Did not performed chart %s(s)', result)
             elif result == 'diff' and not resp[result]:
                 self.logger.info('No Relase changes detected')
 
@@ -167,25 +137,32 @@ class ApplyManifest(CliAction):
                     self.logger.info(ch)
 
     def invoke(self):
-
         if not self.ctx.obj.get('api', False):
-            with open(self.filename) as f:
+            try:
+                doc_data = ReferenceResolver.resolve_reference(self.locations)
+                documents = list()
+                for d in doc_data:
+                    documents.extend(list(yaml.safe_load_all(d.decode())))
+            except InvalidPathException as ex:
+                self.logger.error(str(ex))
+                return
+            except yaml.YAMLError as yex:
+                self.logger.error("Invalid YAML found: %s" % str(yex))
+                return
+
             armada = Armada(
-                    list(yaml.safe_load_all(f.read())),
+                documents, self.disable_update_pre, self.disable_update_post,
-                    self.disable_update_pre,
+                self.enable_chart_cleanup, self.dry_run, self.set, self.wait,
-                    self.disable_update_post,
+                self.timeout, self.tiller_host, self.tiller_port, self.values)
-                    self.enable_chart_cleanup,
-                    self.dry_run,
-                    self.set,
-                    self.wait,
-                    self.timeout,
-                    self.tiller_host,
-                    self.tiller_port,
-                    self.values)
 
             resp = armada.sync()
             self.output(resp)
         else:
+            if len(self.values) > 0:
+                self.logger.error(
+                    "Cannot specify local values files when using the API.")
+                return
+
             query = {
                 'disable_update_post': self.disable_update_post,
                 'disable_update_pre': self.disable_update_pre,
@@ -199,8 +176,6 @@ class ApplyManifest(CliAction):
 
             client = self.ctx.obj.get('CLIENT')
 
-            with open(self.filename, 'r') as f:
             resp = client.post_apply(
-                    manifest=f.read(), values=self.values, set=self.set,
+                manifest_ref=self.locations, set=self.set, query=query)
-                    query=query)
             self.output(resp.get('message'))

armada/cli/validate.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 import click
 import yaml
 
@@ -20,6 +19,7 @@ from armada.cli import CliAction
 from armada.utils.lint import validate_armada_documents
 from armada.utils.lint import validate_armada_object
 from armada.handlers.manifest import Manifest
+from armada.handlers.document import ReferenceResolver
 
 
 @click.group()
@@ -42,38 +42,47 @@ SHORT_DESC = "command validates Armada Manifest"
 
 
 @validate.command(name='validate', help=DESC, short_help=SHORT_DESC)
-@click.argument('filename')
+@click.argument('locations', nargs=-1)
 @click.pass_context
-def validate_manifest(ctx, filename):
+def validate_manifest(ctx, locations):
-    ValidateManifest(ctx, filename).invoke()
+    ValidateManifest(ctx, locations).invoke()
 
 
 class ValidateManifest(CliAction):
-
+    def __init__(self, ctx, locations):
-    def __init__(self, ctx, filename):
         super(ValidateManifest, self).__init__()
         self.ctx = ctx
-        self.filename = filename
+        self.locations = locations
 
     def invoke(self):
         if not self.ctx.obj.get('api', False):
-            documents = yaml.safe_load_all(open(self.filename).read())
+            doc_data = ReferenceResolver.resolve_reference(self.locations)
+            documents = list()
+            for d in doc_data:
+                documents.extend(list(yaml.safe_load_all(d.decode())))
+
             manifest_obj = Manifest(documents).get_manifest()
             obj_check = validate_armada_object(manifest_obj)
             doc_check = validate_armada_documents(documents)
 
             try:
                 if doc_check and obj_check:
-                    self.logger.info(
+                    self.logger.info('Successfully validated: %s',
-                        'Successfully validated: %s', self.filename)
+                                     self.locations)
             except Exception:
-                raise Exception('Failed to validate: %s', self.filename)
+                raise Exception('Failed to validate: %s', self.locations)
         else:
+            if len(self.locations) > 1:
+                self.logger.error(
+                    "Cannot specify multiple locations "
+                    "when using validate API."
+                )
+                return
+
             client = self.ctx.obj.get('CLIENT')
-            with open(self.filename, 'r') as f:
+            resp = client.post_validate(self.locations[0])
-                resp = client.post_validate(f.read())
+
-                if resp.get('valid', False):
+            if resp.get('code') == 200:
-                    self.logger.info(
+                self.logger.info('Successfully validated: %s', self.locations)
-                        'Successfully validated: %s', self.filename)
             else:
-                    self.logger.error("Failed to validate: %s", self.filename)
+                self.logger.error("Failed to validate: %s", self.locations)

armada/common/client.py

@@ -27,7 +27,6 @@ API_VERSION = 'v{}/{}'
 
 
 class ArmadaClient(object):
-
     def __init__(self, session):
         self.session = session
 
@@ -55,22 +54,72 @@ class ArmadaClient(object):
     def post_validate(self, manifest=None):
 
         endpoint = self._set_endpoint('1.0', 'validate')
-        resp = self.session.post(endpoint, body=manifest)
+        # TODO(sh8121att) Look to update the UCP convention to
+        # allow a list of hrefs
+        req_body = {'href': manifest}
+
+        resp = self.session.post(
+            endpoint,
+            data=req_body,
+            headers={
+                'content-type': 'application/json'
+            })
 
         self._check_response(resp)
 
         return resp.json()
 
-    def post_apply(self, manifest=None, values=None, set=None, query=None):
+    def post_apply(self,
+                   manifest=None,
+                   manifest_ref=None,
+                   values=None,
+                   set=None,
+                   query=None):
+        """Call the Armada API to apply a manifest.
 
+        If manifest is not None, then the request body will be a fully
+        rendered set of YAML documents including overrides and
+        values-files application.
+
+        If manifest is None and manifest_ref is not, then the request
+        body will be a JSON structure providing a list of references
+        to Armada manifest documents and a list of overrides. Local
+        values files are not supported when using the API with references.
+
+        :param manifest: string of YAML formatted Armada manifests
+        :param manifest_ref: valid file paths or URIs referring to Armada
+                             manifests
+        :param values: list of local files containing values.yaml overrides
+        :param set: list of single-value overrides
+        :param query: explicit query string parameters
+        """
+        endpoint = self._set_endpoint('1.0', 'apply')
+
+        if manifest:
             if values or set:
                 document = list(yaml.safe_load_all(manifest))
                 override = Override(
                     document, overrides=set, values=values).update_manifests()
                 manifest = yaml.dump(override)
-
+            resp = self.session.post(
-        endpoint = self._set_endpoint('1.0', 'apply')
+                endpoint,
-        resp = self.session.post(endpoint, body=manifest, query=query)
+                body=manifest,
+                query=query,
+                headers={
+                    'content-type': 'application/x-yaml'
+                })
+        elif manifest_ref:
+            req_body = {
+                'hrefs': manifest_ref,
+                'overrides': set or [],
+            }
+            resp = self.session.post(
+                endpoint,
+                data=req_body,
+                query=query,
+                headers={
+                    'content-type': 'application/json'
+                })
 
         self._check_response(resp)
 
@@ -100,8 +149,7 @@ class ArmadaClient(object):
                 "Unauthorized access to %s, include valid token.".format(
                     resp.url))
         elif resp.status_code == 403:
-            raise err.ClientForbiddenError(
+            raise err.ClientForbiddenError("Forbidden access to %s" % resp.url)
-                "Forbidden access to %s" % resp.url)
         elif not resp.ok:
-            raise err.ClientError(
+            raise err.ClientError("Error - received %d: %s" %
-                "Error - received %d: %s" % (resp.status_code, resp.text))
+                                  (resp.status_code, resp.text))

armada/common/session.py

@@ -55,22 +55,23 @@ class ArmadaSession(object):
         self.logger = LOG
 
     # TODO Add keystone authentication to produce a token for this session
-    def get(self, endpoint, query=None):
+    def get(self, endpoint, query=None, headers=None):
         """
         Send a GET request to armada.
 
         :param string endpoint: URL string following hostname and API prefix
         :param dict query: A dict of k, v pairs to add to the query string
+        :param headers: Dictionary of HTTP headers to include in request
 
         :return: A requests.Response object
         """
         api_url = '{}{}'.format(self.base_url, endpoint)
         resp = self._session.get(
-            api_url, params=query, timeout=3600)
+            api_url, params=query, headers=headers, timeout=3600)
 
         return resp
 
-    def post(self, endpoint, query=None, body=None, data=None):
+    def post(self, endpoint, query=None, body=None, data=None, headers=None):
         """
         Send a POST request to armada. If both body and data are specified,
         body will will be used.
@@ -79,6 +80,7 @@ class ArmadaSession(object):
         :param dict query: dict of k, v parameters to add to the query string
         :param string body: string to use as the request body.
         :param data: Something json.dumps(s) can serialize.
+        :param headers: Dictionary of HTTP headers to include in request
         :return: A requests.Response object
         """
         api_url = '{}{}'.format(self.base_url, endpoint)
@@ -86,11 +88,17 @@ class ArmadaSession(object):
         self.logger.debug("Sending POST with armada_client session")
         if body is not None:
             self.logger.debug("Sending POST with explicit body: \n%s" % body)
-            resp = self._session.post(
+            resp = self._session.post(api_url,
-                api_url, params=query, data=body, timeout=3600)
+                                      params=query,
+                                      data=body,
+                                      headers=headers,
+                                      timeout=3600)
         else:
             self.logger.debug("Sending POST with JSON body: \n%s" % str(data))
-            resp = self._session.post(
+            resp = self._session.post(api_url,
-                api_url, params=query, json=data, timeout=3600)
+                                      params=query,
+                                      json=data,
+                                      headers=headers,
+                                      timeout=3600)
 
         return resp

armada/conf/__init__.py

@@ -24,6 +24,9 @@ CONF = cfg.CONF
 
 CONFIG_FILES = ['api-paste.ini', 'armada.conf']
 
+# Load oslo_log options prior to file/CLI parsing
+log.register_options(CONF)
+
 
 def _get_config_files(env=None):
     if env is None:

armada/handlers/document.py

@@ -0,0 +1,140 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for resolving design references."""
+
+import urllib.parse
+import re
+import requests
+
+from oslo_log import log as logging
+
+from armada.exceptions.source_exceptions import InvalidPathException
+from armada.utils.keystone import KeystoneUtils
+
+LOG = logging.getLogger(__name__)
+
+
+class ReferenceResolver(object):
+    """Class for handling different data references to resolve them data."""
+
+    @classmethod
+    def resolve_reference(cls, design_ref):
+        """Resolve a reference to a design document.
+
+        Locate a schema handler based on the URI scheme of the data reference
+        and use that handler to get the data referenced.
+
+        :param design_ref: A list of URI-formatted reference to a data entity
+        :returns: A list of byte arrays
+        """
+        data = []
+        if isinstance(design_ref, str):
+            design_ref = [design_ref]
+
+        for l in design_ref:
+            try:
+                LOG.debug("Resolving reference %s." % l)
+                design_uri = urllib.parse.urlparse(l)
+
+                # when scheme is a empty string assume it is a local
+                # file path
+                if design_uri.scheme == '':
+                    handler = cls.scheme_handlers.get('file')
+                else:
+                    handler = cls.scheme_handlers.get(design_uri.scheme, None)
+
+                if handler is None:
+                    raise InvalidPathException(
+                        "Invalid reference scheme %s: no handler." %
+                        design_uri.scheme)
+                else:
+                    # Have to do a little magic to call the classmethod
+                    # as a pointer
+                    data.append(handler.__get__(None, cls)(design_uri))
+            except ValueError:
+                raise InvalidPathException(
+                    "Cannot resolve design reference %s: unable "
+                    "to parse as valid URI."
+                    % l)
+
+        return data
+
+    @classmethod
+    def resolve_reference_http(cls, design_uri):
+        """Retrieve design documents from http/https endpoints.
+
+        Return a byte array of the response content. Support
+        unsecured or basic auth
+
+        :param design_uri: Tuple as returned by urllib.parse
+                            for the design reference
+        """
+        if design_uri.username is not None and design_uri.password is not None:
+            response = requests.get(
+                design_uri.geturl(),
+                auth=(design_uri.username, design_uri.password),
+                timeout=30)
+        else:
+            response = requests.get(design_uri.geturl(), timeout=30)
+            if response.status_code >= 400:
+                raise InvalidPathException(
+                    "Error received for HTTP reference: %d"
+                    % response.status_code)
+
+        return response.content
+
+    @classmethod
+    def resolve_reference_file(cls, design_uri):
+        """Retrieve design documents from local file endpoints.
+
+        Return a byte array of the file contents
+
+        :param design_uri: Tuple as returned by urllib.parse for the design
+                           reference
+        """
+        if design_uri.path != '':
+            with open(design_uri.path, 'rb') as f:
+                doc = f.read()
+            return doc
+
+    @classmethod
+    def resolve_reference_ucp(cls, design_uri):
+        """Retrieve artifacts from a UCP service endpoint.
+
+        Return a byte array of the response content. Assumes Keystone
+        authentication required.
+
+        :param design_uri: Tuple as returned by urllib.parse for the design
+                           reference
+        """
+        ks_sess = KeystoneUtils.get_session()
+        (new_scheme, foo) = re.subn('^[^+]+\+', '', design_uri.scheme)
+        url = urllib.parse.urlunparse(
+            (new_scheme, design_uri.netloc, design_uri.path, design_uri.params,
+             design_uri.query, design_uri.fragment))
+        LOG.debug("Calling Keystone session for url %s" % str(url))
+        resp = ks_sess.get(url)
+        if resp.status_code >= 400:
+            raise InvalidPathException(
+                "Received error code for reference %s: %s - %s" %
+                (url, str(resp.status_code), resp.text))
+        return resp.content
+
+    scheme_handlers = {
+        'http': resolve_reference_http,
+        'file': resolve_reference_file,
+        'https': resolve_reference_http,
+        'deckhand+http': resolve_reference_ucp,
+        'promenade+http': resolve_reference_ucp,
+    }

armada/handlers/override.py

@@ -31,7 +31,6 @@ class Override(object):
         '''
         Retrieve yaml file as a dictionary.
         '''
-
         try:
             with open(doc) as f:
                 return list(yaml.safe_load_all(f.read()))

armada/tests/unit/api/test_armada_controller.py

@@ -17,7 +17,7 @@ import mock
 
 from oslo_config import cfg
 
-from armada.handlers import armada
+from armada.api.controller import armada as armada_api
 from armada.tests.unit.api import base
 
 CONF = cfg.CONF
@@ -25,11 +25,9 @@ CONF = cfg.CONF
 
 class ArmadaControllerTest(base.BaseControllerTest):
 
-    @mock.patch.object(armada, 'lint')
+    @mock.patch.object(armada_api, 'Armada')
-    @mock.patch.object(armada, 'Manifest')
+    @mock.patch.object(armada_api, 'ReferenceResolver')
-    @mock.patch.object(armada, 'Tiller')
+    def test_armada_apply_resource(self, mock_resolver, mock_armada):
-    def test_armada_apply_resource(self, mock_tiller, mock_manifest,
-                                   mock_lint):
         """Tests the POST /api/v1.0/apply endpoint."""
         rules = {'armada:create_endpoints': '@'}
         self.policy.set_rules(rules)
@@ -42,17 +40,62 @@ class ArmadaControllerTest(base.BaseControllerTest):
                    'dry_run': 'false',
                    'wait': 'false',
                    'timeout': '100'}
-        payload = {'file': '', 'options': options}
+
+        armada_options = {
+            'disable_update_pre': False,
+            'disable_update_post': False,
+            'enable_chart_cleanup': False,
+            'dry_run': False,
+            'wait': False,
+            'timeout': 100,
+            'tiller_host': None,
+            'tiller_port': 44134,
+        }
+
+        payload_url = 'http://foo.com/test.yaml'
+        payload = {'hrefs': [payload_url]}
         body = json.dumps(payload)
         expected = {'message': {'diff': [], 'install': [], 'upgrade': []}}
 
-        result = self.app.simulate_post(path='/api/v1.0/apply', body=body)
+        mock_resolver.resolve_reference.return_value = \
+            [b"---\nfoo: bar"]
+
+        mock_armada.return_value.sync.return_value = \
+            {'diff': [], 'install': [], 'upgrade': []}
+
+        result = self.app.simulate_post(path='/api/v1.0/apply',
+                                        body=body,
+                                        headers={
+                                            'Content-Type': 'application/json'
+                                        },
+                                        params=options)
         self.assertEqual(result.json, expected)
         self.assertEqual('application/json', result.headers['content-type'])
 
-        mock_tiller.assert_called_once_with(tiller_host=None,
+        mock_resolver.resolve_reference.assert_called_with([payload_url])
-                                            tiller_port=44134)
+        mock_armada.assert_called_with([{'foo': 'bar'}], **armada_options)
-        mock_manifest.assert_called_once_with([payload])
+        mock_armada.return_value.sync.assert_called()
-        mock_lint.validate_armada_documents.assert_called_once_with([payload])
+
-        fake_manifest = mock_manifest.return_value.get_manifest.return_value
+    def test_armada_apply_no_href(self):
-        mock_lint.validate_armada_object.assert_called_once_with(fake_manifest)
+        """Tests /api/v1.0/apply returns 400 when hrefs list is empty."""
+        rules = {'armada:create_endpoints': '@'}
+        self.policy.set_rules(rules)
+
+        options = {'debug': 'true',
+                   'disable_update_pre': 'false',
+                   'disable_update_post': 'false',
+                   'enable_chart_cleanup': 'false',
+                   'skip_pre_flight': 'false',
+                   'dry_run': 'false',
+                   'wait': 'false',
+                   'timeout': '100'}
+        payload = {'hrefs': []}
+        body = json.dumps(payload)
+
+        result = self.app.simulate_post(path='/api/v1.0/apply',
+                                        body=body,
+                                        headers={
+                                            'Content-Type': 'application/json'
+                                        },
+                                        params=options)
+        self.assertEqual(result.status_code, 400)

armada/tests/unit/utils/test_lint.py

@@ -152,3 +152,17 @@ class LintTestCase(unittest.TestCase):
         document = yaml.safe_load_all(template_manifest)
         with self.assertRaises(Exception):
             lint.validate_chart_document(document)
+
+    def test_lint_validate_manifest_url(self):
+        value = 'url'
+        assert lint.validate_manifest_url(value) is False
+        value = 'https://raw.githubusercontent.com/att-comdev/' \
+                'armada/master/examples/simple.yaml'
+        assert lint.validate_manifest_url(value) is True
+
+    def test_lint_validate_manifest_filepath(self):
+        value = 'filepath'
+        assert lint.validate_manifest_filepath(value) is False
+        value = '{}/templates/valid_armada_document.yaml'.format(
+            self.basepath)
+        assert lint.validate_manifest_filepath(value) is True

armada/utils/keystone.py

@@ -0,0 +1,64 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Utility functions for accessing Openstack Keystone."""
+
+import os
+
+from keystoneauth1.identity import v3
+from keystoneauth1 import session
+from oslo_config import cfg
+
+
+CONF = cfg.CONF
+
+
+class KeystoneUtils(object):
+    """Utility methods for using Keystone."""
+
+    @staticmethod
+    def get_session():
+        """Get an initialized keystone session.
+
+        Authentication is based on the keystone_authtoken
+        section of the config file primarily. If that fails
+        then attempt to create a session from environmental
+        variables. This is for cases of the CLI needing
+        a token.
+        """
+        auth_info = dict()
+        auth_fields = ['auth_url', 'username', 'password', 'project_id',
+                       'user_domain_name']
+        try:
+            for f in auth_fields:
+                auth_info[f] = getattr(CONF.keystone_authtoken, f)
+            auth = v3.Password(**auth_info)
+            ks_session = session.Session(auth=auth)
+            # Test the session
+            ks_session.get_auth_headers()
+        except Exception:  # nosec this isn't a security issue
+            pass
+        else:
+            return ks_session
+
+        try:
+            for f in auth_fields:
+                auth_info[f] = os.environ.get('os_{}'.format(f).upper())
+            auth = v3.Password(**auth_info)
+            ks_session = session.Session(auth=auth)
+            # Test the session
+            ks_session.get_auth_headers()
+        except Exception:
+            raise Exception('Missing credential information for Keystone.')
+
+        return ks_session

armada/utils/lint.py

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import requests
+
 from armada.const import DOCUMENT_CHART, DOCUMENT_GROUP, DOCUMENT_MANIFEST
 from armada.const import KEYWORD_ARMADA, KEYWORD_PREFIX, KEYWORD_GROUPS, \
     KEYWORD_CHARTS, KEYWORD_RELEASE
@@ -93,3 +96,14 @@ def validate_chart_document(documents):
                         KEYWORD_RELEASE, document.get('metadata').get('name')))
 
     return True
+
+
+def validate_manifest_url(value):
+    try:
+        return (requests.get(value).status_code == 200)
+    except:
+        return False
+
+
+def validate_manifest_filepath(value):
+    return os.path.isfile(value)

tox.ini

@@ -40,6 +40,9 @@ commands =
   bandit -r armada -x armada/tests -n 5
 
 [testenv:coverage]
+passenv=http_proxy https_proxy no_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY
+setenv=
+    VIRTUAL_ENV={envdir}
 commands =
   python -m pytest \
     --cov-branch \