[Feat] Support remote tarball as chart source

-Add functionality to download a tarball from a URL, decompress/extract
 the archive, and use as a chart source
-Compartmentalized functionality to later support extracting, but not
 downloading, local tarballs
-Refactor specific git utils to general source utils
-Small exception handling bug fix

armada/exceptions/armada_exceptions.py

@@ -23,15 +23,3 @@ class KnownReleasesException(ArmadaException):
     '''Exception that occurs when no known releases are found'''
 
     message = 'No known releases found'
-
-class ChartSourceException(ArmadaException):
-    '''Exception for unknown chart source type.'''
-
-    def __init__(self, chart_name, source_type):
-        self._chart_name = chart_name
-        self._source_type = source_type
-
-        self._message = 'Unknown source type \"' + self._source_type + '\" for \
-                        chart \"' + self._chart_name + '\"'
-
-        super(ChartSourceException, self).__init__(self._message)

armada/exceptions/chartbuilder_exceptions.py

@@ -23,7 +23,7 @@ class DependencyException(ChartBuilderException):
     '''Exception that occurs when dependencies cannot be resolved.'''
 
     def __init__(self, chart_name):
-        self._chart_name
+        self._chart_name = chart_name
         self._message = 'Failed to resolve dependencies for ' + \
                         self._chart_name + '.'
 
@@ -33,7 +33,7 @@ class HelmChartBuildException(ChartBuilderException):
     '''Exception that occurs when Helm Chart fails to build.'''
 
     def __init__(self, chart_name):
-        self._chart_name
+        self._chart_name = chart_name
         self._message = 'Failed to build Helm chart for ' + \
                         self._chart_name + '.'
 

armada/exceptions/git_exceptions.py

@@ -1,38 +0,0 @@
-# Copyright 2017 The Armada Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import base_exception
-
-class GitException(base_exception.ArmadaBaseException):
-    '''Base class for Git exceptions and error handling.'''
-
-    message = 'An unknown error occured while cloning a Git repository.'
-
-class GitLocationException(GitException):
-    '''Exception that occurs when an error occurs cloning a Git repository.'''
-
-    def __init__(self, location):
-        self._location = location
-        self._message = self._location + ' is not a valid git repository.'
-
-        super(GitLocationException, self).__init__(self._message)
-
-class SourceCleanupException(GitException):
-    '''Exception that occurs for an invalid dir.'''
-
-    def __init__(self, target_dir):
-        self._target_dir = target_dir
-        self._message = self._target_dir + ' is not a valid directory.'
-
-        super(SourceCleanupException, self).__init__(self._message)

armada/exceptions/source_exceptions.py

@@ -0,0 +1,80 @@
+# Copyright 2017 The Armada Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base_exception
+
+class SourceException(base_exception.ArmadaBaseException):
+    '''Base class for Git exceptions and error handling.'''
+
+    message = 'An unknown error occured while accessing a chart source'
+
+class GitLocationException(SourceException):
+    '''Exception that occurs when an error occurs cloning a Git repository.'''
+
+    def __init__(self, location):
+        self._location = location
+        self._message = self._location + ' is not a valid git repository.'
+
+        super(GitLocationException, self).__init__(self._message)
+
+class SourceCleanupException(SourceException):
+    '''Exception that occurs for an invalid dir.'''
+
+    def __init__(self, target_dir):
+        self._target_dir = target_dir
+        self._message = self._target_dir + ' is not a valid directory.'
+
+        super(SourceCleanupException, self).__init__(self._message)
+
+class TarballDownloadException(SourceException):
+    '''Exception that occurs when the tarball cannot be downloaded
+        from the provided URL
+    '''
+
+    def __init__(self, tarball_url):
+        self._tarball_url = tarball_url
+        self._message = 'Unable to download from ' + self._tarball_url
+
+        super(TarballDownloadException, self).__init__(self._message)
+
+class TarballExtractException(SourceException):
+    '''Exception that occurs when extracting the tarball fails'''
+
+    def __init__(self, tarball_dir):
+        self._tarball_dir = tarball_dir
+        self._message = 'Unable to extract ' + self._tarball_dir
+
+        super(TarballExtractException, self).__init__(self._message)
+
+class InvalidPathException(SourceException):
+    '''Exception that occurs when a nonexistant path is accessed'''
+
+    def __init__(self, path):
+        self._path = path
+        self._message = 'Unable to access path ' + self._path
+
+        super(InvalidPathException, self).__init__(self._message)
+
+
+class ChartSourceException(SourceException):
+    '''Exception for unknown chart source type.'''
+
+    def __init__(self, chart_name, source_type):
+        self._chart_name = chart_name
+        self._source_type = source_type
+
+        self._message = 'Unknown source type \"' + self._source_type + '\" for \
+                        chart \"' + self._chart_name + '\"'
+
+        super(ChartSourceException, self).__init__(self._message)

armada/handlers/armada.py

@@ -24,12 +24,12 @@ from tiller import Tiller
 from manifest import Manifest
 
 from ..exceptions import armada_exceptions
-from ..exceptions import git_exceptions
+from ..exceptions import source_exceptions
 from ..exceptions import lint_exceptions
 from ..exceptions import tiller_exceptions
 
 from ..utils.release import release_prefix
-from ..utils import git
+from ..utils import source
 from ..utils import lint
 from ..const import KEYWORD_ARMADA, KEYWORD_GROUPS, KEYWORD_CHARTS,\
     KEYWORD_PREFIX, STATUS_FAILED
@@ -142,20 +142,24 @@ class Armada(object):
 
         if ct_type == 'local':
             ch.get('chart')['source_dir'] = (location, subpath)
+        elif ct_type == 'tar':
+            LOG.info('Downloading tarball from: %s', location)
+            tarball_dir = source.get_tarball(location)
+            ch.get('chart')['source_dir'] = (tarball_dir, subpath)
         elif ct_type == 'git':
             if location not in repos.keys():
                 try:
                     LOG.info('Cloning repo: %s', location)
-                    repo_dir = git.git_clone(location, reference)
+                    repo_dir = source.git_clone(location, reference)
                 except Exception:
-                    raise git_exceptions.GitLocationException(location)
+                    raise source_exceptions.GitLocationException(location)
                 repos[location] = repo_dir
                 ch.get('chart')['source_dir'] = (repo_dir, subpath)
             else:
                 ch.get('chart')['source_dir'] = (repos.get(location), subpath)
         else:
             chart_name = ch.get('chart').get('chart_name')
-            raise armada_exceptions.ChartSourceException(ct_type, chart_name)
+            raise source_exceptions.ChartSourceException(ct_type, chart_name)
 
     def get_releases_by_status(self, status):
         '''
@@ -305,7 +309,7 @@ class Armada(object):
         for group in self.config.get(KEYWORD_ARMADA).get(KEYWORD_GROUPS):
             for ch in group.get(KEYWORD_CHARTS):
                 if ch.get('chart').get('source').get('type') == 'git':
-                    git.source_cleanup(ch.get('chart').get('source_dir')[0])
+                    source.source_cleanup(ch.get('chart').get('source_dir')[0])
 
     def show_diff(self, chart, installed_chart, installed_values, target_chart,
                   target_values):

armada/handlers/chartbuilder.py

@@ -202,7 +202,7 @@ class ChartBuilder(object):
                 templates=self.get_templates(),
                 dependencies=dependencies,
                 values=self.get_values(),
-                files=self.get_files(), )
+                files=self.get_files())
         except Exception:
             chart_name = self.chart.chart_name
             raise chartbuilder_exceptions.HelmChartBuildException(chart_name)

armada/tests/unit/utils/test_git.py

@@ -1,67 +0,0 @@
-# Copyright 2017 The Armada Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import mock
-import unittest
-
-from armada.exceptions import git_exceptions
-
-from armada.utils import git
-
-class GitTestCase(unittest.TestCase):
-
-    @mock.patch('armada.utils.git.tempfile')
-    @mock.patch('armada.utils.git.pygit2')
-    def test_git_clone_good_url(self, mock_pygit, mock_temp):
-        mock_temp.mkdtemp.return_value = '/tmp/armada'
-        mock_pygit.clone_repository.return_value = "Repository"
-        url = 'http://github.com/att-comdev/armada'
-        dir = git.git_clone(url)
-
-        self.assertIsNotNone(dir)
-
-    def test_git_clone_empty_url(self):
-        url = ''
-
-        with self.assertRaises(Exception):
-            self.assertFalse(git.git_clone(url))
-
-    def test_git_clone_bad_url(self):
-        url = 'http://github.com/dummy/armada'
-
-        with self.assertRaises(Exception):
-            git.git_clone(url)
-
-    @mock.patch('armada.utils.git.shutil')
-    @mock.patch('armada.utils.git.path')
-    def test_source_cleanup(self, mock_path, mock_shutil):
-        mock_path.exists.return_value = True
-        path = 'armada'
-
-        try:
-            git.source_cleanup(path)
-        except git_exceptions.SourceCleanupException:
-            pass
-
-        mock_shutil.rmtree.assert_called_with(path)
-
-    @unittest.skip('not handled correctly')
-    @mock.patch('armada.utils.git.shutil')
-    @mock.patch('armada.utils.git.path')
-    def test_source_cleanup_bad_path(self, mock_path, mock_shutil):
-        mock_path.exists.return_value = False
-        path = 'armada'
-        with self.assertRaises(Exception):
-            git.source_cleanup(path)
-        mock_shutil.rmtree.assert_not_called()

armada/tests/unit/utils/test_source.py

@@ -0,0 +1,117 @@
+# Copyright 2017 The Armada Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import unittest
+
+from armada.exceptions import source_exceptions
+
+from armada.utils import source
+
+class GitTestCase(unittest.TestCase):
+
+    SOURCE_UTILS_LOCATION = 'armada.utils.source'
+
+    @mock.patch('armada.utils.source.tempfile')
+    @mock.patch('armada.utils.source.pygit2')
+    def test_git_clone_good_url(self, mock_pygit, mock_temp):
+        mock_temp.mkdtemp.return_value = '/tmp/armada'
+        mock_pygit.clone_repository.return_value = "Repository"
+        url = 'http://github.com/att-comdev/armada'
+        dir = source.git_clone(url)
+
+        self.assertIsNotNone(dir)
+
+    def test_git_clone_empty_url(self):
+        url = ''
+
+        with self.assertRaises(Exception):
+            self.assertFalse(source.git_clone(url))
+
+    def test_git_clone_bad_url(self):
+        url = 'http://github.com/dummy/armada'
+
+        with self.assertRaises(Exception):
+            source.git_clone(url)
+
+    @mock.patch('armada.utils.source.tempfile')
+    @mock.patch('armada.utils.source.requests')
+    def test_tarball_download(self, mock_requests, mock_temp):
+        url = 'http://localhost:8879/charts/mariadb-0.1.0.tgz'
+        mock_temp.mkstemp.return_value = (None, '/tmp/armada')
+        mock_response = mock.Mock()
+        mock_response.content = 'some string'
+        mock_requests.get.return_value = mock_response
+
+        mock_open = mock.mock_open()
+        with mock.patch('{}.open'.format(self.SOURCE_UTILS_LOCATION),
+                        mock_open, create=True):
+            source.download_tarball(url)
+
+        mock_temp.mkstemp.assert_called_once()
+        mock_requests.get.assert_called_once_with(url)
+        mock_open.assert_called_once_with('/tmp/armada', 'wb')
+        mock_open().write.assert_called_once_with(
+            mock_requests.get(url).content)
+
+    @mock.patch('armada.utils.source.tempfile')
+    @mock.patch('armada.utils.source.path')
+    @mock.patch('armada.utils.source.tarfile')
+    def test_tarball_extract(self, mock_tarfile, mock_path, mock_temp):
+        mock_path.exists.return_value = True
+        mock_temp.mkdtemp.return_value = '/tmp/armada'
+        mock_opened_file = mock.Mock()
+        mock_tarfile.open.return_value = mock_opened_file
+
+        path = '/tmp/mariadb-0.1.0.tgz'
+        source.extract_tarball(path)
+
+        mock_path.exists.assert_called_once()
+        mock_temp.mkdtemp.assert_called_once()
+        mock_tarfile.open.assert_called_once_with(path)
+        mock_opened_file.extractall.assert_called_once_with('/tmp/armada')
+
+    @mock.patch('armada.utils.source.path')
+    @mock.patch('armada.utils.source.tarfile')
+    def test_tarball_extract_bad_path(self, mock_tarfile, mock_path):
+        mock_path.exists.return_value = False
+        path = '/tmp/armada'
+        with self.assertRaises(Exception):
+            source.extract_tarball(path)
+
+        mock_tarfile.open.assert_not_called()
+        mock_tarfile.extractall.assert_not_called()
+
+    @mock.patch('armada.utils.source.shutil')
+    @mock.patch('armada.utils.source.path')
+    def test_source_cleanup(self, mock_path, mock_shutil):
+        mock_path.exists.return_value = True
+        path = 'armada'
+
+        try:
+            source.source_cleanup(path)
+        except source_exceptions.SourceCleanupException:
+            pass
+
+        mock_shutil.rmtree.assert_called_with(path)
+
+    @unittest.skip('not handled correctly')
+    @mock.patch('armada.utils.source.shutil')
+    @mock.patch('armada.utils.source.path')
+    def test_source_cleanup_bad_path(self, mock_path, mock_shutil):
+        mock_path.exists.return_value = False
+        path = 'armada'
+        with self.assertRaises(Exception):
+            source.source_cleanup(path)
+        mock_shutil.rmtree.assert_not_called()

armada/utils/git.py

@@ -1,30 +0,0 @@
-import pygit2
-import tempfile
-import shutil
-from os import path
-
-from ..exceptions import git_exceptions
-
-def git_clone(repo_url, branch='master'):
-    '''
-    clones repo to a /tmp/ dir
-    '''
-
-    if repo_url == '':
-        raise git_exceptions.GitLocationException(repo_url)
-
-    _tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
-
-    try:
-        pygit2.clone_repository(repo_url, _tmp_dir, checkout_branch=branch)
-    except Exception:
-        raise git_exceptions.GitLocationException(repo_url)
-
-    return _tmp_dir
-
-def source_cleanup(target_dir):
-    '''
-    Clean up source
-    '''
-    if path.exists(target_dir):
-        shutil.rmtree(target_dir)

armada/utils/source.py

@@ -0,0 +1,65 @@
+import pygit2
+import requests
+import tarfile
+import tempfile
+import shutil
+from os import path
+
+from ..exceptions import source_exceptions
+
+def git_clone(repo_url, branch='master'):
+    '''
+    clones repo to a /tmp/ dir
+    '''
+
+    if repo_url == '':
+        raise source_exceptions.GitLocationException(repo_url)
+
+    _tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
+
+    try:
+        pygit2.clone_repository(repo_url, _tmp_dir, checkout_branch=branch)
+    except Exception:
+        raise source_exceptions.GitLocationException(repo_url)
+
+    return _tmp_dir
+
+def get_tarball(tarball_url):
+    tarball_path = download_tarball(tarball_url)
+    return extract_tarball(tarball_path)
+
+def download_tarball(tarball_url):
+    '''
+    Downloads a tarball to /tmp and returns the path
+    '''
+    try:
+        tarball_filename = tempfile.mkstemp(prefix='armada', dir='/tmp')[1]
+        response = requests.get(tarball_url)
+        with open(tarball_filename, 'wb') as f:
+            f.write(response.content)
+    except Exception:
+        raise source_exceptions.TarballDownloadException(tarball_url)
+    return tarball_filename
+
+def extract_tarball(tarball_path):
+    '''
+    Extracts a tarball to /tmp and returns the path
+    '''
+    if not path.exists(tarball_path):
+        raise source_exceptions.InvalidPathException(tarball_path)
+
+    _tmp_dir = tempfile.mkdtemp(prefix='armada', dir='/tmp')
+
+    try:
+        file = tarfile.open(tarball_path)
+        file.extractall(_tmp_dir)
+    except Exception:
+        raise source_exceptions.TarballExtractException(tarball_path)
+    return _tmp_dir
+
+def source_cleanup(target_dir):
+    '''
+    Clean up source
+    '''
+    if path.exists(target_dir):
+        shutil.rmtree(target_dir)

docs/source/operations/guide-build-armada-yaml.rst

@@ -151,7 +151,7 @@ Source
 +-------------+----------+---------------------------------------------------------------+
 | keyword     | type     | action                                                        |
 +=============+==========+===============================================================+
-| type        | string   | source to build the chart: ``git`` or ``local``               |
+| type        | string   | source to build the chart: ``git``, ``local``, or ``tar``     |
 +-------------+----------+---------------------------------------------------------------+
 | location    | string   | ``url`` or ``path`` to the chart's parent directory           |
 +-------------+----------+---------------------------------------------------------------+
@@ -247,9 +247,9 @@ Simple Example
       namespace: default
       values: {}
       source:
-        type: git
+        type: tar
-        location: https://github.com/namespace/repo
+        location: http://localhost:8879/namespace/repo
-        subpath: .
+        subpath: blog-2
         reference: master
       dependencies: []
     ---

docs/source/operations/guide-exceptions.rst

@@ -4,8 +4,6 @@ Armada Exceptions
 +------------------------+----------------------------------------------------------+
 | Exception              | Error Description                                        |
 +========================+==========================================================+
-| ChartSourceException   | Occurs when an unknown chart source type is encountered. |
-+------------------------+----------------------------------------------------------+
 | KnownReleasesException | Occurs when no known releases are found.                 |
 +------------------------+----------------------------------------------------------+
 
@@ -53,16 +51,24 @@ Chartbuilder Exceptions
 | UnknownChartSourceException | The chart source is unknown or invalid.                     |
 +-----------------------------+-------------------------------------------------------------+
 
-Git Exceptions
+Source Exceptions
 ===============
 
-+------------------------+---------------------------------------------+
++--------------------------+---------------------------------------------------------------------+
 | Exception                | Error Description                                                   |
-+========================+=============================================+
++==========================+=====================================================================+
 | GitLocationException     | Repository location is not valid.                                   |
-+------------------------+---------------------------------------------+
++--------------------------+---------------------------------------------------------------------+
 | SourceCleanupException   | The source dir of a chart no longer exists.                         |
-+------------------------+---------------------------------------------+
++--------------------------+---------------------------------------------------------------------+
+| TarballDownloadException | Occurs when the tarball cannot be downloaded from the provided URL. |
++--------------------------+---------------------------------------------------------------------+
+| TarballExtractException  | Occurs when extracting a tarball fails.                             |
++--------------------------+---------------------------------------------------------------------+
+| InvalidPathException     | Occurs when a nonexistant path is accessed.                         |
++--------------------------+---------------------------------------------------------------------+
+| ChartSourceException     | Occurs when an unknown chart source type is encountered.            |
++--------------------------+---------------------------------------------------------------------+
 
 Lint Exceptions
 ===============

examples/tar_example.yaml

@@ -0,0 +1,57 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: helm-toolkit
+data:
+  chart_name: helm-toolkit
+  release: helm-toolkit
+  namespace: helm-tookit
+  values: {}
+  source:
+    type: git
+    location: git://github.com/openstack/openstack-helm
+    subpath: helm-toolkit
+    reference: master
+  dependencies: []
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: mariadb-tarball
+data:
+  chart_name: mariadb-tarball
+  release: mariadb-tarball
+  namespace: tarball
+  timeout: 3600
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+  values: {}
+  source:
+    type: tar
+    location: http://localhost:8879/charts/mariadb-0.1.0.tgz
+    subpath: mariadb
+    reference: null
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: tar-example
+data:
+  description: "Deploying mariadb tarball URL"
+  sequenced: True
+  chart_group:
+    - mariadb-tarball
+---
+schema: armada/Manifest/v1
+metadata:
+  schema: metadata/Document/v1
+  name: armada-manifest
+data:
+  release_prefix: armada
+  chart_groups:
+    - tar-example