airship/armada
Code Issues Proposed changes

feat(armada): adding helm testing framework

Browse Source 
- added helm test framework to armada
- added helm test status

Closes #151

Change-Id: I417cae04b4595ad0d4fd05889d90c83907607c47
This commit is contained in:
Alexis Rivera De La Torre 2017-08-07 18:22:09 +00:00 committed by gardlt
parent 70e95d64f5
commit d5f4378731
11 changed files with 447 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
armada/cli/test.py Normal file
View File

@ -0,0 +1,107 @@
# Copyright 2017 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import yaml
from cliff import command as cmd
from oslo_config import cfg
from oslo_log import log as logging
from armada import const
from armada.handlers.manifest import Manifest
from armada.handlers.tiller import Tiller
from armada.utils.release import release_prefix
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
def testService(args):
tiller = Tiller(tiller_host=args.tiller_host, tiller_port=args.tiller_port)
known_release_names = [release[0] for release in tiller.list_charts()]
if args.release:
LOG.info("RUNNING: %s tests", args.release)
resp = tiller.testing_release(args.release)
if not resp:
LOG.info("FAILED: %s", args.release)
return
test_status = getattr(resp.info.status, 'last_test_suite_run',
'FAILED')
if test_status.results[0].status:
LOG.info("PASSED: %s", args.release)
else:
LOG.info("FAILED: %s", args.release)
if args.file:
documents = yaml.safe_load_all(open(args.file).read())
armada_obj = Manifest(documents).get_manifest()
prefix = armada_obj.get(const.KEYWORD_ARMADA).get(const.KEYWORD_PREFIX)
for group in armada_obj.get(const.KEYWORD_ARMADA).get(
const.KEYWORD_GROUPS):
for ch in group.get(const.KEYWORD_CHARTS):
release_name = release_prefix(
prefix, ch.get('chart').get('chart_name'))
if release_name in known_release_names:
LOG.info('RUNNING: %s tests', release_name)
resp = tiller.testing_release(release_name)
if not resp:
continue
test_status = getattr(resp.info.status,
'last_test_suite_run', 'FAILED')
if test_status.results[0].status:
LOG.info("PASSED: %s", release_name)
else:
LOG.info("FAILED: %s", release_name)
else:
LOG.info('Release %s not found - SKIPPING', release_name)
class TestServerCommand(cmd.Command):
def get_parser(self, prog_name):
parser = super(TestServerCommand, self).get_parser(prog_name)
parser.add_argument(
'--release', action='store', help='testing Helm in Release')
parser.add_argument(
'-f',
'--file',
type=str,
metavar='FILE',
help='testing Helm releases in Manifest')
parser.add_argument(
'--tiller-host',
action='store',
type=str,
default=None,
help='Specify the tiller host')
parser.add_argument(
'--tiller-port',
action='store',
type=int,
default=44134,
help='Specify the tiller port')
return parser
def take_action(self, parsed_args):
testService(parsed_args)

4
armada/cli/tiller.py
View File

@ -28,7 +28,9 @@ def tillerServer(args):
tiller = Tiller()
if args.status:
LOG.info('Tiller is Active: %s', tiller.tiller_status())
resp = tiller.tiller_version()
LOG.info('Tiller Service: %s', tiller.tiller_status())
LOG.info('Tiller Version: %s', getattr(resp.Version, 'sem_ver', False))
if args.releases:
for release in tiller.list_releases():

96
armada/exceptions/tiller_exceptions.py
View File

@ -12,9 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import base_exception
from base_exception import ArmadaBaseException as ex
class TillerException(base_exception.ArmadaBaseException):
class TillerException(ex):
'''Base class for Tiller exceptions and error handling.'''
message = 'An unknown Tiller error occured.'
@ -27,10 +27,10 @@ class TillerServicesUnavailableException(TillerException):
class ChartCleanupException(TillerException):
'''Exception that occures during chart cleanup.'''
def __init__(self, chart_name, source_type):
super(ChartCleanupException, self).__init__('An error occred during \
cleanup while removing \
the chart ' + chart_name)
def __init__(self, chart_name):
message = 'An error occred during cleanup while removing {}'.format(
chart_name)
super(ChartCleanupException, self).__init__(message)
class ListChartsException(TillerException):
'''Exception that occurs when listing charts'''
@ -41,78 +41,76 @@ class PostUpdateJobDeleteException(TillerException):
'''Exception that occurs when a job deletion'''
def __init__(self, name, namespace):
self._name = name
self._namespace = namespace
self._message = 'Failed to delete k8s job ' + self._name + ' in ' + \
self._namespace + ' namspace.'
message = 'Failed to delete k8s job {} in {}'.format(
name, namespace)
super(PostUpdateJobDeleteException, self).__init__(self._message)
super(PostUpdateJobDeleteException, self).__init__(message)
class PostUpdateJobCreateException(TillerException):
'''Exception that occurs when a job creation fails.'''
def __init__(self, name, namespace):
self._name = name
self._namespace = namespace
self._message = 'Failed to create k8s job ' + self._name + ' in ' + \
self._namespace + ' namespace.'
message = 'Failed to create k8s job {} in {}'.format(
name, namespace)
super(PostUpdateJobCreateException, self).__init__(self._message)
super(PostUpdateJobCreateException, self).__init__(message)
class PreUpdateJobDeleteException(TillerException):
'''Exception that occurs when a job deletion'''
def __init__(self, name, namespace):
self._name = name
self._namespace = namespace
self._message = 'Failed to delete k8s job ' + self._name + ' in ' + \
self._namespace + ' namspace.'
message = 'Failed to delete k8s job {} in {}'.format(
name, namespace)
super(PreUpdateJobDeleteException, self).__init__(self._message)
super(PreUpdateJobDeleteException, self).__init__(message)
class PreUpdateJobCreateException(TillerException):
'''Exception that occurs when a job creation fails.'''
def __init__(self, name, namespace):
self._name = name
self._namespace = namespace
self._message = 'Failed to create k8s job ' + self._name + ' in ' + \
self._namespace + ' namespace.'
message = 'Failed to create k8s job {} in {}'.format(
name, namespace)
super(PreUpdateJobCreateException, self).__init__(self._message)
super(PreUpdateJobCreateException, self).__init__(message)
class ReleaseUninstallException(TillerException):
'''Exception that occurs when a release fails to uninstall.'''
def __init__(self, name, namespace):
self._name = name
self._message = 'Failed to uninstall release' + self._name + '.'
super(ReleaseUninstallException, self).__init__(self._message)
class ReleaseInstallException(TillerException):
class ReleaseException(TillerException):
'''Exception that occurs when a release fails to install.'''
def __init__(self, name, namespace):
self._name = name
self._message = 'Failed to install release' + self._name + '.'
def __init__(self, name, status, action):
til_msg = getattr(status.info, 'Description').encode()
message = 'Failed to {} release: {} - Tiller Message: {}'.format(
action, name, til_msg)
super(ReleaseInstallException, self).__init__(self._message)
class ReleaseUpdateException(TillerException):
'''Exception that occurs when a release fails to update.'''
def __init__(self, name, namespace):
self._name = name
self._message = 'Failed to update release' + self._name + '.'
super(ReleaseUpdateException, self).__init__(self._message)
super(ReleaseException, self).__init__(message)
class ChannelException(TillerException):
'''Exception that occurs during a failed GRPC channel creation'''
message = 'Failed to create GRPC channel.'
class GetReleaseStatusException(TillerException):
'''Exception that occurs during a failed Release Testing'''
def __init__(self, release, version):
message = 'Failed to get {} status {} version'.format(
release, version)
super(GetReleaseStatusException, self).__init__(message)
class GetReleaseContentException(TillerException):
'''Exception that occurs during a failed Release Testing'''
def __init__(self, release, version):
message = 'Failed to get {} content {} version {}'.format(
release, version)
super(GetReleaseContentException, self).__init__(message)
class TillerVersionException(TillerException):
'''Exception that occurs during a failed Release Testing'''
message = 'Failed to get Tiller Version'

20
armada/handlers/armada.py
View File

@ -204,8 +204,9 @@ class Armada(object):
desc = entry.get('description', 'A Chart Group')
chart_group = entry.get(KEYWORD_CHARTS, [])
test_charts = entry.get('test_charts', False)
if entry.get('sequenced', False):
if entry.get('sequenced', False) or test_charts:
chart_wait = True
LOG.info('Deploying: %s', desc)
@ -213,13 +214,18 @@ class Armada(object):
for gchart in chart_group:
chart = dotify(gchart['chart'])
values = gchart.get('chart').get('values', {})
test_chart = gchart.get('chart').get('test', False)
pre_actions = {}
post_actions = {}
LOG.info('%s', chart.release)
if chart.release is None:
continue
if test_chart:
chart_wait = True
# retrieve appropriate timeout value if 'wait' is specified
chart_timeout = self.timeout
if chart_wait:
@ -271,6 +277,7 @@ class Armada(object):
continue
# do actual update
LOG.info('wait: %s', chart_wait)
self.tiller.update_release(protoc_chart,
prefix_chart,
chart.namespace,
@ -297,6 +304,17 @@ class Armada(object):
LOG.debug("Cleaning up chart source in %s",
chartbuilder.source_directory)
if test_charts or test_chart:
LOG.info('Testing: %s', prefix_chart)
resp = self.tiller.testing_release(prefix_chart)
test_status = getattr(resp.info.status,
'last_test_suite_run', 'FAILED')
LOG.info("Test INFO: %s", test_status)
if resp:
LOG.info("PASSED: %s", prefix_chart)
else:
LOG.info("FAILED: %s", prefix_chart)
LOG.info("Performing Post-Flight Operations")
self.post_flight_ops()

17
armada/handlers/k8s.py
View File

@ -173,3 +173,20 @@ class K8s(object):
LOG.info('New pod %s deployed', new_pod_name)
w.stop()
def wait_get_completed_podphase(self, release, timeout=300):
'''
:param release - part of namespace
:param timeout - time before disconnecting stream
'''
w = watch.Watch()
for event in w.stream(self.client.list_pod_for_all_namespaces,
timeout_seconds=timeout):
pod_name = event['object'].metadata.name
if release in pod_name:
pod_state = event['object'].status.phase
if pod_state == 'Succeeded':
w.stop()
break

136
armada/handlers/tiller.py
View File

@ -12,26 +12,33 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import grpc
import yaml
import grpc
from hapi.services.tiller_pb2 import ReleaseServiceStub, ListReleasesRequest, \
InstallReleaseRequest, UpdateReleaseRequest, UninstallReleaseRequest
from hapi.chart.config_pb2 import Config
from k8s import K8s
from ..const import STATUS_DEPLOYED, STATUS_FAILED
from ..exceptions import tiller_exceptions
from ..utils.release import release_prefix
from hapi.services.tiller_pb2 import GetReleaseContentRequest
from hapi.services.tiller_pb2 import GetReleaseStatusRequest
from hapi.services.tiller_pb2 import GetVersionRequest
from hapi.services.tiller_pb2 import InstallReleaseRequest
from hapi.services.tiller_pb2 import ListReleasesRequest
from hapi.services.tiller_pb2 import ReleaseServiceStub
from hapi.services.tiller_pb2 import TestReleaseRequest
from hapi.services.tiller_pb2 import UninstallReleaseRequest
from hapi.services.tiller_pb2 import UpdateReleaseRequest
from oslo_config import cfg
from oslo_log import log as logging
from ..const import STATUS_DEPLOYED, STATUS_FAILED
from ..exceptions import tiller_exceptions as ex
from ..utils.release import release_prefix
from k8s import K8s
TILLER_PORT = 44134
TILLER_VERSION = b'2.5.0'
TILLER_TIMEOUT = 300
RELEASE_LIMIT = 64
RUNTEST_SUCCESS = 9
# the standard gRPC max message size is 4MB
# this expansion comes at a performance penalty
@ -88,7 +95,7 @@ class Tiller(object):
]
)
except Exception:
raise tiller_exceptions.ChannelException()
raise ex.ChannelException()
def _get_tiller_pod(self):
'''
@ -200,8 +207,8 @@ class Tiller(object):
self.delete_resources(
release_name, name, 'pod', labels, namespace)
except Exception:
raise tiller_exceptions.PreUpdateJobDeleteException(name,
namespace)
raise ex.PreUpdateJobDeleteException(name, namespace)
LOG.debug("PRE: Could not delete anything, please check yaml")
try:
@ -213,8 +220,8 @@ class Tiller(object):
self.k8s.create_job_action(name, action_type)
continue
except Exception:
raise tiller_exceptions.PreUpdateJobCreateException(name,
namespace)
raise ex.PreUpdateJobCreateException(name, namespace)
LOG.debug("PRE: Could not create anything, please check yaml")
def delete_resource(self, release_name, resource_name, resource_type,
@ -260,7 +267,7 @@ class Tiller(object):
self.k8s.create_job_action(name, action_type)
continue
except Exception:
raise tiller_exceptions.PreUpdateJobCreateException()
raise ex.PreUpdateJobCreateException()
LOG.debug("POST: Could not create anything, please check yaml")
def list_charts(self):
@ -319,7 +326,8 @@ class Tiller(object):
stub.UpdateRelease(
release_request, self.timeout, metadata=self.metadata)
except Exception:
raise tiller_exceptions.ReleaseInstallException(release, namespace)
status = self.get_release_status(release)
raise ex.ReleaseException(release, status, 'Upgrade')
self._post_update_actions(post_actions, namespace)
@ -331,6 +339,7 @@ class Tiller(object):
'''
Create a Helm Release
'''
LOG.debug("wait: %s", wait)
LOG.debug("timeout: %s", timeout)
@ -355,7 +364,95 @@ class Tiller(object):
release_request, self.timeout, metadata=self.metadata)
except Exception:
raise tiller_exceptions.ReleaseInstallException(release, namespace)
status = self.get_release_status(release)
raise ex.ReleaseException(release, status, 'Install')
def testing_release(self, release, timeout=300, cleanup=True):
'''
:param release - name of release to test
:param timeout - runtime before exiting
:param cleanup - removes testing pod created
:returns - results of test pod
'''
try:
stub = ReleaseServiceStub(self.channel)
release_request = TestReleaseRequest(name=release, timeout=timeout,
cleanup=cleanup)
content = self.get_release_content(release)
if not len(content.release.hooks):
LOG.info('No test found')
return False
if content.release.hooks[0].events[0] == RUNTEST_SUCCESS:
test = stub.RunReleaseTest(
release_request, self.timeout, metadata=self.metadata)
if test.running():
self.k8s.wait_get_completed_podphase(release)
test.cancel()
return self.get_release_status(release)
except Exception:
status = self.get_release_status(release)
raise ex.ReleaseException(release, status, 'Test')
def get_release_status(self, release, version=0):
'''
:param release - name of release to test
:param version - version of release status
'''
try:
stub = ReleaseServiceStub(self.channel)
status_request = GetReleaseStatusRequest(
name=release, version=version)
return stub.GetReleaseStatus(
status_request, self.timeout, metadata=self.metadata)
except Exception:
raise ex.GetReleaseStatusException(release, version)
def get_release_content(self, release, version=0):
'''
:param release - name of release to test
:param version - version of release status
'''
try:
stub = ReleaseServiceStub(self.channel)
status_request = GetReleaseContentRequest(
name=release, version=version)
return stub.GetReleaseContent(
status_request, self.timeout, metadata=self.metadata)
except Exception:
raise ex.GetReleaseContentException(release, version)
def tiller_version(self):
'''
:returns - tiller version
'''
try:
stub = ReleaseServiceStub(self.channel)
release_request = GetVersionRequest()
return stub.GetVersion(
release_request, self.timeout, metadata=self.metadata)
except Exception:
raise ex.TillerVersionException()
def uninstall_release(self, release, disable_hooks=False, purge=True):
'''
@ -375,7 +472,8 @@ class Tiller(object):
release_request, self.timeout, metadata=self.metadata)
except Exception:
raise tiller_exceptions.ReleaseUninstallException(release)
status = self.get_release_status(release)
raise ex.ReleaseException(release, status, 'Delete')
def chart_cleanup(self, prefix, charts):
'''

3
docs/source/commands/index.rst
View File

@ -11,5 +11,6 @@ Commands Guide
:caption: Contents:
apply.rst
validate.rst
test.rst
tiller.rst
validate.rst

21
docs/source/commands/test.rst Normal file
View File

@ -0,0 +1,21 @@
Armada - Test
=============
Commands
--------
.. code:: bash
Usage: armada test
Options:
[-h] [--release RELEASE] [--file FILE]
Synopsis
--------
The test command will perform helm test defined on the release. Test command can
test a single release or a manifest.

123
docs/source/operations/guide-build-armada-yaml.rst
View File

@ -7,16 +7,16 @@ armada/Manifest/v1
+---------------------+--------+----------------------+
| keyword | type | action |
+=====================+========+======================+
| ``release_prefix`` | string | tag appended to the |
| ``release_prefix`` | string | appends to the |
| | | front of all |
| | | charts |
| | | released |
| | | by the |
| | | yaml in |
| | | manifest in |
| | | order to |
| | | manage releses |
| | | manage releases |
| | | throughout their |
| | | lifecycles |
| | | lifecycle |
+---------------------+--------+----------------------+
| ``chart_groups`` | array | references |
| | | ChartGroup document |
@ -52,6 +52,8 @@ armada/ChartGroup/v1
+-----------------+----------+------------------------------------------------------------------------+
| sequenced | bool | enables sequenced chart deployment in a group |
+-----------------+----------+------------------------------------------------------------------------+
| test_charts | bool | run pre-defined helm tests helm in a ChartGroup |
+-----------------+----------+------------------------------------------------------------------------+
Example
~~~~~~~~
@ -87,6 +89,8 @@ Chart
+-----------------+----------+---------------------------------------------------------------------------+
| timeout | int | time (in seconds) allotted for chart to deploy when 'wait' flag is set |
+-----------------+----------+---------------------------------------------------------------------------+
| test | bool | run pre-defined helm tests helm in a chart |
+-----------------+----------+---------------------------------------------------------------------------+
| install | object | install the chart into your Kubernetes cluster |
+-----------------+----------+---------------------------------------------------------------------------+
| update | object | update the chart managed by the armada yaml |
@ -135,7 +139,7 @@ Update - Actions - Update/Delete
+=============+==========+===============================================================+
| name | string | name of action |
+-------------+----------+---------------------------------------------------------------+
| type | string | type of K8s kind to execute |
| type | string | type of kubernetes workload to execute |
+-------------+----------+---------------------------------------------------------------+
| labels | object | array of labels to query against kinds. (key: value) |
+-------------+----------+---------------------------------------------------------------+
@ -144,6 +148,84 @@ Update - Actions - Update/Delete
Update Actions only support type: 'daemonset'
.. note::
Delete Actions only support type: 'job'
Example
~~~~~~~
::
# type git
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
chart_name: blog-1
release_name: blog-1
namespace: default
timeout: 100
install:
no_hook: false
upgrade:
no_hook: false
values: {}
source:
type: git
location: https://github.com/namespace/repo
subpath: .
reference: master
dependencies: []
# type local
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
chart_name: blog-1
release_name: blog-1
namespace: default
timeout: 100
install:
no_hook: false
upgrade:
no_hook: false
values: {}
source:
type: local
location: /path/to/charts
subpath: chart
reference: master
dependencies: []
# type tar
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: blog-1
data:
chart_name: blog-1
release_name: blog-1
namespace: default
timeout: 100
install:
no_hook: false
upgrade:
no_hook: false
values: {}
source:
type: tar
location: https://localhost:8879/charts/chart-0.1.0.tgz
subpath: mariadb
reference: null
dependencies: []
Source
^^^^^^
@ -160,6 +242,7 @@ Source
| reference | string | (optional) branch of the repo (``master`` if not specified) |
+-------------+----------+-------------------------------------------------------------------------------+
Example
~~~~~~~
@ -205,31 +288,39 @@ Example
Defining a Manifest
~~~~~~~~~~~~~~~~~~~
Defining a Chart
~~~~~~~~~~~~~~~~
To define your Manifest you need to define a ``armada/Manifest/v1`` document,
``armada/ChartGroup/v1`` document, ``armada/Chart/v1``.
Following the definitions above for each document you will be able to construct
an armada manifest.
To define your charts is not any different than helm. we do provide some
post/pre actions that will help us manage our charts better.
Armada - Deploy Behavior
^^^^^^^^^^^^^^^^^^^^^^^^
Behavior
^^^^^^^^
1. Armada will perform set of pre-flight checks to before applying the manifest
- validate input manifest
- check tiller service is Running
- check chart source locations are valid
1. will check if chart exists
2. Deploying Armada Manifest
1. if it does not exist
1. If the chart is not found
- we will install the chart
2. if exist then
- armada will check if there are any differences in the charts
3. If exist then
- Armada will check if there are any differences in the chart
- if the charts are different then it will execute an upgrade
- else it will not perform any actions
.. note::
You can use references in order to build your charts, this will reduce the size of the chart definition will show example in multichart below
You can use references in order to build your charts, this will reduce
the size of the chart definition will show example in multichart below
Simple Example
~~~~~~~~~~~~~~

13
examples/armada-manifest-v1.yaml
View File

@ -109,19 +109,14 @@ metadata:
name: keystone
data:
chart_name: keystone
test: true
release: keystone
namespace: undercloud
timeout: 100
timeout: 3600
install:
no_hooks: false
upgrade:
no_hooks: false
pre:
delete:
- name: keystone-db-sync
type: job
- name: keystone-db-init
type: job
values:
replicas: 2
source:
@ -142,7 +137,6 @@ data:
chart_group:
- mariadb
- etcd
- rabbitmq
- memcached
---
schema: armada/ChartGroup/v1
@ -151,7 +145,8 @@ metadata:
name: openstack-keystone
data:
description: "Deploying OpenStack Keystone"
sequenced: False
sequenced: True
test_charts: False
chart_group:
- keystone
---

3
setup.cfg
View File

@ -40,6 +40,9 @@ armada =
apply = armada.cli.apply:ApplyChartsCommand
tiller = armada.cli.tiller:TillerServerCommand
validate = armada.cli.validate:ValidateYamlCommand
test = armada.cli.test:TestServerCommand
oslo.config.opts =
armada.conf = armada.conf.opts:list_opts
[pbr]
warnerrors = True