airship/armada
Code Issues Proposed changes

Extract pre-update actions out of tiller handler

Browse Source 
This is a pre-requisite for Helm 3 integration, so that these
actions run regardless of whether we are going through the
tiller handler.

Change-Id: I97d7bcc823d11b527fcdaa7967fcab62af1c8161
This commit is contained in:
Sean Eagan 2019-04-22 16:28:27 -05:00 committed by Sean Eagan
parent 5f1ffbbbbe
commit 58c0df5201
9 changed files with 245 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
armada/exceptions/armada_exceptions.py
View File

@ -102,3 +102,18 @@ class DeploymentLikelyPendingException(ArmadaException):
'(last deployment age={}s) < (chart wait timeout={}s)'.format(
release, status, last_deployment_age, timeout))
super(DeploymentLikelyPendingException, self).__init__(self._message)
class PreUpdateJobDeleteException(ArmadaException):
'''
Exception that occurs when a job deletion.
**Troubleshoot:**
*Coming Soon*
'''
def __init__(self, name, namespace):
message = 'Failed to delete k8s job {} in {}'.format(name, namespace)
super(PreUpdateJobDeleteException, self).__init__(message)

40
armada/exceptions/tiller_exceptions.py
View File

@ -47,46 +47,6 @@ class ListChartsException(TillerException):
message = 'There was an error listing the Helm chart releases.'
class PostUpdateJobDeleteException(TillerException):
'''Exception that occurs when a job deletion'''
def __init__(self, name, namespace):
message = 'Failed to delete k8s job {} in {}'.format(name, namespace)
super(PostUpdateJobDeleteException, self).__init__(message)
class PostUpdateJobCreateException(TillerException):
'''
Exception that occurs when a job creation fails.
**Troubleshoot:**
*Coming Soon*
'''
def __init__(self, name, namespace):
message = 'Failed to create k8s job {} in {}'.format(name, namespace)
super(PostUpdateJobCreateException, self).__init__(message)
class PreUpdateJobDeleteException(TillerException):
'''
Exception that occurs when a job deletion.
**Troubleshoot:**
*Coming Soon*
'''
def __init__(self, name, namespace):
message = 'Failed to delete k8s job {} in {}'.format(name, namespace)
super(PreUpdateJobDeleteException, self).__init__(message)
class ReleaseException(TillerException):
'''
Exception that occurs when a release fails to install, upgrade, delete,

14
armada/handlers/chart_deploy.py
View File

@ -23,10 +23,11 @@ from armada.handlers import metrics
from armada.handlers.chartbuilder import ChartBuilder
from armada.handlers.release_diff import ReleaseDiff
from armada.handlers.chart_delete import ChartDelete
from armada.handlers.pre_update_actions import PreUpdateActions
from armada.handlers.schema import get_schema_info
from armada.handlers.test import Test
from armada.handlers.wait import ChartWait
from armada.exceptions import tiller_exceptions
from armada.exceptions import tiller_exceptions as tiller_ex
import armada.utils.release as r
LOG = logging.getLogger(__name__)
@ -87,7 +88,6 @@ class ChartDeploy(object):
# Resolve action
values = chart.get('values', {})
pre_actions = {}
post_actions = {}
status = None
if old_release:
@ -133,9 +133,8 @@ class ChartDeploy(object):
if not self.disable_update_post and upgrade_post:
LOG.warning(
'Post upgrade actions are ignored by Armada '
'Post upgrade actions are ignored by Armada'
'and will not affect deployment.')
post_actions = upgrade_post
try:
old_values = yaml.safe_load(old_values_string)
@ -159,6 +158,9 @@ class ChartDeploy(object):
def upgrade():
# do actual update
timer = int(round(deadline - time.time()))
PreUpdateActions(self.tiller.k8s).execute(
pre_actions, release, namespace, chart, disable_hooks,
values, timer)
LOG.info(
"Upgrading release %s in namespace %s, wait=%s, "
"timeout=%ss", release_name, namespace,
@ -167,8 +169,6 @@ class ChartDeploy(object):
new_chart,
release_name,
namespace,
pre_actions=pre_actions,
post_actions=post_actions,
disable_hooks=disable_hooks,
values=yaml.safe_dump(values),
wait=native_wait_enabled,
@ -315,7 +315,7 @@ class ChartDeploy(object):
def _test_chart(self, release_name, test_handler):
success = test_handler.test_release_for_success()
if not success:
raise tiller_exceptions.TestFailedException(release_name)
raise tiller_ex.TestFailedException(release_name)
def get_diff(self, old_chart, old_values, new_chart, values):
return ReleaseDiff(old_chart, old_values, new_chart, values).get_diff()

217
armada/handlers/pre_update_actions.py Normal file
View File

@ -0,0 +1,217 @@
# Copyright 2019 The Armada Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log as logging
from armada import const
from armada.conf import get_current_chart
from armada.exceptions import armada_exceptions as ex
from armada.handlers import schema
from armada.utils.release import label_selectors
LOG = logging.getLogger(__name__)
class PreUpdateActions():
def __init__(self, k8s):
self.k8s = k8s
def execute(
self, actions, release_name, namespace, chart, disable_hooks,
values, timeout):
'''
:param actions: array of items actions
:param namespace: name of pod for actions
'''
# TODO: Remove when v1 doc support is removed.
try:
for action in actions.get('update', []):
name = action.get('name')
LOG.info('Updating %s ', name)
action_type = action.get('type')
labels = action.get('labels')
self.rolling_upgrade_pod_deployment(
name, release_name, namespace, labels, action_type, chart,
disable_hooks, values, timeout)
except Exception:
LOG.exception(
"Pre-action failure: could not perform rolling upgrade for "
"%(res_type)s %(res_name)s.", {
'res_type': action_type,
'res_name': name
})
raise ex.PreUpdateJobDeleteException(name, namespace)
try:
for action in actions.get('delete', []):
name = action.get('name')
action_type = action.get('type')
labels = action.get('labels', None)
self.delete_resources(
action_type, labels, namespace, timeout=timeout)
except Exception:
LOG.exception(
"Pre-action failure: could not delete %(res_type)s "
"%(res_name)s.", {
'res_type': action_type,
'res_name': name
})
raise ex.PreUpdateJobDeleteException(name, namespace)
def delete_resources(
self,
resource_type,
resource_labels,
namespace,
wait=False,
timeout=const.DEFAULT_TILLER_TIMEOUT):
'''
Delete resources matching provided resource type, labels, and
namespace.
:param resource_type: type of resource e.g. job, pod, etc.
:param resource_labels: labels for selecting the resources
:param namespace: namespace of resources
'''
timeout = self._check_timeout(wait, timeout)
label_selector = ''
if resource_labels is not None:
label_selector = label_selectors(resource_labels)
LOG.debug(
"Deleting resources in namespace: %s, matching "
"selectors: %s (timeout=%s).", namespace, label_selector, timeout)
handled = False
if resource_type == 'job':
get_jobs = self.k8s.get_namespace_job(
namespace, label_selector=label_selector)
for jb in get_jobs.items:
jb_name = jb.metadata.name
LOG.info(
"Deleting job: %s in namespace: %s", jb_name, namespace)
self.k8s.delete_job_action(jb_name, namespace, timeout=timeout)
handled = True
# TODO: Remove when v1 doc support is removed.
chart = get_current_chart()
schema_info = schema.get_schema_info(chart['schema'])
job_implies_cronjob = schema_info.version < 2
implied_cronjob = resource_type == 'job' and job_implies_cronjob
if resource_type == 'cronjob' or implied_cronjob:
get_jobs = self.k8s.get_namespace_cron_job(
namespace, label_selector=label_selector)
for jb in get_jobs.items:
jb_name = jb.metadata.name
# TODO: Remove when v1 doc support is removed.
if implied_cronjob:
LOG.warn(
"Deleting cronjobs via `type: job` is "
"deprecated, use `type: cronjob` instead")
LOG.info(
"Deleting cronjob %s in namespace: %s", jb_name, namespace)
self.k8s.delete_cron_job_action(jb_name, namespace)
handled = True
if resource_type == 'pod':
release_pods = self.k8s.get_namespace_pod(
namespace, label_selector=label_selector)
for pod in release_pods.items:
pod_name = pod.metadata.name
LOG.info(
"Deleting pod %s in namespace: %s", pod_name, namespace)
self.k8s.delete_pod_action(pod_name, namespace)
if wait:
self.k8s.wait_for_pod_redeployment(pod_name, namespace)
handled = True
if not handled:
LOG.error(
'No resources found with labels=%s type=%s namespace=%s',
resource_labels, resource_type, namespace)
def rolling_upgrade_pod_deployment(
self,
name,
release_name,
namespace,
resource_labels,
action_type,
chart,
disable_hooks,
values,
timeout=const.DEFAULT_TILLER_TIMEOUT):
'''
update statefulsets (daemon, stateful)
'''
if action_type == 'daemonset':
LOG.info('Updating: %s', action_type)
label_selector = ''
if resource_labels is not None:
label_selector = label_selectors(resource_labels)
get_daemonset = self.k8s.get_namespace_daemon_set(
namespace, label_selector=label_selector)
for ds in get_daemonset.items:
ds_name = ds.metadata.name
ds_labels = ds.metadata.labels
if ds_name == name:
LOG.info(
"Deleting %s : %s in %s", action_type, ds_name,
namespace)
self.k8s.delete_daemon_action(ds_name, namespace)
# update the daemonset yaml
template = self.get_chart_templates(
ds_name, name, release_name, namespace, chart,
disable_hooks, values)
template['metadata']['labels'] = ds_labels
template['spec']['template']['metadata'][
'labels'] = ds_labels
self.k8s.create_daemon_action(
namespace=namespace, template=template)
# delete pods
self.delete_resources(
'pod',
resource_labels,
namespace,
wait=True,
timeout=timeout)
else:
LOG.error("Unable to exectue name: % type: %s", name, action_type)
def _check_timeout(self, wait, timeout):
if timeout is None or timeout <= 0:
if wait:
LOG.warn(
'Pre-update actions timeout is invalid or unspecified, '
'using default %ss.', self.timeout)
timeout = self.timeout
return timeout

190
armada/handlers/tiller.py
View File

@ -29,12 +29,10 @@ from oslo_log import log as logging
import yaml
from armada import const
from armada.conf import get_current_chart
from armada.exceptions import tiller_exceptions as ex
from armada.handlers.k8s import K8s
from armada.handlers import schema
from armada.utils import helm
from armada.utils.release import label_selectors, get_release_status
from armada.utils.release import get_release_status
TILLER_VERSION = b'2.16.9'
GRPC_EPSILON = 60
@ -301,51 +299,6 @@ class Tiller(object):
LOG.info(template_name)
return template
def _pre_update_actions(
self, actions, release_name, namespace, chart, disable_hooks,
values, timeout):
'''
:param actions: array of items actions
:param namespace: name of pod for actions
'''
# TODO: Remove when v1 doc support is removed.
try:
for action in actions.get('update', []):
name = action.get('name')
LOG.info('Updating %s ', name)
action_type = action.get('type')
labels = action.get('labels')
self.rolling_upgrade_pod_deployment(
name, release_name, namespace, labels, action_type, chart,
disable_hooks, values, timeout)
except Exception:
LOG.exception(
"Pre-action failure: could not perform rolling upgrade for "
"%(res_type)s %(res_name)s.", {
'res_type': action_type,
'res_name': name
})
raise ex.PreUpdateJobDeleteException(name, namespace)
try:
for action in actions.get('delete', []):
name = action.get('name')
action_type = action.get('type')
labels = action.get('labels', None)
self.delete_resources(
action_type, labels, namespace, timeout=timeout)
except Exception:
LOG.exception(
"Pre-action failure: could not delete %(res_type)s "
"%(res_name)s.", {
'res_type': action_type,
'res_name': name
})
raise ex.PreUpdateJobDeleteException(name, namespace)
def list_charts(self):
'''
List Helm Charts from Latest Releases
@ -375,8 +328,6 @@ class Tiller(object):
chart,
release,
namespace,
pre_actions=None,
post_actions=None,
disable_hooks=False,
values=None,
wait=False,
@ -397,10 +348,6 @@ class Tiller(object):
else:
values = Config(raw=values)
self._pre_update_actions(
pre_actions, release, namespace, chart, disable_hooks, values,
timeout)
update_msg = None
# build release install request
try:
@ -617,141 +564,6 @@ class Tiller(object):
status = self.get_release_status(release)
raise ex.ReleaseException(release, status, 'Delete')
def delete_resources(
self,
resource_type,
resource_labels,
namespace,
wait=False,
timeout=const.DEFAULT_TILLER_TIMEOUT):
'''
Delete resources matching provided resource type, labels, and
namespace.
:param resource_type: type of resource e.g. job, pod, etc.
:param resource_labels: labels for selecting the resources
:param namespace: namespace of resources
'''
timeout = self._check_timeout(wait, timeout)
label_selector = ''
if resource_labels is not None:
label_selector = label_selectors(resource_labels)
LOG.debug(
"Deleting resources in namespace: %s, matching "
"selectors: %s (timeout=%s).", namespace, label_selector, timeout)
handled = False
if resource_type == 'job':
get_jobs = self.k8s.get_namespace_job(
namespace, label_selector=label_selector)
for jb in get_jobs.items:
jb_name = jb.metadata.name
LOG.info(
"Deleting job: %s in namespace: %s", jb_name, namespace)
self.k8s.delete_job_action(jb_name, namespace, timeout=timeout)
handled = True
# TODO: Remove when v1 doc support is removed.
chart = get_current_chart()
schema_info = schema.get_schema_info(chart['schema'])
job_implies_cronjob = schema_info.version < 2
implied_cronjob = resource_type == 'job' and job_implies_cronjob
if resource_type == 'cronjob' or implied_cronjob:
get_jobs = self.k8s.get_namespace_cron_job(
namespace, label_selector=label_selector)
for jb in get_jobs.items:
jb_name = jb.metadata.name
# TODO: Remove when v1 doc support is removed.
if implied_cronjob:
LOG.warn(
"Deleting cronjobs via `type: job` is "
"deprecated, use `type: cronjob` instead")
LOG.info(
"Deleting cronjob %s in namespace: %s", jb_name, namespace)
self.k8s.delete_cron_job_action(jb_name, namespace)
handled = True
if resource_type == 'pod':
release_pods = self.k8s.get_namespace_pod(
namespace, label_selector=label_selector)
for pod in release_pods.items:
pod_name = pod.metadata.name
LOG.info(
"Deleting pod %s in namespace: %s", pod_name, namespace)
self.k8s.delete_pod_action(pod_name, namespace)
if wait:
self.k8s.wait_for_pod_redeployment(pod_name, namespace)
handled = True
if not handled:
LOG.error(
'No resources found with labels=%s type=%s namespace=%s',
resource_labels, resource_type, namespace)
def rolling_upgrade_pod_deployment(
self,
name,
release_name,
namespace,
resource_labels,
action_type,
chart,
disable_hooks,
values,
timeout=const.DEFAULT_TILLER_TIMEOUT):
'''
update statefulsets (daemon, stateful)
'''
if action_type == 'daemonset':
LOG.info('Updating: %s', action_type)
label_selector = ''
if resource_labels is not None:
label_selector = label_selectors(resource_labels)
get_daemonset = self.k8s.get_namespace_daemon_set(
namespace, label_selector=label_selector)
for ds in get_daemonset.items:
ds_name = ds.metadata.name
ds_labels = ds.metadata.labels
if ds_name == name:
LOG.info(
"Deleting %s : %s in %s", action_type, ds_name,
namespace)
self.k8s.delete_daemon_action(ds_name, namespace)
# update the daemonset yaml
template = self.get_chart_templates(
ds_name, name, release_name, namespace, chart,
disable_hooks, values)
template['metadata']['labels'] = ds_labels
template['spec']['template']['metadata'][
'labels'] = ds_labels
self.k8s.create_daemon_action(
namespace=namespace, template=template)
# delete pods
self.delete_resources(
'pod',
resource_labels,
namespace,
wait=True,
timeout=timeout)
else:
LOG.error("Unable to exectue name: % type: %s", name, action_type)
def rollback_release(
self,
release_name,

2
armada/tests/unit/handlers/test_armada.py
View File

@ -480,8 +480,6 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
['release_prefix'],
chart['release']),
chart['namespace'],
pre_actions={},
post_actions={},
disable_hooks=disable_hooks,
force=force,
recreate_pods=recreate_pods,

13
armada/tests/unit/handlers/test_tiller.py
View File

@ -13,7 +13,6 @@
# limitations under the License.
import mock
from mock import MagicMock
from armada.exceptions import tiller_exceptions as ex
from armada.handlers import tiller
@ -464,12 +463,6 @@ class TillerTestCase(base.ArmadaTestCase):
tiller_obj = tiller.Tiller('host', '8080', None)
# TODO: Test these methods as well, either by unmocking, or adding
# separate tests for them.
tiller_obj._pre_update_actions = MagicMock()
pre_actions = {}
post_actions = {}
disable_hooks = False
wait = True
timeout = 123
@ -480,8 +473,6 @@ class TillerTestCase(base.ArmadaTestCase):
chart,
release,
namespace,
pre_actions=pre_actions,
post_actions=post_actions,
disable_hooks=disable_hooks,
values=values,
wait=wait,
@ -489,10 +480,6 @@ class TillerTestCase(base.ArmadaTestCase):
force=force,
recreate_pods=recreate_pods)
tiller_obj._pre_update_actions.assert_called_once_with(
pre_actions, release, namespace, chart, disable_hooks, values,
timeout)
mock_update_release_request.assert_called_once_with(
chart=chart,
name=release,

5
doc/source/operations/exceptions/armada-exceptions.inc
View File

@ -53,3 +53,8 @@ Armada Exceptions
:members:
:show-inheritance:
:undoc-members:
.. autoexception:: PreUpdateJobDeleteException
:members:
:show-inheritance:
:undoc-members:

10
doc/source/operations/exceptions/tiller-exceptions.inc
View File

@ -29,16 +29,6 @@ Tiller Exceptions
:show-inheritance:
:undoc-members:
.. autoexception:: PostUpdateJobCreateException
:members:
:show-inheritance:
:undoc-members:
.. autoexception:: PreUpdateJobDeleteException
:members:
:show-inheritance:
:undoc-members:
.. autoexception:: ReleaseException
:members:
:show-inheritance: