Create an initial Automation Validation Testing framework

- Setting up a runtime environment
- Add to Zuul 'check' and 'gate' pipeline
- Validate concepts:
  1) List a few unit tests successfully
  2) Execute a few simple tests successfully (pass or fail)
  3) Generate docsbuild successfully
- Each UC will have their own test suite and test cases
and will be handled seperately
- add compute-utility deployment into the gate
- add logging collector of primary
- add false / positive test cases
- add unit-test and feature-test gates

Change-Id: I55a0dcf440e9694b041d5fb8eb75bd6f4adb8913

Makefile

@@ -34,6 +34,7 @@ COMMIT            ?= $(shell git rev-parse HEAD)
 DISTRO_SUFFIX     ?= $(DISTRO)
 IMAGE              = $(DOCKER_REGISTRY)/$(IMAGE_PREFIX)/$(IMAGE_NAME):$(IMAGE_TAG)$(IMAGE_TAG_SUFFIX)
 BASE_IMAGE        ?=
+
 # TODO(roman_g): DISTRO_SUFFIX should be autogenerated
 # from Dockerfile extensions, see $(suffix ) Makefile function
 ifeq "$(DISTRO_SUFFIX)" ""
@@ -129,8 +130,14 @@ run_images:
 run_$(IMAGE_NAME):
 	@echo "Not implemented." >&2; exit 2
 
-tests:
+unit_tests:
-	@echo "Not implemented." >&2; exit 2
+	@echo "Run Unit Validation Testing"
+	./tools/run_avt.sh unit_tests
+
+feature_tests:
+	@echo "Run Feature Validation Testing"
+	./tools/run_avt.sh feature_tests
+	tox -e docs
 
 format:
 	@echo "Not implemented." >&2; exit 2
@@ -176,5 +183,4 @@ endif
 
 .PHONY: $(CHARTS) $(IMAGES) all build chartbanner charts check-docker clean \
         docs dry-run-% dry-run format helm-init-% helm-install helm-lint-% \
-        helm-lint helm-serve images info lint run_$(IMAGE_NAME) run_images \
+        helm-lint helm-serve images info lint run_$(IMAGE_NAME) run_images
-        tests

docs/requirements.txt

@@ -0,0 +1,4 @@
+sphinx>=1.6.2
+sphinx_rtd_theme==0.2.4
+falcon==1.2.0
+oslo.config==6.6.2

docs/source/conf.py

@@ -0,0 +1,137 @@
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.todo',
+    'sphinx.ext.viewcode'
+]
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.md'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Porthole'
+copyright = u'2020, Porthole Authors'
+author = u'Porthole Authors'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = u'0.1.0'
+# The full version, including alpha/beta/rc tags.
+release = u'0.1.0'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+import sphinx_rtd_theme
+html_theme = "sphinx_rtd_theme"
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# NOTE(mark-burnett): Currently, we don't have any static files and the
+# non-existence of this directory causes a sphinx exception.
+#html_static_path = ['_static']
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'portholedoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'porthole.tex', u'Porthole Documentation',
+     u'Porthole Authors', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'Porthole', u'Porthole Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'Porthole', u'Porthole Documentation',
+     author, 'Porthole',
+     'Tool for bootstrapping a resilient Kubernetes cluster and managing its life-cycle.',
+     'Miscellaneous'),
+]
+

docs/source/index.md

@@ -0,0 +1,7 @@
+# Utility Containers
+
+Utility containers give Operations staff an interface to an Airship
+environment that enables them to perform routine operations and
+troubleshooting activities. Utility containers support Airship
+environments without exposing secrets and credentials while at
+the same time restricting access to the actual containers.

kube_utility_container/kubecfg/kube_cfg.py

@@ -0,0 +1,97 @@
+# Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from kubeconfig import KubeConfig
+
+class KubeCfg(KubeConfig):
+    """This class inherits from the KubeConfig module. It overides the
+
+    set_credentials method to add the user exec parameters to the kube config
+    file that is generated.
+    """
+
+    def set_credentials(
+            self,
+            name,
+            auth_provider=None,
+            auth_provider_args=None,
+            client_certificate=None,
+            client_key=None,
+            embed_certs=None,
+            password=None,
+            token=None,
+            username=None,
+            exec_command=None,
+            exec_api_version=None,
+            exec_arg=None,
+            exec_env=None):
+        """Creates or updates a ``user`` entry under the ``users`` entry.
+
+        In the case where you are updating an existing user, only the optional
+        keyword args that you pass in will be updated on the entry.
+
+        :param str name: The name of the user to add or update.
+        :param str auth_provider: The auth provider name to use. For example,
+            ``oidc``, ``gcp``, etc.
+        :param dict auth_provider_args: Some providers support extra config
+            params, which can be passed in as a flat dict.
+        :param str client_certificate: Path to your X.509 client cert (if
+            using cert auth).
+        :param str client_key: Path to your cert's private key (if using
+            cert auth).
+        :param bool embed_certs: Combined with ``client_certificate``,
+
+            setting this to ``True`` will cause the cert to be embedded
+            directly in the written config. If ``False`` or unspecified,
+            the path to the cert will be used instead.
+        :param str username: Your username (if using basic auth).
+        :param str password: Your user's password (if using basic auth).
+        :param str token: Your private token (if using token auth).
+        :param str exec_command: The command executable name to use. For
+        example, ``client-keystone-auth``
+        :param str exec_api_version: The api version to use. For example,
+            ``client.authentication.k8s.io/v1beta1``
+        """
+        flags = []
+        if auth_provider is not None:
+            flags += ['--auth-provider=%s' % auth_provider]
+        if auth_provider_args is not None:
+            arg_pairs = [
+                "%s=%s" % (k, v) for k, v in auth_provider_args.items()
+            ]
+            for arg_pair in arg_pairs:
+                flags += ['--auth-provider-arg=%s' % arg_pair]
+        if client_certificate is not None:
+            flags += ['--client-certificate=%s' % client_certificate]
+        if client_key is not None:
+            flags += ['--client-key=%s' % client_key]
+        if embed_certs is not None:
+            flags += ['--embed-certs=%s' % self._bool_to_cli_str(embed_certs)]
+        if password is not None:
+            flags += ['--password=%s' % password]
+        if token is not None:
+            flags += ['--token=%s' % token]
+        if username is not None:
+            flags += ['--username=%s' % username]
+        if exec_command is not None:
+            flags += ['--exec-command=%s' % exec_command]
+        if exec_api_version is not None:
+            flags += ['--exec-api-version=%s' % exec_api_version]
+        if exec_arg is not None:
+            flags += ['--exec-arg=%s' % exec_arg]
+        if exec_env is not None:
+            arg_pairs = ["%s=%s" % (k, v) for k, v in exec_env.items()]
+            for arg_pair in arg_pairs:
+                flags += ['--exec-env=%s' % arg_pair]
+        self._run_kubectl_config('set-credentials', name, *flags)

kube_utility_container/services/exceptions.py

@@ -0,0 +1,63 @@
+# Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class KubeUtilityContainerException(Exception):
+    """Class for Kube Utility Container Plugin Exceptions"""
+
+    def __init__(self, error="", message=""):
+        self.error = error or self.__class__.error
+        self.message = message or self.__class__.message
+        super(KubeUtilityContainerException, self).__init__(
+            ''.join([self.error, '::', self.message]))
+
+
+class KubeConfigException(Exception):
+    """Exception class when Kubernetes config is not found"""
+
+    def __init__(self, message):
+        self.message = "Kubernetes config not found: {}".format(message)
+        super(KubeConfigException, self).__init__(self.message)
+
+
+class KubeApiException(Exception):
+    """Exception class for error in accessing Kubernetes APIs"""
+
+    def __init__(self, message):
+        self.message = "Exception occurred while accessing Kubernetes APIs: " \
+                       "{}".format(message)
+        super(KubeApiException, self).__init__(self.message)
+
+
+class KubeDeploymentNotFoundException(Exception):
+    """Exception class for Kube Deployment not found in a namespace"""
+
+    def __init__(self, message):
+        self.message = "Deployment not found Error: {}".format(message)
+        super(KubeDeploymentNotFoundException, self).__init__(self.message)
+
+
+class KubePodNotFoundException(Exception):
+    """Exception class for specific utility pod not found in running state"""
+
+    def __init__(self, message):
+        self.message = "Pod not found: {}".format(message)
+        super(KubePodNotFoundException, self).__init__(self.message)
+
+
+class KubeEnvVarException(Exception):
+    """Exception class for environment variable not set."""
+
+    def __init__(self, message):
+        self.message = "Environment Variable Not Found: {}".format(message)
+        super(KubeEnvVarException, self).__init__(self.message)

kube_utility_container/services/utility_container_client.py

@@ -0,0 +1,247 @@
+# Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from pathlib import Path
+
+from kube_utility_container.kubecfg.kube_cfg import KubeCfg
+
+from kube_utility_container.services.exceptions import \
+    KubeApiException
+from kube_utility_container.services.exceptions import \
+    KubeConfigException
+from kube_utility_container.services.exceptions import \
+    KubeDeploymentNotFoundException
+from kube_utility_container.services.exceptions import \
+    KubeEnvVarException
+from kube_utility_container.services.exceptions import \
+    KubePodNotFoundException
+
+from kubernetes import client as kubeclient
+from kubernetes import config as kubeconf
+
+from kubernetes.client.rest import ApiException
+from kubernetes.stream import stream
+
+from oslo_log import log as logging
+from urllib3.exceptions import MaxRetryError
+
+LOG = logging.getLogger(__name__)
+
+
+class UtilityContainerClient(object):
+    """Client to execute utilscli command on utility containers"""
+
+    NAMESPACE = 'utility'
+
+    def __init__(self):
+        # Initialize variables
+        self._corev1api_client = None
+        self._appsv1api_client = None
+
+    @property
+    def _corev1api_api_client(self):
+        """Property to get the V1CoreAPI client object"""
+        if self._corev1api_client:
+            return self._corev1api_client
+        else:
+            try:
+                kubeconf.load_kube_config(config_file=self._kubeconfig_file)
+                self._corev1api_client = kubeclient.CoreV1Api()
+                return self._corev1api_client
+            except EnvironmentError as err:
+                LOG.exception(
+                    'Failed to load Kubernetes config file: {}'.format(err))
+                raise KubeConfigException(err)
+
+    @property
+    def _appsv1api_api_client(self):
+        """Property to get the V1AppsAPI client object"""
+        if self._appsv1api_client:
+            return self._appsv1api_client
+        else:
+            try:
+                kubeconf.load_kube_config(config_file=self._kubeconfig_file)
+                self._appsv1api_client = kubeclient.AppsV1Api()
+                return self._appsv1api_client
+            except EnvironmentError as err:
+                LOG.exception(
+                    'Failed to load Kubernetes config file: {}'.format(err))
+                raise KubeConfigException(err)
+
+    @property
+    def _kubeconfig_file(self):
+        """Property to generate kubeconfig file from environment variables"""
+        key = 'KUBECONFIG'
+        if os.environ.get(key) is not None:
+            kube_conf_filename = os.environ.get(key)
+        else:
+            raise KubeEnvVarException(key)
+        if os.path.isfile(kube_conf_filename):
+            return kube_conf_filename
+        else:
+            self._prepare_kube_config(kube_conf_filename)
+            return kube_conf_filename
+
+    def _prepare_kube_config(self, kube_conf_filename):
+        """Method to generate the kube config file"""
+        Path(Path.cwd() / 'etc').mkdir(exist_ok=True)
+        Path(kube_conf_filename).touch()
+        conf = KubeCfg(kube_conf_filename)
+        region_key = 'OS_REGION_NAME'
+        kube_server_key = 'KUBE_SERVER'
+        if os.environ.get(region_key) is not None:
+            server = os.environ.get(kube_server_key)
+        else:
+            raise KubeEnvVarException(kube_server_key)
+
+        if os.environ.get(region_key) is not None:
+            conf.set_cluster(
+                name=os.environ.get(region_key),
+                server=server,
+                insecure_skip_tls_verify=True)
+        else:
+            raise KubeEnvVarException(region_key)
+        username_key = 'OS_USERNAME'
+        if os.environ.get(username_key) is not None:
+            conf.set_context(
+                name='context_uc',
+                user=os.environ.get(username_key),
+                namespace='utility',
+                cluster=os.environ.get(region_key))
+        else:
+            raise KubeEnvVarException(username_key)
+        conf.use_context('context_uc')
+        exec_command_key = 'KUBE_KEYSTONE_AUTH_EXEC'
+        if os.environ.get(exec_command_key) is not None:
+            conf.set_credentials(
+                name=os.environ.get(username_key),
+                exec_command=os.environ.get(exec_command_key),
+                exec_api_version='client.authentication.k8s.io/v1beta1')
+        else:
+            raise KubeEnvVarException(exec_command_key)
+
+    def _get_deployment_selectors(self, deployment_name):
+        """Method to get the deployment selectors of the deployment queried.
+
+        :param deployment_name: if specified the deployment name of the utility pod
+            where the utilscli command is to be executed.
+        :type deployment_name: string
+            where the utilscli command is to be executed.
+        :return: selectors extracted from the deployment
+            returned as a string in the format: "key=value, key1=value2..."
+        :exception:
+            KubeDeploymentNotFoundException -- A custom exception
+            KubeDeploymentNotFoundException is raised if no deployment is
+            found with the with the parameters namespace and deployment_name
+            which is passed as a field_selector.
+        """
+        # Get a specific deployment by passing the deployment metadata name
+        # and the namespace
+        deployment = self._appsv1api_api_client.list_namespaced_deployment(
+            self.NAMESPACE,
+            field_selector='metadata.name={}'.format(deployment_name)).items
+        if deployment:
+            # Get the selectors from the deployment object returned.
+            selector_dict = deployment[0].spec.selector.match_labels
+            # Convert the selector dictionary to a string object.
+            selectors = ', '.join(
+                "{!s}={!s}".format(k, v) for (k, v) in selector_dict.items())
+            return selectors
+        else:
+            raise KubeDeploymentNotFoundException(
+                'Deployment with name {} not found in {} namespace'.format(
+                    deployment_name, self.NAMESPACE))
+
+    def _get_utility_container(self, deployment_name):
+        """Method to get a specific utility container filtered by the selectors
+
+        :param deployment_name: if specified the deployment name of the utility pod
+            where the utilscli command is to be executed.
+        :type deployment_name: string
+            where the utilscli command is to be executed.
+        :return: selectors extracted from the deployment
+            utility_container {V1Pod} -- Returns the first pod matched.
+        :exception: KubePodNotFoundException -- Exception raised if not pods are found.
+        """
+        deployment_selectors = self._get_deployment_selectors(deployment_name)
+        utility_containers = self._corev1api_api_client.list_namespaced_pod(
+            self.NAMESPACE, label_selector=deployment_selectors).items
+        if utility_containers:
+            return utility_containers[0]
+        else:
+            raise KubePodNotFoundException(
+                'No Pods found in Deployment {} with selectors {} in {} '
+                'namespace'.format(
+                    deployment_name, deployment_selectors, self.NAMESPACE))
+
+    def _get_exec_cmd_output(self, utility_container, ex_cmd, default=1):
+        """Exec into a specific utility container, then execute the utilscli
+            command and return the output of the command
+
+        :params utility_container: Utility container where the
+            utilscli command will be executed.
+        :type utility_container: string
+        :params ex_cmd: command to be executed inside the utility container
+        :type ex_cmd: strings
+        :params default: return of cmd_output. optionally can be disabled
+        :type integer: 1 (true)
+        :type ex_cmd: strings
+        :return: Output of command executed in the utility container
+        """
+
+        try:
+            container = utility_container.spec.containers[0].name
+            LOG.info(
+                '\nPod Name: {} \nNamespace: {} \nContainer Name: {} '
+                '\nCommand: {}'.format(
+                    utility_container.metadata.name, self.NAMESPACE, container,
+                    ex_cmd))
+            cmd_output = stream(
+                self._corev1api_api_client.connect_get_namespaced_pod_exec,
+                utility_container.metadata.name,
+                self.NAMESPACE,
+                container=container,
+                command=ex_cmd,
+                stderr=True,
+                stdin=False,
+                stdout=True,
+                tty=False)
+            LOG.info(
+                'Pod Name: {} Command Output: {}'.format(
+                    utility_container.metadata.name, cmd_output))
+            if default is 1:
+                return cmd_output
+        except (ApiException, MaxRetryError) as err:
+            LOG.exception(
+                "An exception occurred in pod "
+                "exec command: {}".format(err))
+            raise KubeApiException(err)
+
+    def exec_cmd(self, deployment_name, cmd):
+        """Get specific utility container using deployment name, call
+
+        method to execute utilscli command and return the output of
+        the command.
+
+        :params deployment_name: deployment name of the utility pod
+            where the utilscli command is to be executed.
+        :type deployment_name: string
+        :params cmd: command to be executed inside the utility container
+        :type cmd: strings
+        :return: Output of command executed in the utility container
+        """
+
+        utility_container = self._get_utility_container(deployment_name)
+        return self._get_exec_cmd_output(utility_container, cmd)

kube_utility_container/tests/unit/services/test_utility_container_client.py

@@ -0,0 +1,103 @@
+# Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from kubernetes import client
+import unittest
+
+from unittest.mock import MagicMock as Mock
+from unittest.mock import patch
+
+from kube_utility_container.services.exceptions import \
+    KubeDeploymentNotFoundException
+from kube_utility_container.services.exceptions import \
+    KubeEnvVarException
+from kube_utility_container.services.exceptions import \
+    KubePodNotFoundException
+from kube_utility_container.services.utility_container_client import \
+    UtilityContainerClient
+
+
+class TestUtilityContainerClient(unittest.TestCase):
+    """Unit tests for Utility Container Client"""
+
+    @patch(
+        'kube_utility_container.services.utility_container_client.'
+        'UtilityContainerClient._get_utility_container')
+    @patch(
+        'kube_utility_container.services.utility_container_client.'
+        'UtilityContainerClient._get_exec_cmd_output')
+    def test_exec_cmd(self, mock_get_exec_cmd_output, mock_utility_container):
+        v1_container_obj = Mock(
+            spec=client.V1Container(
+                name='ceph_utility', image='sha', image_pull_policy='Always'))
+        v1_spec_obj = Mock(spec=client.V1PodSpec(containers=v1_container_obj))
+        v1_meta_obj = Mock(
+            spec=client.V1ObjectMeta(
+                name='clcp-ceph-utility-5454794df8-xqwj5', labels='app=ceph'))
+
+        v1_pod_obj = Mock(
+            spec=client.V1Pod(
+                api_version='v1', metadata=v1_meta_obj, spec=v1_spec_obj))
+        mock_utility_container.return_value = v1_pod_obj
+        mock_get_exec_cmd_output.return_value = "Health OK"
+
+        utility_container_client = UtilityContainerClient()
+        response = utility_container_client.exec_cmd(
+            'clcp-utility', ['utilscli', 'ceph', 'status'])
+
+        self.assertIsNotNone(response)
+        self.assertIsInstance(response, str)
+        self.assertEqual(response, mock_get_exec_cmd_output.return_value)
+
+    @patch(
+        'kube_utility_container.services.utility_container_client.'
+        'UtilityContainerClient._get_utility_container',
+        side_effect=KubePodNotFoundException('utility'))
+    def test_exec_cmd_no_utility_pods_returned(self, mock_list_pods):
+        mock_list_pods.return_value = []
+        utility_container_client = UtilityContainerClient()
+        with self.assertRaises(KubePodNotFoundException):
+            utility_container_client.exec_cmd(
+                'clcp-utility', ['utilscli', 'ceph', 'status'])
+
+    @patch(
+        'kube_utility_container.services.utility_container_client.'
+        'UtilityContainerClient._get_deployment_selectors',
+        side_effect=KubeDeploymentNotFoundException('utility'))
+    @patch(
+        'kube_utility_container.services.utility_container_client.'
+        'UtilityContainerClient._corev1api_api_client')
+    def test_exec_cmd_no_deployments_returned(self, deployment, api_client):
+        deployment.return_value = []
+        api_client.return_value = []
+        utility_container_client = UtilityContainerClient()
+        with self.assertRaises(KubeDeploymentNotFoundException):
+            utility_container_client.exec_cmd(
+                'clcp-ceph-utility', ['utilscli', 'ceph', 'status'])
+
+    @patch(
+        'kube_utility_container.services.utility_container_client.'
+        'UtilityContainerClient._get_deployment_selectors',
+        side_effect=KubeEnvVarException('utility'))
+    @patch(
+        'kube_utility_container.services.utility_container_client.'
+        'UtilityContainerClient._appsv1api_api_client',
+        side_effect=KubeEnvVarException('KUBECONFIG'))
+    def test_env_var_kubeconfig_not_set_raises_exception(self, deployment, api_client):
+        deployment.return_value = []
+        api_client.return_value = []
+        utility_container_client = UtilityContainerClient()
+        with self.assertRaises(KubeEnvVarException):
+            utility_container_client.exec_cmd(
+                'clcp-ceph-utility', ['utilscli', 'ceph', 'status'])

kube_utility_container/tests/utility/base.py

@@ -0,0 +1,25 @@
+# Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from kube_utility_container.services.utility_container_client\
+    import UtilityContainerClient
+
+
+class TestBase(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.client = UtilityContainerClient()
+

kube_utility_container/tests/utility/calico/test_calico_utility_container.py

@@ -0,0 +1,32 @@
+# Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from kube_utility_container.tests.utility.base import TestBase
+
+class TestCalicoUtilityContainer(TestBase):
+    @classmethod
+    def setUpClass(cls):
+        cls.deployment_name = 'calicoctl-utility'
+        super(TestCalicoUtilityContainer, cls).setUpClass()
+
+    def test_verify_calico_client_calicoctl_is_present(self):
+        """To verify calico-utility calicoctl is present"""
+        exec_cmd = ['utilscli', 'calicoctl', 'version']
+        expected = 'Client Version:'
+        result_set = self.client.exec_cmd(self.deployment_name, exec_cmd)
+        self.assertIn(
+            expected, result_set, 'Unexpected value for command: {}, '
+            'Command Output: {}'.format(exec_cmd, result_set))

kube_utility_container/tests/utility/ceph/test_ceph_utility_container.py

@@ -0,0 +1,31 @@
+# Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from kube_utility_container.tests.utility.base import TestBase
+
+class TestCephUtilityContainer(TestBase):
+    @classmethod
+    def setUpClass(cls):
+        cls.deployment_name = 'ceph-utility'
+        super(TestCephUtilityContainer, cls).setUpClass()
+
+    def test_verify_ceph_is_healthy(self):
+        """To verify ceph-utility is healthy"""
+        exec_cmd = ['utilscli', 'ceph', 'status']
+        expected = 'HEALTH_OK'
+        result_set = self.client.exec_cmd(self.deployment_name, exec_cmd)
+        self.assertIn(
+            expected, result_set, 'Unexpected value for command: {}, '
+            'Command Output: {}'.format(exec_cmd, result_set))

kube_utility_container/tests/utility/compute/test_compute_utility_container.py

@@ -0,0 +1,49 @@
+# Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+import re
+import os
+
+from kube_utility_container.tests.utility.base import TestBase
+
+node = os.uname().nodename
+
+class TestComputeUtilityContainer(TestBase):
+    @classmethod
+    def setUpClass(cls):
+        cls.deployment_name = 'compute-utility'
+        super(TestComputeUtilityContainer, cls).setUpClass()
+
+    @unittest.expectedFailure
+    def test_verify_compute_ovsclient_is_present(self):
+        """To verify compute-utility ovs-client is present."""
+        cmd = 'ovs-client '
+        exec_cmd = ['utilscli', cmd + node, 'ovs-vsctl -V']
+        expected = 'ovs-vsctl'
+        result_set = self.client.exec_cmd(self.deployment_name, exec_cmd)
+        self.assertIn(
+            expected, result_set, 'Unexpected value for command: {}, '
+            'Command Output: {}'.format(exec_cmd, result_set))
+
+    @unittest.expectedFailure
+    def test_verify_compute_libvirtclient_is_present_on_host(self):
+        """To verify compute-utility Libvirt-client is present."""
+        cmd = 'libvirt-client '
+        exec_cmd = ['utilscli', cmd + node, 'virsh list']
+        expected = 'Id'
+        result_set = self.client.exec_cmd(self.deployment_name, exec_cmd)
+        self.assertIn(
+            expected, result_set, 'Unexpected value for command: {}, '
+            'Command Output: {}'.format(exec_cmd, result_set))

kube_utility_container/tests/utility/etcd/test_etcd_utility_container.py

@@ -0,0 +1,43 @@
+# Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import unittest
+
+from kube_utility_container.tests.utility.base import TestBase
+
+class TestEtcdUtilityContainer(TestBase):
+    @classmethod
+    def setUpClass(cls):
+        cls.deployment_name = 'etcdctl-utility'
+        super(TestEtcdUtilityContainer, cls).setUpClass()
+
+    def test_verify_etcd_ctl_is_present(self):
+        """To verify etcdctl-utility etcdctl is present."""
+        exec_cmd = ['utilscli', 'etcdctl', 'version']
+        expected = 'etcdctl version:'
+        result_set = self.client.exec_cmd(self.deployment_name, exec_cmd)
+        self.assertIn(
+            expected, result_set, 'Unexpected value for command: {}, '
+            'Command Output: {}'.format(exec_cmd, result_set))
+
+    @unittest.expectedFailure
+    def test_verify_etcd_endpoint_is_healthy(self):
+        """To verify etcdctl-utility endpoint is healthy"""
+        exec_cmd = ['utilscli', 'etcdctl', 'endpoint health']
+        expected = 'is health: successfully'
+        result_set = self.client.exec_cmd(self.deployment_name, exec_cmd)
+        self.assertIn(
+            expected, result_set, 'Unexpected value for command: {}, '
+            'Command Output: {}'.format(exec_cmd, result_set))

requirements-frozen.txt

@@ -0,0 +1,56 @@
+Babel==2.8.0
+cachetools==4.0.0
+certifi==2020.4.5.1
+chardet==3.0.4
+cliff==3.1.0
+cmd2==0.8.9
+coverage==4.5.1
+debtcollector==2.0.1
+extras==1.0.0
+fixtures==3.0.0
+future==0.18.2
+google-auth==1.13.1
+idna==2.9
+iso8601==0.1.12
+kubeconfig==1.0.1
+kubernetes==10.0.1
+linecache2==1.0.0
+monotonic==1.5
+msgpack==1.0.0
+netaddr==0.7.19
+netifaces==0.10.9
+oauthlib==3.1.0
+oslo.config==6.7.0
+oslo.context==3.0.2
+oslo.i18n==4.0.1
+oslo.log==3.40.1
+oslo.serialization==3.1.1
+oslo.utils==4.1.1
+pbr==3.1.1
+prettytable==0.7.2
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pyinotify==0.9.6
+pyparsing==2.4.7
+pyperclip==1.8.0
+pytest==5.4.1
+python-dateutil==2.8.1
+python-mimeparse==1.6.0
+python-subunit==1.4.0
+pytz==2019.3
+PyYAML==5.3.1
+requests==2.23.0
+requests-oauthlib==1.3.0
+rfc3986==1.4.0
+rsa==4.0
+six==1.14.0
+stestr==2.1.1
+stevedore==1.32.0
+testtools==2.4.0
+traceback2==1.4.0
+unittest2==1.1.0
+urllib3==1.25.8
+voluptuous==0.11.7
+wcwidth==0.1.9
+websocket-client==0.57.0
+wrapt==1.12.1

requirements.txt

@@ -0,0 +1,12 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+# When modifying this file `tox -e freeze-req` must be run to regenerate the requirements-frozen.txt.
+coverage==4.5.1
+kubeconfig==1.0.1
+kubernetes==10.0.1
+oslo.config==6.7.0 # Apache-2.0
+oslo.log==3.40.1 # Apache-2.0
+pbr==3.1.1
+stestr==2.1.1 # Apache-2.0

setup.cfg

@@ -0,0 +1,21 @@
+[metadata]
+name = porthole_plugin
+version = 1.0.0
+summary = 'This plugin consists set of testcases for utility containers based on kubernetes exec api.'
+description-file = README.rst
+author = The Airship Authors
+author-email = airship-discuss@lists.airshipit.org
+home-page = https://opendev.org/airship/porthole
+
+classifier =
+    Intended Audience :: Information Technology
+    Intended Audience :: System Administrators
+    License :: OSI Approved :: Apache Software License
+    Operating System :: POSIX :: Linux
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.5
+    Programming Language :: Python :: 3.6
+
+[files]
+packages =
+    porthole

setup.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+# Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from setuptools import setup
+
+try:
+    import multiprocessing  # noqa
+except ImportError:
+    pass
+
+setup(
+    setup_requires=['setuptools>=17.1', 'pbr>=2.0.0'],
+    pbr=True
+)

test-requirements.txt

@@ -0,0 +1,22 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+
+# When modifying this file `tox -e freeze-testreq` must be run to regenerate the test-requirements-frozen.txt.
+
+astroid==2.3.3
+bandit==1.5.1
+
+flake8==3.6.0
+hacking==0.12.0 # Apache-2.0
+
+coverage==4.5.1 # Apache-2.0
+pylint==2.4.4
+python-subunit==1.3.0 # Apache-2.0/BSD
+oslotest==3.7.0 # Apache-2.0
+stestr==2.1.1 # Apache-2.0
+testtools==2.3.0 # MIT
+mock==3.0.5
+nose==1.3.7
+responses==0.10.2
+yapf==0.24.0

tools/gate/playbooks/make-feature-tests.yaml

@@ -0,0 +1,23 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Execute the make target for features testing
+      make:
+        chdir: "{{ zuul.project.src_dir }}"
+        target: feature_tests
+      register: result
+      failed_when: result.failed
+
+- name: Include collecting running logs
+  import_playbook: airship-porthole-collect-logs.yaml

tools/gate/playbooks/make-unit-tests.yaml

@@ -0,0 +1,23 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Execute the make target for unit testing
+      make:
+        chdir: "{{ zuul.project.src_dir }}"
+        target: unit_tests
+      register: result
+      failed_when: result.failed
+
+- name: Include collecting running logs
+  import_playbook: airship-porthole-collect-logs.yaml

tools/run_avt.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# Copyright 2020 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+TYPE=$1
+VENV=$(mktemp -d)
+PLUGINS=kube_utility_container
+export KUBECONFIG=${KUBECONFIG:-~/.kube/config}
+
+function setup_venv() {
+    sudo  apt-get install libffi-dev libssl-dev -y
+    python3 -m venv ${VENV}
+    if [[ -f ${VENV}/bin/activate ]] ;then
+      source $VENV/bin/activate
+      ${VENV}/bin/pip install -r requirements-frozen.txt
+      ${VENV}/bin/python -m pip list --format=columns
+      kubectl get deployment -n utility
+      kubectl get nodes -o wide
+      kubectl get po --all-namespaces -o wide
+      stestr init
+    fi
+}
+
+function run_avt() {
+    setup_venv
+    if [[ ${TYPE} == 'unit_tests' ]] ; then
+	run_unit_tests
+    elif [[ ${TYPE} == 'feature_tests' ]] ; then
+	run_feature_tests
+    else
+	echo "No validating tests performed..skip"
+    fi
+}
+
+function run_feature_tests() {
+      python -m unittest discover -s ${PLUGINS}/tests/utility/compute -v
+      python -m unittest discover -s ${PLUGINS}/tests/utility/etcd -v
+      python -m unittest discover -s ${PLUGINS}/tests/utility/calico -v
+      python -m unittest discover -s ${PLUGINS}/tests/utility/ceph -v
+}
+
+function run_unit_tests() {
+      python -m unittest discover -s ${PLUGINS}/tests/unit/services -v
+}
+
+run_avt

tox.ini

@@ -0,0 +1,56 @@
+[tox]
+minversion = 3.4
+envlist = dev,pep8,py36,bandit,docs,list-tests
+skipsdist = true
+
+[testenv:dev]
+useddevelop = True
+basepython = python3
+passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
+setenv =
+  VIRTUAL_ENV={envdir}
+  PYTHONWARNINGS=default::DeprecationWarning
+
+deps =
+  freeze-req: -r{toxinidir}/requirements-frozen.txt
+  list-tests: -r {toxinidir}/requirements-frozen.txt
+
+envdir=
+  {toxinidir}/.tox/{envname}
+
+commands_pre =
+  find . -type f -name "*.pyc"
+  list-tests: stestr init
+
+[testenv:venv]
+commands = {posargs}
+
+[testenv:py36]
+setenv =
+  PYTHONWARNING=all
+deps = -r{toxinidir}/requirements-frozen.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+    pytest  {posargs}
+
+[testenv:bandit]
+deps =
+    -r{toxinidir}/test-requirements.txt
+commands =
+    bandit -r {toxinidir}
+
+[testenv:docs]
+whitelist_externals = rm
+deps =
+    -r{toxinidir}/docs/requirements.txt
+commands =
+    rm -rf docs/build
+    sphinx-build -W -b html docs/source docs/build/html
+
+[testenv:pep8]
+deps =
+    -r{toxinidir}/test-requirements.txt
+commands =
+    yapf -rd {toxinidir} {toxinidir}/tests
+    flake8 {toxinidir}
+    bandit -r {toxinidir}

zuul.d/base.yaml

@@ -25,6 +25,8 @@
         - airship-porthole-images-build-gate-openstack-utility
         - airship-porthole-images-build-gate-postgresql-utility
         - airship-porthole-deploy
+        - airship-porthole-unit-tests
+        - airship-porthole-feature-tests
 
     gate:
       jobs:
@@ -39,6 +41,8 @@
         - airship-porthole-deploy
         - airship-porthole-apparmor:
             voting: false
+        - airship-porthole-unit-tests
+        - airship-porthole-feature-tests
 
     experimental:
       jobs:
@@ -130,6 +134,56 @@
       args:
         chdir: "{{ zuul.project.src_dir }}"
 
+- job:
+    name: airship-porthole-unit-tests
+    description: |
+      Executes unit tests
+    dependencies:
+      - airship-porthole-deploy
+    run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
+    nodeset: airship-porthole-single-node
+    timeout: 7200
+    post-run: tools/gate/playbooks/make-unit-tests.yaml
+    vars:
+      gate_scripts:
+         - ./tools/deployment/utilities/000-install-packages.sh
+         - ./tools/deployment/utilities/001-setup-apparmor-profiles.sh
+         - ./tools/deployment/utilities/002-deploy-k8s.sh
+         - ./tools/deployment/utilities/005-calicoctl-utility.sh
+         - ./tools/deployment/utilities/010-ceph-utility.sh
+         - ./tools/deployment/utilities/020-compute-utility.sh
+         - ./tools/deployment/utilities/030-etcdctl-utility.sh
+         - ./tools/deployment/utilities/040-mysqlclient-utility.sh
+         - ./tools/deployment/utilities/050-openstack-utility.sh
+         - ./tools/deployment/utilities/060-postgresql-utility.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+
+- job:
+    name: airship-porthole-feature-tests
+    description: |
+      Executes feature tests
+    dependencies:
+      - airship-porthole-deploy
+    run: tools/gate/playbooks/airship-porthole-gate-runner.yaml
+    nodeset: airship-porthole-single-node
+    timeout: 7200
+    post-run: tools/gate/playbooks/make-feature-tests.yaml
+    vars:
+      gate_scripts:
+         - ./tools/deployment/utilities/000-install-packages.sh
+         - ./tools/deployment/utilities/001-setup-apparmor-profiles.sh
+         - ./tools/deployment/utilities/002-deploy-k8s.sh
+         - ./tools/deployment/utilities/005-calicoctl-utility.sh
+         - ./tools/deployment/utilities/010-ceph-utility.sh
+         - ./tools/deployment/utilities/020-compute-utility.sh
+         - ./tools/deployment/utilities/030-etcdctl-utility.sh
+         - ./tools/deployment/utilities/040-mysqlclient-utility.sh
+         - ./tools/deployment/utilities/050-openstack-utility.sh
+         - ./tools/deployment/utilities/060-postgresql-utility.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+
 - secret:
     name: quay_credentials
     data: