Add profile argument

The patch implements --profile argument, which defines a path to
a profile.yaml file - a file which contains python-tempestconf
arguments and their values.
Also --generate-profile argument is introduced. A user can generate
a sample profile.yaml file using this argument. The sample contains
definitions of all python-tempestconf arguments set to their default
values.

Story: 2004503
Task: 28225

Change-Id: I80848d8ef9868ed3975b938d61880753eadc5ae9

config_tempest/main.py

@@ -49,6 +49,7 @@ from config_tempest import constants as C
 from config_tempest.constants import LOG
 from config_tempest.credentials import Credentials
 from config_tempest.flavors import Flavors
+from config_tempest import profile
 from config_tempest.services.services import Services
 from config_tempest.tempest_conf import TempestConf
 from config_tempest.users import Users
@@ -275,6 +276,21 @@ def get_arg_parser():
                                 For example:
                                   --create-accounts-file $HOME/accounts.yaml
                              """)
+    parser.add_argument('--profile', default=None, metavar='PATH',
+                        help="""python-tempestconf's profile.yaml file
+                                A file which contains definition of
+                                python-tempestconf's arguments.
+                                NOTE: If this argument is used, other
+                                arguments cannot be defined!""")
+    parser.add_argument('--generate-profile', default=None,
+                        metavar='PATH',
+                        help="""Generate a sample profile.yaml file.
+                                A sample profile.yaml will be generated in the
+                                specified path. After that python-tempestconf
+                                ends.
+                                For example:
+                                  --generate-profile $HOME/profile.yaml
+                             """)
     parser.add_argument('--image-disk-format', default=C.DEFAULT_IMAGE_FORMAT,
                         help="""A format of an image to be uploaded to glance.
                                 Default is '%s'""" % C.DEFAULT_IMAGE_FORMAT)
@@ -526,6 +542,16 @@ def config_tempest(**kwargs):
 
 def main():
     args = parse_arguments()
+    if args.generate_profile:
+        profile.generate_profile(args, args.generate_profile)
+        sys.exit(0)
+    if args.profile:
+        profile_args = profile.read_profile_file(args.profile)
+        # update default args by values gained from the profile
+        # Namespace can't be updated, so translate it to a dict first
+        args_dict = vars(args)
+        args_dict.update(profile_args)
+        args = argparse.Namespace(**args_dict)
     cloud_creds = get_cloud_creds(args)
     config_tempest(
         append=args.append,

config_tempest/profile.py

@@ -0,0 +1,103 @@
+# Copyright 2018 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import yaml
+
+
+def _convert_remove_append(options):
+    """Convert append and remove dicts read from .yaml file.
+
+    :param options: {'section.key': 'value1,value2', ...}
+    :type options: dict
+    :return: ['section.key=value1,value2', ...]
+    :rtype: list
+    """
+    converted = []
+    for key in options:
+        v = options[key]
+        if isinstance(v, list):
+            v = ','.join(v)
+        converted.append(key + "=" + v)
+    return converted
+
+
+def read_profile_file(path):
+    """Read python-tempestconf arguments from a .yaml file.
+
+    :param path: path to the profile.yaml file
+    :type path: string
+    :return: profile arguments
+    :rtype: dict
+    """
+    with open(path, 'r') as stream:
+        profile_args = yaml.load(stream)
+    # convert overrides, to a list of tuples (s, k, v)
+    overrides = []
+    if 'overrides' in profile_args:
+        for key in profile_args['overrides']:
+            s, k = key.split('.')
+            v = profile_args['overrides'][key]
+            if isinstance(v, list):
+                v = ','.join(v)
+            overrides.append((s, k, v))
+    profile_args['overrides'] = overrides
+    # convert remove
+    remove = []
+    if 'remove' in profile_args:
+        remove = _convert_remove_append(profile_args['remove'])
+    profile_args['remove'] = remove
+    # convert append values
+    append = []
+    if 'append' in profile_args:
+        append = _convert_remove_append(profile_args['append'])
+    profile_args['append'] = append
+    return profile_args
+
+
+def generate_profile(args, path):
+    """Generates a sample profile.yaml file.
+
+    :type args: argparse.Namespace
+    :param path: Specifies where the sample file will be generated in.
+    :type path: string
+    """
+    iterable_args = vars(args)
+    # pop following arguments as they are written
+    # more detailed to the file below
+    iterable_args.pop('append')
+    iterable_args.pop('overrides')
+    iterable_args.pop('remove')
+    # pop profile as that shouldn't be in a profile.yaml
+    iterable_args.pop('profile')
+    with open(path, 'w') as outfile:
+        yaml.safe_dump(iterable_args, outfile, default_flow_style=False)
+        outfile.write("""append: {}
+  #identity.username: username
+  #compute-feature-enabled.api_extensions:
+  #  - dvr
+  #  - extension
+overrides: {}
+  #identity.username: username
+  #identity.password:
+  #  - my_password
+  #compute-feature-enabled.api_extensions:
+  #  - dvr
+  #  - extension
+remove: {}
+  #identity.username: username
+  #compute-feature-enabled.api_extensions:
+  #  - dvr
+  #  - extension
+""")

config_tempest/tests/test_profile.py

@@ -0,0 +1,32 @@
+# Copyright 2018 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from config_tempest import profile
+from config_tempest.tests.base import BaseConfigTempestTest
+
+
+class TestProfile(BaseConfigTempestTest):
+
+    def test_convert_remove_append(self):
+        in_data = {
+            'section.key1': 'value1',
+            'section.key2': 'value1,value2'
+        }
+        expected = [
+            'section.key1=value1',
+            'section.key2=value1,value2'
+        ]
+        out_data = profile._convert_remove_append(in_data)
+        self.assertItemsEqual(expected, out_data)

doc/source/user/import.rst

@@ -79,6 +79,10 @@ List of arguments which may be passed to `config_tempest`
  * test_accounts
  * verbose
 
+ OR
+
+ * profile, see why **or** in `CLI documentation`_
+
 .. note::
 
     For detailed description of the options see our `CLI documentation`_

doc/source/user/index.rst

@@ -7,4 +7,5 @@ User Guide
 
   usage
   import
+  profile
   default

doc/source/user/profile.rst

@@ -0,0 +1,159 @@
+===============================================
+Use python-tempestconf with a profile.yaml file
+===============================================
+
+A ``profile.yaml`` is helpful mainly in jobs running on different versions of
+OpenStack, because arguments for ``python-tempestconf`` may differ in each
+OpenStack version. Using the ``--profile`` argument those jobs can use
+a ``profile.yaml`` file for each OpenStack version which makes the code of
+those jobs much clearer and more readable, as for example they contain less if
+else statements, ...
+
+.. note::
+
+    Apart from ``--deployer-input`` config file which specifies content of
+    a tempest.conf, ``profile.yaml`` file defines arguments which are passed
+    to ``python-tempestconf``, see `CLI documentation`_.
+
+    .. _CLI documentation: ../cli/cli_options.html
+
+.. warning::
+
+    If this argument is used, other arguments cannot be defined, it means,
+    user uses either CLI arguments or profile.yaml file.
+
+Generating a sample profile.yaml file
++++++++++++++++++++++++++++++++++++++
+
+.. code-block:: Bash
+
+    $ discover-tempest-config --generate-profile ./etc/profile.yaml
+
+.. code-block:: yaml
+
+    $ cat ./etc/profile.yaml
+    collect_timing: false
+    create: false
+    create_accounts_file: null
+    debug: false
+    deployer_input: null
+    endpoint_type: null
+    generate_profile: ./etc/profile.yaml
+    http_timeout: null
+    image: http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img
+    image_disk_format: qcow2
+    insecure: false
+    network_id: null
+    no_default_deployer: false
+    non_admin: false
+    os_api_version: null
+    os_auth_type: password
+    os_auth_url: null
+    os_cacert: null
+    os_cert: null
+    os_cloud: null
+    os_default_domain_id: null
+    os_default_domain_name: null
+    os_domain_id: null
+    os_domain_name: null
+    os_endpoint_override: null
+    os_endpoint_type: null
+    os_interface: public
+    os_key: null
+    os_password: null
+    os_project_domain_id: null
+    os_project_domain_name: null
+    os_project_id: null
+    os_project_name: null
+    os_region_name: null
+    os_service_name: null
+    os_service_type: null
+    os_system_scope: null
+    os_trust_id: null
+    os_user_domain_id: null
+    os_user_domain_name: null
+    os_user_id: null
+    os_username: null
+    out: etc/tempest.conf
+    test_accounts: null
+    timeout: 600
+    verbose: false
+    append: {}
+      #identity.username: username
+      #compute-feature-enabled.api_extensions:
+      #  - dvr
+      #  - extension
+    overrides: {}
+      #identity.username: username
+      #identity.password:
+      #  - my_password
+      #compute-feature-enabled.api_extensions:
+      #  - dvr
+      #  - extension
+    remove: {}
+      #identity.username: username
+      #compute-feature-enabled.api_extensions:
+      #  - dvr
+      #  - extension
+
+.. note::
+
+    The generated sample of a ``profile.yaml`` file contains all
+    ``python-tempestconf`` arguments set to their default values. That means,
+    that you can **remove arguments you didn't modify** to keep the file simple
+    and more readable.
+
+
+``python-tempestconf`` accepts both of the following inputs, so you can use
+what suits you better, either strings or lists:
+
+.. code-block:: yaml
+
+    create: True
+    out: ./etc/tempest.conf
+    deployer-input: ./deploy.txt
+    no-default-deployer: False
+    overrides:
+      identity.username: my_override
+      identity.password: my_password
+      network-feature-enabled.api_extensions: all
+      compute-feature-enabled.api_extensions: dvr,mine
+    remove:
+      auth.identity: username
+      network-feature-enabled.api_extensions: ''
+      compute-feature-enabled.api_extensions: dvr,mine
+
+
+.. code-block:: yaml
+
+    create: True
+    out: ./etc/tempest.conf
+    deployer-input: ./deploy.txt
+    no-default-deployer: False
+    overrides:
+      identity.username: my_override
+      identity.password:
+        - my_password
+      network-feature-enabled.api_extensions:
+        - all
+      compute-feature-enabled.api_extensions:
+        - dvr
+        - mine
+    remove:
+      auth.identity: username
+      network-feature-enabled.api_extensions:
+        - ''
+      compute-feature-enabled.api_extensions:
+        - dvr
+        - mine
+
+
+Using profile.yaml file
++++++++++++++++++++++++
+
+After you've created your customized ``profile.yaml`` file, let's say in
+``./etc/profile.yaml``, use it as follows:
+
+.. code-block:: Bash
+
+    $ discover-tempest-config --profile ./etc/profile.yaml

releasenotes/notes/Add-profile-argument-70bfa9606826ff81.yaml

@@ -0,0 +1,15 @@
+---
+features:
+  - |
+    --profile argument specifies a path to a profile.yaml file which contains
+    definitions of python-tempestconf arguments.
+
+    NOTE: If this argument is used, other arguments cannot be defined, it means
+    a user uses either CLI arguments or profile.yaml file.
+
+    A user can generate a sample profile.yaml file using --generate-profile
+    argument. The sample contains definitions of all python-tempestconf
+    arguments set to their default values.
+
+    NOTE: If this argument is used, python-tempestconf ends right after the
+    a sample profile.yaml file is generated.

roles/generate-tempestconf-file-cloud/tasks/main.yaml

@@ -44,6 +44,53 @@
         chdir: "{{ tempestconf_src_relative_path }}"
         executable: /bin/bash
 
+  - set_fact:
+        profile:
+            debug: true
+            out: etc/tempest_profile.conf
+            verbose: true
+            create: "{{ cloud_user == 'devstack-admin' }}"
+            non_admin: "{{ cloud_user == 'devstack' }}"
+            os_cloud: "{{ cloud_user }}"
+            overrides:
+                auth.tempest_roles: Member
+
+  - name: Generate sample profile.yaml file
+    shell: |
+        set -ex
+        export PATH=$PATH:/usr/local/sbin:/usr/sbin
+        source {{ virtualenvs.tempestconf }}/bin/activate
+        printenv
+        discover-tempest-config \
+          --generate-profile ./etc/profile.yaml
+    args:
+        chdir: "{{ tempestconf_src_relative_path }}"
+        executable: /bin/bash
+
+  - name: Edit the profile.yaml file
+    copy:
+        content: "{{ profile | to_nice_yaml }}"
+        dest: "{{ tempestconf_src_relative_path }}/etc/profile.yaml"
+
+  - name: Generate tempest configuration file with profile.yaml
+    shell: |
+        set -ex
+        export PATH=$PATH:/usr/local/sbin:/usr/sbin
+        source {{ virtualenvs.tempestconf }}/bin/activate
+        printenv
+        cat ./etc/profile.yaml
+        discover-tempest-config \
+          --profile ./etc/profile.yaml
+    args:
+        chdir: "{{ tempestconf_src_relative_path }}"
+        executable: /bin/bash
+
+  - name: Diff tempest.conf and tempest_profile.conf
+    shell: |
+        diff ./etc/cloud_tempest.conf ./etc/tempest_profile.conf
+    args:
+        chdir: "{{ tempestconf_src_relative_path }}"
+
   - name: Print generated tempest.conf
     shell: |
         set -ex