Merge "Implement AIO changes to allow test usage by roles"

doc/source/user/aio/quickstart.rst

@@ -204,14 +204,6 @@ executing:
 
    # scripts/bootstrap-aio.sh
 
-If you wish to use a different scenario, for example, the Ceph scenario,
-execute the following:
-
-.. code-block:: shell-session
-
-   # export SCENARIO='ceph'
-   # scripts/bootstrap-aio.sh
-
 To add OpenStack Services over and above the bootstrap-aio default services
 for the applicable scenario, copy the ``conf.d`` files with the ``.aio`` file
 extension into ``/etc/openstack_deploy`` and rename then to ``.yml`` files.
@@ -224,6 +216,29 @@ following:
    # cp etc/openstack_deploy/conf.d/{aodh,gnocchi,ceilometer}.yml.aio /etc/openstack_deploy/conf.d/
    # for f in $(ls -1 /etc/openstack_deploy/conf.d/*.aio); do mv -v ${f} ${f%.*}; done
 
+It is possible to also do this (and change other defaults) during the bootstrap
+script initial execution by changing the SCENARIO environment variable before
+running the script. The key word 'aio' will ensure that a basic set of
+OpenStack services (cinder, glance, horizon, neutron, nova) will be deployed.
+The key words 'lxc' and 'nspawn' can be used to set the container back-end,
+while the key word 'metal' will deploy all services without containers. In
+order to implement any other services, add the name of the conf.d file name
+without the `.yml.aio` extension into the SCENARIO environment variable. Each
+key word should be delimited by an underscore. For example, the following will
+implement an AIO with barbican, cinder, glance, horizon, neutron, and nova. It
+will set the cinder storage back-end to ceph and will make use of LXC as the
+container back-end.
+
+.. code-block:: shell-session
+
+   # export SCENARIO='aio_lxc_barbican'
+   # scripts/bootstrap-aio.sh
+
+.. note::
+   If the 'metal' and 'aio' key words are used together, horizon will not be
+   deployed because haproxy and horizon will conflict on the same listening
+   ports.
+
 To add any global overrides, over and above the defaults for the applicable
 scenario, edit ``/etc/openstack_deploy/user_variables.yml``. In order to
 understand the various ways that you can override the default behaviour

etc/openstack_deploy/openstack_user_config.yml.aio.j2

@@ -1,9 +1,9 @@
 ---
 cidr_networks:
-{% if bootstrap_host_scenario == "translations" %}
+{% if 'trove' in bootstrap_host_services %}
   dbaas: 172.29.232.0/22
 {% endif %}
-{% if bootstrap_host_scenario in ["octavia"] %}
+{% if 'octavia' in bootstrap_host_services %}
   lbaas: 172.29.252.0/22
 {% endif %}
   container: 172.29.236.0/22
@@ -11,11 +11,11 @@ cidr_networks:
   storage: 172.29.244.0/22
 
 used_ips:
-{% if bootstrap_host_scenario == "translations" %}
+{% if 'trove' in bootstrap_host_services %}
   - "172.29.232.1,172.29.232.50"
   - "172.29.232.100"
 {% endif %}
-{% if bootstrap_host_scenario in ["translations", "octavia"] %}
+{% if 'octavia' in bootstrap_host_services %}
   - "172.29.252.1,172.29.252.50"
   - "172.29.252.100"
 {% endif %}
@@ -55,7 +55,7 @@ global_overrides:
         net_name: "vxlan"
         group_binds:
           - neutron_linuxbridge_agent
-{% if bootstrap_host_scenario == "translations" %}
+{% if 'trove' in bootstrap_host_services %}
     - network:
         container_bridge: "br-dbaas"
         container_type: "veth"
@@ -68,7 +68,7 @@ global_overrides:
           - neutron_linuxbridge_agent
           - rabbitmq
 {% endif %}
-{% if bootstrap_host_scenario in ["octavia"] %}
+{% if 'octavia' in bootstrap_host_services %}
     - network:
         container_bridge: "br-lbaas"
         container_type: "veth"

tests/bootstrap-aio.yml

@@ -22,7 +22,6 @@
     - role: "bootstrap-host"
   vars_files:
     - "{{ playbook_dir }}/../playbooks/defaults/repo_packages/openstack_services.yml"
-    - vars/bootstrap-aio-vars.yml
   environment: "{{ deployment_environment_variables | default({}) }}"
   vars:
     ansible_python_interpreter: "/usr/bin/python"

tests/roles/bootstrap-host/defaults/main.yml

@@ -40,23 +40,6 @@ bootstrap_host_target_config_paths:
 # The user variables template to use
 bootstrap_user_variables_template: user_variables.aio.yml.j2
 
-# Extra user variables files can be loaded into /etc/openstack_deploy by
-# test scenarios. The dict uses scenario as the key to load a list of extra
-# templates if necessary.
-bootstrap_user_variables_extra_templates:
-  ceph:
-    - src: user_variables_ceph.yml.j2
-      dest: user_variables_ceph.yml
-  congress:
-    - src: user_variables_congress.yml.j2
-      dest: user_variables_congress.yml
-  translations:
-    - src: user_variables_translations.yml.j2
-      dest: user_variables_translations.yml
-  barbican:
-    - src: user_variables_barbican.yml.j2
-      dest: user_variables_barbican.yml
-
 ## Loopback volumes
 # Sparse loopback disks are used for the containers if there is no secondary
 # disk available to partition for btrfs. They are also used for Ceph, Cinder,
@@ -99,7 +82,7 @@ bootstrap_host_loopback_zfs: yes
 bootstrap_host_loopback_zfs_size: 1024
 #
 # Boolean option to deploy the OSD loopback disks and cluster UUID for Ceph
-bootstrap_host_ceph: "{{ (bootstrap_host_scenario == 'ceph') | bool }}"
+bootstrap_host_ceph: "{{ 'ceph' in bootstrap_host_services }}"
 # Size of the Ceph OSD loopbacks
 bootstrap_host_loopback_ceph_size: 1024
 # Ceph OSDs to create on the AIO host

tests/roles/bootstrap-host/tasks/check-requirements.yml

@@ -20,11 +20,32 @@
         (ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'bionic') or
         (ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7') or
         (ansible_os_family == 'Suse' and ansible_distribution_major_version in ['42', '15'])
-    msg: "The only supported platforms for this release are Ubuntu 16.04 LTS (Xenial), Ubuntu 18.04 LTS (Bionic), CentOS 7 (WIP), openSUSE Leap 42.X and openSUSE Leap 15.X"
+    msg: >-
+      The only supported platforms for this release are Ubuntu 16.04 LTS (Xenial),
+      Ubuntu 18.04 LTS (Bionic), CentOS 7, openSUSE Leap 42.X and openSUSE Leap 15.X
   when: (check_operating_system | default(True))| bool
   tags:
     - check-operating-system
 
+- name: Check that bootstrap_host_scenario is not set with mutually exclusive options
+  assert:
+    that:
+      - not item.conditional
+    msg: "{{ item.msg }}"
+  with_items:
+    - conditional: "{{ (bootstrap_host_scenario is search('distro')) and (bootstrap_host_scenario is search('source')) }}"
+      msg: "The scenario key words 'distro' and 'source' are mutually exclusive."
+    - conditional: "{{ (bootstrap_host_scenario is search('lxc')) and (bootstrap_host_scenario is search('nspawn')) }}"
+      msg: "The scenario key words 'lxc' and 'nspawn' are mutually exclusive."
+    - conditional: "{{ (bootstrap_host_scenario is search('lxc')) and (bootstrap_host_scenario is search('metal')) }}"
+      msg: "The scenario key words 'lxc' and 'metal' are mutually exclusive."
+    - conditional: "{{ (bootstrap_host_scenario is search('metal')) and (bootstrap_host_scenario is search('nspawn')) }}"
+      msg: "The scenario key words 'metal' and 'nspawn' are mutually exclusive."
+    - conditional: "{{ (bootstrap_host_scenario is search('metal')) and (bootstrap_host_scenario is search('haproxy')) }}"
+      msg: "The scenario key words 'metal' and 'haproxy' are mutually exclusive."
+    - conditional: "{{ (bootstrap_host_scenario is search('metal')) and (bootstrap_host_scenario is search('horizon')) }}"
+      msg: "The scenario key words 'metal' and 'horizon' are mutually exclusive."
+
 - name: Identify the space available in /
   # NOTE(hwoarang): df does not work reliably on btrfs filesystems
   # https://btrfs.wiki.kernel.org/index.php/FAQ#How_much_free_space_do_I_have.3F

tests/roles/bootstrap-host/tasks/prepare_aio_config.yml

@@ -23,11 +23,11 @@
 
 - name: Deploy user conf.d configuration
   config_template:
-    src: "{{ item.path | default(bootstrap_host_aio_config_path ~ '/conf.d') }}/{{ item.name }}"
+    src: "{{ bootstrap_host_aio_config_path }}/conf.d/{{ item }}.yml.aio"
-    dest: "/etc/openstack_deploy/conf.d/{{ item.name | regex_replace('.aio$', '') }}"
+    dest: "/etc/openstack_deploy/conf.d/{{ item }}.yml"
     config_overrides: "{{ item.override | default({}) }}"
     config_type: "yaml"
-  with_items: "{{ confd_overrides[bootstrap_host_scenario] | default([]) }}"
+  with_items: "{{ bootstrap_host_services }}"
   tags:
     - deploy-confd
 
@@ -175,21 +175,35 @@
     dest: "/etc/openstack_deploy/{{ item.dest }}"
     config_overrides: "{{ item.config_overrides | default({}) }}"
     config_type: yaml
-  with_items: "{{ bootstrap_user_variables_extra_templates[bootstrap_host_scenario] | default([]) }}"
+  when:
+    - "item.condition | bool"
+  with_items:
+    - src: user_variables_ceph.yml.j2
+      dest: user_variables_ceph.yml
+      condition: "{{ 'ceph' in bootstrap_host_services }}"
+    - src: user_variables_congress.yml.j2
+      dest: user_variables_congress.yml
+      condition: "{{ 'congress' in bootstrap_host_services }}"
+    - src: user_variables_translations.yml.j2
+      dest: user_variables_translations.yml
+      condition: "{{ bootstrap_host_scenario is search('translations') }}"
+    - src: user_variables_barbican.yml.j2
+      dest: user_variables_barbican.yml
+      condition: "{{ 'barbican' in bootstrap_host_services }}"
 
 - name: Copy modified cinder-volume env.d file for ceph scenario
   copy:
     src: "{{ playbook_dir }}/../etc/openstack_deploy/env.d/cinder-volume.yml.container.example"
     dest: "/etc/openstack_deploy/env.d/cinder-volume.yml"
   when:
-    - "bootstrap_host_scenario == 'ceph'"
+    - "'ceph' in bootstrap_host_services"
 
 - name: Copy modified env.d file for metal scenario
   copy:
     src: "{{ playbook_dir }}/../etc/openstack_deploy/env.d/aio_metal.yml.example"
     dest: "/etc/openstack_deploy/env.d/aio_metal.yml"
   when:
-    - "bootstrap_host_scenario == 'aio_metal'"
+    - "bootstrap_host_scenario is search('metal')"
 
 - name: Create vars override folders if we need to test them
   file:

tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2

@@ -202,7 +202,7 @@ openstack_user_kernel_options:
   - key: 'kernel.printk'
     value: '4 1 7 4'
 
-{% if bootstrap_host_scenario in ['octavia'] %}
+{% if 'octavia' in bootstrap_host_services %}
 # Octavia specific stuff
 neutron_lbaas_octavia: True
 octavia_management_net_subnet_cidr: "{{ (bootstrap_host_container_tech == 'nspawn') | ternary('172.29.240.0/22', '172.29.252.0/22') }}"

tests/roles/bootstrap-host/vars/main.yml

@@ -0,0 +1,33 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bootstrap_host_services: >-
+  {%- set scenario_list = (bootstrap_host_scenario.split('_') | reject('equalto', '')) | list %}
+  {%- set service_list = ['keystone'] %}
+  {%- set service_list_extra = scenario_list | difference(['aio', 'distro', 'lxc', 'nspawn', 'metal', 'source', 'translations']) %}
+  {%- if 'metal' not in scenario_list %}
+  {%-   set _ = service_list.append('haproxy') %}
+  {%- endif %}
+  {%- if 'aio' in scenario_list or 'translations' in scenario_list %}
+  {%-   set _ = service_list.extend(['cinder', 'glance', 'neutron', 'nova']) %}
+  {%-   if 'metal' not in scenario_list %}
+  {%-     set _ = service_list.append('horizon') %}
+  {%-   endif %}
+  {%- endif %}
+  {%- if 'translations' in scenario_list %}
+  {%-   set _ = service_list.extend(['designate', 'heat', 'magnum', 'sahara', 'swift', 'trove']) %}
+  {%- endif %}
+  {%- set _ = service_list.extend(service_list_extra) %}
+  {{- (service_list | unique) | sort }}

tests/vars/bootstrap-aio-vars.yml

@@ -1,90 +0,0 @@
----
-# Copyright 2017, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-confd_overrides:
-  aio_lxc:
-    - name: cinder.yml.aio
-    - name: glance.yml.aio
-    - name: haproxy.yml.aio
-    - name: horizon.yml.aio
-    - name: keystone.yml.aio
-    - name: neutron.yml.aio
-    - name: nova.yml.aio
-  aio_metal:
-    - name: cinder.yml.aio
-    - name: glance.yml.aio
-    - name: keystone.yml.aio
-    - name: neutron.yml.aio
-    - name: nova.yml.aio
-  aio_nspawn:
-    - name: cinder.yml.aio
-    - name: glance.yml.aio
-    - name: haproxy.yml.aio
-    - name: horizon.yml.aio
-    - name: keystone.yml.aio
-    - name: neutron.yml.aio
-    - name: nova.yml.aio
-  blazar:
-    - name: haproxy.yml.aio
-    - name: nova.yml.aio
-    - name: neutron.yml.aio
-    - name: keystone.yml.aio
-    - name: glance.yml.aio
-    - name: blazar.yml.aio
-  ceph:
-    - name: haproxy.yml.aio
-    - name: ceph.yml.aio
-    - name: cinder.yml.aio
-    - name: glance.yml.aio
-    - name: keystone.yml.aio
-    - name: neutron.yml.aio
-    - name: nova.yml.aio
-  congress:
-    - name: congress.yml.aio
-    - name: haproxy.yml.aio
-    - name: glance.yml.aio
-    - name: keystone.yml.aio
-    - name: nova.yml.aio
-    - name: neutron.yml.aio
-  translations:
-    - name: cinder.yml.aio
-    - name: designate.yml.aio
-    - name: glance.yml.aio
-    - name: haproxy.yml.aio
-    - name: heat.yml.aio
-    - name: horizon.yml.aio
-    - name: keystone.yml.aio
-    - name: magnum.yml.aio
-    - name: neutron.yml.aio
-    - name: nova.yml.aio
-    - name: sahara.yml.aio
-    - name: swift.yml.aio
-    - name: trove.yml.aio
-  octavia:
-    - name: glance.yml.aio
-    - name: haproxy.yml.aio
-    - name: keystone.yml.aio
-    - name: neutron.yml.aio
-    - name: nova.yml.aio
-    - name: octavia.yml.aio
-  tacker:
-    - name: haproxy.yml.aio
-    - name: heat.yml.aio
-    - name: keystone.yml.aio
-    - name: tacker.yml.aio
-  barbican:
-    - name: haproxy.yml.aio
-    - name: keystone.yml.aio
-    - name: barbican.yml.aio

zuul.d/jobs.yaml

@@ -85,20 +85,20 @@
       install_method: distro
 
 - job:
-    name: openstack-ansible-deploy-ceph-ubuntu-bionic
+    name: openstack-ansible-deploy-aio_ceph-ubuntu-bionic
     parent: openstack-ansible-deploy-aio
     nodeset: ubuntu-bionic
     vars:
       action: deploy
-      scenario: ceph
+      scenario: aio_ceph
 
 - job:
-    name: openstack-ansible-deploy-distro_ceph-ubuntu-bionic
+    name: openstack-ansible-deploy-aio_distro_ceph-ubuntu-bionic
     parent: openstack-ansible-deploy-aio
     nodeset: ubuntu-bionic
     vars:
       action: deploy
-      scenario: ceph
+      scenario: aio_ceph
       install_method: distro
 
 - job:
@@ -126,21 +126,21 @@
       scenario: aio_nspawn
 
 - job:
-    name: openstack-ansible-upgrade-ceph-ubuntu-bionic
+    name: openstack-ansible-upgrade-aio_ceph-ubuntu-bionic
     parent: openstack-ansible-deploy-aio
     nodeset: ubuntu-bionic
     timeout: 10800
     vars:
       action: upgrade
-      scenario: ceph
+      scenario: aio_ceph
 
 - job:
-    name: openstack-ansible-deploy-octavia-ubuntu-bionic
+    name: openstack-ansible-deploy-aio_octavia-ubuntu-bionic
     parent: openstack-ansible-deploy-aio
     nodeset: ubuntu-bionic
     vars:
       action: deploy
-      scenario: octavia
+      scenario: aio_octavia
 
 - job:
     name: openstack-ansible-deploy-translations-ubuntu-bionic
@@ -151,12 +151,12 @@
       scenario: translations
 
 - job:
-    name: openstack-ansible-deploy-congress-ubuntu-bionic
+    name: openstack-ansible-deploy-aio_congress-ubuntu-bionic
     parent: openstack-ansible-deploy-aio
     nodeset: ubuntu-bionic
     vars:
       action: deploy
-      scenario: congress
+      scenario: aio_congress
 
 - job:
     name: openstack-ansible-deploy-barbican-ubuntu-bionic
@@ -187,20 +187,20 @@
       install_method: distro
 
 - job:
-    name: openstack-ansible-deploy-ceph-centos-7
+    name: openstack-ansible-deploy-aio_ceph-centos-7
     parent: openstack-ansible-deploy-aio
     nodeset: centos-7
     vars:
       action: deploy
-      scenario: ceph
+      scenario: aio_ceph
 
 - job:
-    name: openstack-ansible-deploy-distro_ceph-centos-7
+    name: openstack-ansible-deploy-aio_distro_ceph-centos-7
     parent: openstack-ansible-deploy-aio
     nodeset: centos-7
     vars:
       action: deploy
-      scenario: ceph
+      scenario: aio_ceph
       install_method: distro
 
 - job:
@@ -228,13 +228,13 @@
       scenario: aio_nspawn
 
 - job:
-    name: openstack-ansible-upgrade-ceph-centos-7
+    name: openstack-ansible-upgrade-aio_ceph-centos-7
     parent: openstack-ansible-deploy-aio
     nodeset: centos-7
     timeout: 10800
     vars:
       action: upgrade
-      scenario: ceph
+      scenario: aio_ceph
 
 # opensuse
 - job:
@@ -262,37 +262,37 @@
       install_method: distro
 
 - job:
-    name: openstack-ansible-deploy-ceph-opensuse-423
+    name: openstack-ansible-deploy-aio_ceph-opensuse-423
     parent: openstack-ansible-deploy-aio
     nodeset: opensuse-423
     vars:
       action: deploy
-      scenario: ceph
+      scenario: aio_ceph
 
 - job:
-    name: openstack-ansible-deploy-ceph-opensuse-150
+    name: openstack-ansible-deploy-aio_ceph-opensuse-150
     parent: openstack-ansible-deploy-aio
     nodeset: opensuse-150
     vars:
       action: deploy
-      scenario: ceph
+      scenario: aio_ceph
 
 - job:
-    name: openstack-ansible-deploy-distro_ceph-opensuse-423
+    name: openstack-ansible-deploy-aio_distro_ceph-opensuse-423
     parent: openstack-ansible-deploy-aio
     nodeset: opensuse-423
     vars:
       action: deploy
-      scenario: ceph
+      scenario: aio_ceph
       install_method: distro
 
 - job:
-    name: openstack-ansible-deploy-distro_ceph-opensuse-150
+    name: openstack-ansible-deploy-aio_distro_ceph-opensuse-150
     parent: openstack-ansible-deploy-aio
     nodeset: opensuse-150
     vars:
       action: deploy
-      scenario: ceph
+      scenario: aio_ceph
       install_method: distro
 
 - job:

zuul.d/project-templates.yaml

@@ -31,8 +31,8 @@
       jobs:
         - openstack-ansible-deploy-aio_nspawn-ubuntu-bionic
         - openstack-ansible-deploy-barbican-ubuntu-bionic
-        - openstack-ansible-deploy-congress-ubuntu-bionic
+        - openstack-ansible-deploy-aio_congress-ubuntu-bionic
-        - openstack-ansible-deploy-octavia-ubuntu-bionic
+        - openstack-ansible-deploy-aio_octavia-ubuntu-bionic
         - openstack-ansible-deploy_with_ansible_devel-aio-ubuntu-bionic
         - openstack-ansible-deploy_with_ansible_next-aio-ubuntu-bionic
         - openstack-ansible-deploy-blazar-ubuntu-bionic
@@ -105,22 +105,22 @@
     name: openstack-ansible-deploy-ceph-jobs
     check:
       jobs:
-        - openstack-ansible-deploy-ceph-ubuntu-bionic
+        - openstack-ansible-deploy-aio_ceph-ubuntu-bionic
     gate:
       jobs:
-        - openstack-ansible-deploy-ceph-ubuntu-bionic
+        - openstack-ansible-deploy-aio_ceph-ubuntu-bionic
     periodic:
       jobs:
-        - openstack-ansible-deploy-ceph-ubuntu-bionic
+        - openstack-ansible-deploy-aio_ceph-ubuntu-bionic
-        - openstack-ansible-upgrade-ceph-ubuntu-bionic
+        - openstack-ansible-upgrade-aio_ceph-ubuntu-bionic
 
 - project-template:
     name: openstack-ansible-deploy-distro_ceph-jobs
     check:
       jobs:
-        - openstack-ansible-deploy-distro_ceph-opensuse-423:
+        - openstack-ansible-deploy-aio_distro_ceph-opensuse-423:
             voting: false
-        - openstack-ansible-deploy-distro_ceph-opensuse-150:
+        - openstack-ansible-deploy-aio_distro_ceph-opensuse-150:
             voting: false
     gate:
       jobs: []