Merge "Quick start Bifrost CLI"

.gitignore

@@ -71,3 +71,7 @@ playbooks/library/os_keystone_service.py
 # Other
 *.DS_Store
 .idea
+
+# Generated by bifrost-cli
+baremetal-inventory.json
+baremetal-nodes.json

bifrost-cli

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+if ! python3 --version > /dev/null; then
+    echo "Python 3 not found, version 3.6 or newer is required for Bifrost"
+    exit 1
+fi
+
+if ! python3 -c "import sys; assert sys.version_info >= (3, 6)" 2> /dev/null; then
+    echo "Python 3.6 or newer is required for Bifrost"
+    exit 1
+fi
+
+PYTHONPATH=$(dirname $0) exec python3 -m bifrost.cli $@

bifrost/__init__.py

@@ -12,11 +12,13 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import pbr.version
+try:
+    import pbr.version
 
-
-__version__ = pbr.version.VersionInfo(
-    'bifrost').version_string()
+    __version__ = pbr.version.VersionInfo(
+        'bifrost').version_string()
+except ImportError:
+    pass  # Allow the CLI to work without pbr installed
 
 __all__ = [
     'inventory'

bifrost/cli.py

@@ -0,0 +1,234 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import configparser
+import ipaddress
+import itertools
+import os
+import subprocess
+import sys
+
+
+VENV = "/opt/stack/bifrost"
+ANSIBLE = os.path.join(VENV, 'bin', 'ansible-playbook')
+COMMON_ENV = {
+    'VENV': VENV,
+    'USE_VENV': 'true',
+    'ENABLE_VENV': 'true',
+}
+COMMON_PARAMS = [
+    '-e', 'ansible_python_interpreter=%s/bin/python3' % VENV,
+    '-e', 'enable_venv=true',
+    '-e', 'bifrost_venv_dir=%s' % VENV,
+]
+BASE = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+PLAYBOOKS = os.path.join(BASE, 'playbooks')
+
+
+def get_env(extra=None):
+    # NOTE(dtantsur): the order here matters!
+    result = os.environ.copy()
+    result.update(COMMON_ENV)
+    if extra:
+        result.update(extra)
+    return result
+
+
+def log(*message, only_if=True):
+    if only_if:
+        print(*message, file=sys.stderr)
+
+
+def ansible(playbook, inventory, verbose=False, env=None, **params):
+    extra = COMMON_PARAMS + list(itertools.chain.from_iterable(
+        ('-e', '%s=%s' % pair) for pair in params.items()
+        if pair[1] is not None))
+    if verbose:
+        extra.append('-vvvv')
+    args = [ANSIBLE, playbook, '-i', inventory] + extra
+    log('Calling ansible with', args, 'and environment', env, only_if=verbose)
+    subprocess.check_call(args, env=get_env(env), cwd=PLAYBOOKS)
+
+
+def env_setup(args):
+    if os.path.exists(VENV):
+        log(VENV, 'exists, skipping environment preparation',
+            only_if=args.debug)
+        return
+
+    log('Installing dependencies and preparing an environment in', VENV)
+    subprocess.check_call(["bash", "scripts/env-setup.sh"],
+                          env=get_env(), cwd=BASE)
+
+
+def get_release(release):
+    if release:
+        if release != 'master' and not release.startswith('stable/'):
+            release = 'stable/%s' % release
+        return release
+    else:
+        try:
+            gr = configparser.ConfigParser()
+            gr.read(os.path.join(BASE, '.gitreview'))
+            release = gr.get('gerrit', 'defaultbranch', fallback='master')
+            log('Using release', release, 'detected from the checkout')
+            return release
+        except (FileNotFoundError, configparser.Error):
+            log('Cannot read .gitreview, falling back to release "master"')
+            return 'master'
+
+
+def cmd_testenv(args):
+    release = get_release(args.release)
+
+    env_setup(args)
+    log('Creating', args.count, 'test node(s) with', args.memory,
+        'MiB RAM and', args.disk, 'GiB of disk')
+
+    kwargs = {}
+    if args.storage_pool_path:
+        kwargs['test_vm_storage_pool_path'] = os.path.abspath(
+            args.storage_pool_path)
+
+    ansible('test-bifrost-create-vm.yaml',
+            inventory='inventory/localhost',
+            verbose=args.debug,
+            git_branch=release,
+            test_vm_num_nodes=args.count,
+            test_vm_memory_size=args.memory,
+            test_vm_disk_gib=args.disk,
+            test_vm_domain_type=args.domain_type,
+            baremetal_json_file=os.path.abspath(args.inventory),
+            baremetal_nodes_json=os.path.abspath(args.output),
+            **kwargs)
+    log('Inventory generated in', args.output)
+
+
+def cmd_install(args):
+    release = get_release(args.release)
+
+    kwargs = {}
+    if args.dhcp_pool:
+        try:
+            start, end = args.dhcp_pool.split('-')
+            ipaddress.ip_address(start)
+            ipaddress.ip_address(end)
+        except ValueError as e:
+            sys.exit("Malformed --dhcp-pool, expected two IP addresses. "
+                     "Error: %s" % e)
+        else:
+            kwargs['dhcp_pool_start'] = start
+            kwargs['dhcp_pool_end'] = end
+
+    env_setup(args)
+    ansible('install.yaml',
+            inventory='inventory/target',
+            verbose=args.debug,
+            git_branch=release,
+            ipa_upstream_release=release.replace('/', '-'),
+            create_ipa_image='false',
+            create_image_via_dib='false',
+            install_dib='true',
+            network_interface=args.network_interface,
+            enable_keystone=args.enable_keystone,
+            use_public_urls=args.enable_keystone,
+            noauth_mode=not args.enable_keystone,
+            testing=args.testenv,
+            use_cirros=args.testenv,
+            use_tinyipa=args.testenv,
+            **kwargs)
+    log("Ironic is installed and running, try it yourself:\n",
+        " $ source %s/bin/activate\n" % VENV,
+        " $ export OS_CLOUD=bifrost\n",
+        " $ baremetal driver list\n"
+        "See documentation for next steps")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser("Bifrost CLI")
+    parser.add_argument('--debug', action='store_true',
+                        help='output extensive logging')
+
+    subparsers = parser.add_subparsers()
+
+    testenv = subparsers.add_parser(
+        'testenv', help='Prepare a virtual testing environment')
+    testenv.set_defaults(func=cmd_testenv)
+    testenv.add_argument('--release', default='master',
+                         help='release branch to use (master, ussuri, etc)')
+    testenv.add_argument('--count', type=int, default=2,
+                         help='number of nodes to create')
+    testenv.add_argument('--memory', type=int, default=3072,
+                         help='memory (in MiB) for test nodes')
+    testenv.add_argument('--disk', type=int, default=10,
+                         help='disk size (in GiB) for test nodes')
+    testenv.add_argument('--domain-type', default='qemu',
+                         help='domain type: qemu or kvm')
+    testenv.add_argument('--storage-pool-path',
+                         help='path to libvirt storage pool to setup')
+    testenv.add_argument('--inventory', default='baremetal-inventory.json',
+                         help='output file with the inventory for using '
+                              'with dynamic playbooks')
+    testenv.add_argument('-o', '--output', default='baremetal-nodes.json',
+                         help='output file with the nodes information for '
+                              'importing into ironic')
+
+    install = subparsers.add_parser('install', help='Install ironic')
+    install.set_defaults(func=cmd_install)
+    install.add_argument('--testenv', action='store_true',
+                         help='running in a virtual environment')
+    install.add_argument('--dhcp-pool', metavar='START-END',
+                         help='DHCP pool to use')
+    install.add_argument('--release',
+                         help='release branch to use (master, ussuri, etc), '
+                              'the default value is determined from the '
+                              '.gitreview file in the source tree')
+    install.add_argument('--network-interface',
+                         help='the network interface to use')
+    install.add_argument('--enable-keystone', action='store_true',
+                         help='enable keystone and use authentication')
+
+    args = parser.parse_args()
+    if getattr(args, 'func', None) is None:
+        parser.print_usage(file=sys.stderr)
+        sys.exit("Bifrost CLI: error: a command is required")
+    return args
+
+
+def check_for_root():
+    try:
+        subprocess.check_call(
+            '[ $(whoami) == root ] || sudo --non-interactive true',
+            shell=True, stderr=subprocess.DEVNULL)
+    except subprocess.CalledProcessError:
+        # TODO(dtantsur): tell ansible to ask for password
+        sys.exit('Sudo without password is required for Bifrost')
+
+
+def main():
+    args = parse_args()
+    try:
+        check_for_root()
+        args.func(args)
+    except Exception as exc:
+        if args.debug:
+            raise
+        else:
+            sys.exit(str(exc))
+    except KeyboardInterrupt:
+        sys.exit('Aborting by user request')
+
+
+if __name__ == '__main__':
+    sys.exit(main())

playbooks/ci/run.yaml

@@ -5,6 +5,7 @@
         chdir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/bifrost'].src_dir }}"
       environment:
         BUILD_IMAGE: "{{ build_image | default(false) | bool | lower }}"
+        CLI_TEST: "{{ cli_test | default(false) | bool | lower }}"
         ENABLE_KEYSTONE: "{{ enable_keystone | default(false) | bool | lower }}"
         LOG_LOCATION: "{{ ansible_user_dir }}/logs"
         UPPER_CONSTRAINTS_FILE: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/requirements'].src_dir }}/upper-constraints.txt"

scripts/test-bifrost.sh

@@ -11,7 +11,8 @@ BUILD_IMAGE="${BUILD_IMAGE:-false}"
 BAREMETAL_DATA_FILE=${BAREMETAL_DATA_FILE:-'/tmp/baremetal.json'}
 ENABLE_KEYSTONE="${ENABLE_KEYSTONE:-false}"
 ZUUL_BRANCH=${ZUUL_BRANCH:-}
-ENABLE_VENV=${ENABLE_VENV:-true}
+ENABLE_VENV=true
+CLI_TEST=${CLI_TEST:-false}
 
 # Set defaults for ansible command-line options to drive the different
 # tests.
@@ -50,22 +51,18 @@ OS_DISTRO="$ID"
 # Setup openstack_ci test database if run in OpenStack CI.
 if [ "$ZUUL_BRANCH" != "" ]; then
     sudo mkdir -p /opt/libvirt/images
-    VM_SETUP_EXTRA="-e test_vm_storage_pool_path=/opt/libvirt/images"
+    VM_SETUP_EXTRA="--storage-pool-path /opt/libvirt/images"
 fi
 
 source $SCRIPT_HOME/env-setup.sh
-if [ ${ENABLE_VENV} = "true" ]; then
-    # Note(cinerama): activate is not compatible with "set -u";
-    # disable it just for this line.
-    set +u
-    source ${VENV}/bin/activate
-    set -u
-    ANSIBLE=${VENV}/bin/ansible-playbook
-    ANSIBLE_PYTHON_INTERP=${VENV}/bin/python3
-else
-    ANSIBLE=${HOME}/.local/bin/ansible-playbook
-    ANSIBLE_PYTHON_INTERP=$(which python3)
-fi
+
+# Note(cinerama): activate is not compatible with "set -u";
+# disable it just for this line.
+set +u
+source ${VENV}/bin/activate
+set -u
+ANSIBLE=${VENV}/bin/ansible-playbook
+ANSIBLE_PYTHON_INTERP=${VENV}/bin/python3
 
 # Adjust options for DHCP, VM, or Keystone tests
 if [ ${USE_DHCP} = "true" ]; then
@@ -120,19 +117,13 @@ for task in syntax-check list-tasks; do
            -e testing_user=${TESTING_USER}
 done
 
-# Create the test VM
-${ANSIBLE} -vvvv \
-       -i inventory/localhost \
-       test-bifrost-create-vm.yaml \
-       -e ansible_python_interpreter="${ANSIBLE_PYTHON_INTERP}" \
-       -e test_vm_num_nodes=${TEST_VM_NUM_NODES} \
-       -e test_vm_memory_size=${VM_MEMORY_SIZE:-512} \
-       -e test_vm_domain_type=${VM_DOMAIN_TYPE} \
-       -e test_vm_disk_gib=${VM_DISK:-5} \
-       -e baremetal_json_file=${BAREMETAL_DATA_FILE} \
-       -e enable_venv=${ENABLE_VENV} \
-       -e bifrost_venv_dir=${VENV} \
-       ${VM_SETUP_EXTRA:-}
+# Create the test VMs
+../bifrost-cli --debug testenv \
+    --count ${TEST_VM_NUM_NODES} \
+    --memory ${VM_MEMORY_SIZE:-512} \
+    --disk ${VM_DISK:-5} \
+    --inventory "${BAREMETAL_DATA_FILE}" \
+    ${VM_SETUP_EXTRA:-}
 
 if [ ${USE_DHCP} = "true" ]; then
     # reduce the number of nodes in JSON file
@@ -144,6 +135,16 @@ if [ ${USE_DHCP} = "true" ]; then
         && mv ${BAREMETAL_DATA_FILE}.new ${BAREMETAL_DATA_FILE}
 fi
 
+if [ ${CLI_TEST} = "true" ]; then
+    # FIXME(dtantsur): bifrost-cli does not use opendev-provided repos.
+    ../bifrost-cli --debug install --release ${ZUUL_BRANCH:-master} --testenv
+    CLOUD_CONFIG+=" -e skip_install=true"
+    CLOUD_CONFIG+=" -e skip_package_install=true"
+    CLOUD_CONFIG+=" -e skip_bootstrap=true"
+    CLOUD_CONFIG+=" -e skip_start=true"
+    CLOUD_CONFIG+=" -e skip_migrations=true"
+fi
+
 set +e
 
 # Set BIFROST_INVENTORY_SOURCE

zuul.d/bifrost-jobs.yaml

@@ -99,6 +99,12 @@
     vars:
       enable_keystone: true
 
+- job:
+    name: bifrost-cli-ubuntu-bionic
+    parent: bifrost-integration-tinyipa-ubuntu-bionic
+    vars:
+      cli_test: true
+
 - job:
     name: bifrost-integration-tinyipa-ubuntu-focal
     parent: bifrost-integration-tinyipa

zuul.d/project.yaml

@@ -34,6 +34,8 @@
            voting: false
         - bifrost-integration-dhcp-debian-buster:
            voting: false
+        - bifrost-cli-ubuntu-bionic:
+           voting: false
     gate:
       jobs:
         - bifrost-integration-tinyipa-ubuntu-bionic