diff --git a/.gitignore b/.gitignore index a2f5dbe67..cafdacd5a 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,7 @@ playbooks/library/os_keystone_service.py # Other *.DS_Store .idea + +# Generated by bifrost-cli +baremetal-inventory.json +baremetal-nodes.json diff --git a/bifrost-cli b/bifrost-cli new file mode 100755 index 000000000..95efe299d --- /dev/null +++ b/bifrost-cli @@ -0,0 +1,13 @@ +#!/bin/sh + +if ! python3 --version > /dev/null; then + echo "Python 3 not found, version 3.6 or newer is required for Bifrost" + exit 1 +fi + +if ! python3 -c "import sys; assert sys.version_info >= (3, 6)" 2> /dev/null; then + echo "Python 3.6 or newer is required for Bifrost" + exit 1 +fi + +PYTHONPATH=$(dirname $0) exec python3 -m bifrost.cli $@ diff --git a/bifrost/__init__.py b/bifrost/__init__.py index 7621e0dbf..3a68a217c 100644 --- a/bifrost/__init__.py +++ b/bifrost/__init__.py @@ -12,11 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -import pbr.version +try: + import pbr.version - -__version__ = pbr.version.VersionInfo( - 'bifrost').version_string() + __version__ = pbr.version.VersionInfo( + 'bifrost').version_string() +except ImportError: + pass # Allow the CLI to work without pbr installed __all__ = [ 'inventory' diff --git a/bifrost/cli.py b/bifrost/cli.py new file mode 100644 index 000000000..3d0e91667 --- /dev/null +++ b/bifrost/cli.py @@ -0,0 +1,234 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import configparser +import ipaddress +import itertools +import os +import subprocess +import sys + + +VENV = "/opt/stack/bifrost" +ANSIBLE = os.path.join(VENV, 'bin', 'ansible-playbook') +COMMON_ENV = { + 'VENV': VENV, + 'USE_VENV': 'true', + 'ENABLE_VENV': 'true', +} +COMMON_PARAMS = [ + '-e', 'ansible_python_interpreter=%s/bin/python3' % VENV, + '-e', 'enable_venv=true', + '-e', 'bifrost_venv_dir=%s' % VENV, +] +BASE = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +PLAYBOOKS = os.path.join(BASE, 'playbooks') + + +def get_env(extra=None): + # NOTE(dtantsur): the order here matters! + result = os.environ.copy() + result.update(COMMON_ENV) + if extra: + result.update(extra) + return result + + +def log(*message, only_if=True): + if only_if: + print(*message, file=sys.stderr) + + +def ansible(playbook, inventory, verbose=False, env=None, **params): + extra = COMMON_PARAMS + list(itertools.chain.from_iterable( + ('-e', '%s=%s' % pair) for pair in params.items() + if pair[1] is not None)) + if verbose: + extra.append('-vvvv') + args = [ANSIBLE, playbook, '-i', inventory] + extra + log('Calling ansible with', args, 'and environment', env, only_if=verbose) + subprocess.check_call(args, env=get_env(env), cwd=PLAYBOOKS) + + +def env_setup(args): + if os.path.exists(VENV): + log(VENV, 'exists, skipping environment preparation', + only_if=args.debug) + return + + log('Installing dependencies and preparing an environment in', VENV) + subprocess.check_call(["bash", "scripts/env-setup.sh"], + env=get_env(), cwd=BASE) + + +def get_release(release): + if release: + if release != 'master' and not release.startswith('stable/'): + release = 'stable/%s' % release + return release + else: + try: + gr = configparser.ConfigParser() + gr.read(os.path.join(BASE, '.gitreview')) + release = gr.get('gerrit', 'defaultbranch', fallback='master') + log('Using release', release, 'detected from the checkout') + return release + except (FileNotFoundError, configparser.Error): + log('Cannot read .gitreview, falling back to release "master"') + return 'master' + + +def cmd_testenv(args): + release = get_release(args.release) + + env_setup(args) + log('Creating', args.count, 'test node(s) with', args.memory, + 'MiB RAM and', args.disk, 'GiB of disk') + + kwargs = {} + if args.storage_pool_path: + kwargs['test_vm_storage_pool_path'] = os.path.abspath( + args.storage_pool_path) + + ansible('test-bifrost-create-vm.yaml', + inventory='inventory/localhost', + verbose=args.debug, + git_branch=release, + test_vm_num_nodes=args.count, + test_vm_memory_size=args.memory, + test_vm_disk_gib=args.disk, + test_vm_domain_type=args.domain_type, + baremetal_json_file=os.path.abspath(args.inventory), + baremetal_nodes_json=os.path.abspath(args.output), + **kwargs) + log('Inventory generated in', args.output) + + +def cmd_install(args): + release = get_release(args.release) + + kwargs = {} + if args.dhcp_pool: + try: + start, end = args.dhcp_pool.split('-') + ipaddress.ip_address(start) + ipaddress.ip_address(end) + except ValueError as e: + sys.exit("Malformed --dhcp-pool, expected two IP addresses. " + "Error: %s" % e) + else: + kwargs['dhcp_pool_start'] = start + kwargs['dhcp_pool_end'] = end + + env_setup(args) + ansible('install.yaml', + inventory='inventory/target', + verbose=args.debug, + git_branch=release, + ipa_upstream_release=release.replace('/', '-'), + create_ipa_image='false', + create_image_via_dib='false', + install_dib='true', + network_interface=args.network_interface, + enable_keystone=args.enable_keystone, + use_public_urls=args.enable_keystone, + noauth_mode=not args.enable_keystone, + testing=args.testenv, + use_cirros=args.testenv, + use_tinyipa=args.testenv, + **kwargs) + log("Ironic is installed and running, try it yourself:\n", + " $ source %s/bin/activate\n" % VENV, + " $ export OS_CLOUD=bifrost\n", + " $ baremetal driver list\n" + "See documentation for next steps") + + +def parse_args(): + parser = argparse.ArgumentParser("Bifrost CLI") + parser.add_argument('--debug', action='store_true', + help='output extensive logging') + + subparsers = parser.add_subparsers() + + testenv = subparsers.add_parser( + 'testenv', help='Prepare a virtual testing environment') + testenv.set_defaults(func=cmd_testenv) + testenv.add_argument('--release', default='master', + help='release branch to use (master, ussuri, etc)') + testenv.add_argument('--count', type=int, default=2, + help='number of nodes to create') + testenv.add_argument('--memory', type=int, default=3072, + help='memory (in MiB) for test nodes') + testenv.add_argument('--disk', type=int, default=10, + help='disk size (in GiB) for test nodes') + testenv.add_argument('--domain-type', default='qemu', + help='domain type: qemu or kvm') + testenv.add_argument('--storage-pool-path', + help='path to libvirt storage pool to setup') + testenv.add_argument('--inventory', default='baremetal-inventory.json', + help='output file with the inventory for using ' + 'with dynamic playbooks') + testenv.add_argument('-o', '--output', default='baremetal-nodes.json', + help='output file with the nodes information for ' + 'importing into ironic') + + install = subparsers.add_parser('install', help='Install ironic') + install.set_defaults(func=cmd_install) + install.add_argument('--testenv', action='store_true', + help='running in a virtual environment') + install.add_argument('--dhcp-pool', metavar='START-END', + help='DHCP pool to use') + install.add_argument('--release', + help='release branch to use (master, ussuri, etc), ' + 'the default value is determined from the ' + '.gitreview file in the source tree') + install.add_argument('--network-interface', + help='the network interface to use') + install.add_argument('--enable-keystone', action='store_true', + help='enable keystone and use authentication') + + args = parser.parse_args() + if getattr(args, 'func', None) is None: + parser.print_usage(file=sys.stderr) + sys.exit("Bifrost CLI: error: a command is required") + return args + + +def check_for_root(): + try: + subprocess.check_call( + '[ $(whoami) == root ] || sudo --non-interactive true', + shell=True, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + # TODO(dtantsur): tell ansible to ask for password + sys.exit('Sudo without password is required for Bifrost') + + +def main(): + args = parse_args() + try: + check_for_root() + args.func(args) + except Exception as exc: + if args.debug: + raise + else: + sys.exit(str(exc)) + except KeyboardInterrupt: + sys.exit('Aborting by user request') + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/playbooks/ci/run.yaml b/playbooks/ci/run.yaml index 0ae150373..e1ec411a7 100644 --- a/playbooks/ci/run.yaml +++ b/playbooks/ci/run.yaml @@ -5,6 +5,7 @@ chdir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/bifrost'].src_dir }}" environment: BUILD_IMAGE: "{{ build_image | default(false) | bool | lower }}" + CLI_TEST: "{{ cli_test | default(false) | bool | lower }}" ENABLE_KEYSTONE: "{{ enable_keystone | default(false) | bool | lower }}" LOG_LOCATION: "{{ ansible_user_dir }}/logs" UPPER_CONSTRAINTS_FILE: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/requirements'].src_dir }}/upper-constraints.txt" diff --git a/scripts/test-bifrost.sh b/scripts/test-bifrost.sh index 14a1ee51e..4bd611321 100755 --- a/scripts/test-bifrost.sh +++ b/scripts/test-bifrost.sh @@ -11,7 +11,8 @@ BUILD_IMAGE="${BUILD_IMAGE:-false}" BAREMETAL_DATA_FILE=${BAREMETAL_DATA_FILE:-'/tmp/baremetal.json'} ENABLE_KEYSTONE="${ENABLE_KEYSTONE:-false}" ZUUL_BRANCH=${ZUUL_BRANCH:-} -ENABLE_VENV=${ENABLE_VENV:-true} +ENABLE_VENV=true +CLI_TEST=${CLI_TEST:-false} # Set defaults for ansible command-line options to drive the different # tests. @@ -50,22 +51,18 @@ OS_DISTRO="$ID" # Setup openstack_ci test database if run in OpenStack CI. if [ "$ZUUL_BRANCH" != "" ]; then sudo mkdir -p /opt/libvirt/images - VM_SETUP_EXTRA="-e test_vm_storage_pool_path=/opt/libvirt/images" + VM_SETUP_EXTRA="--storage-pool-path /opt/libvirt/images" fi source $SCRIPT_HOME/env-setup.sh -if [ ${ENABLE_VENV} = "true" ]; then - # Note(cinerama): activate is not compatible with "set -u"; - # disable it just for this line. - set +u - source ${VENV}/bin/activate - set -u - ANSIBLE=${VENV}/bin/ansible-playbook - ANSIBLE_PYTHON_INTERP=${VENV}/bin/python3 -else - ANSIBLE=${HOME}/.local/bin/ansible-playbook - ANSIBLE_PYTHON_INTERP=$(which python3) -fi + +# Note(cinerama): activate is not compatible with "set -u"; +# disable it just for this line. +set +u +source ${VENV}/bin/activate +set -u +ANSIBLE=${VENV}/bin/ansible-playbook +ANSIBLE_PYTHON_INTERP=${VENV}/bin/python3 # Adjust options for DHCP, VM, or Keystone tests if [ ${USE_DHCP} = "true" ]; then @@ -120,19 +117,13 @@ for task in syntax-check list-tasks; do -e testing_user=${TESTING_USER} done -# Create the test VM -${ANSIBLE} -vvvv \ - -i inventory/localhost \ - test-bifrost-create-vm.yaml \ - -e ansible_python_interpreter="${ANSIBLE_PYTHON_INTERP}" \ - -e test_vm_num_nodes=${TEST_VM_NUM_NODES} \ - -e test_vm_memory_size=${VM_MEMORY_SIZE:-512} \ - -e test_vm_domain_type=${VM_DOMAIN_TYPE} \ - -e test_vm_disk_gib=${VM_DISK:-5} \ - -e baremetal_json_file=${BAREMETAL_DATA_FILE} \ - -e enable_venv=${ENABLE_VENV} \ - -e bifrost_venv_dir=${VENV} \ - ${VM_SETUP_EXTRA:-} +# Create the test VMs +../bifrost-cli --debug testenv \ + --count ${TEST_VM_NUM_NODES} \ + --memory ${VM_MEMORY_SIZE:-512} \ + --disk ${VM_DISK:-5} \ + --inventory "${BAREMETAL_DATA_FILE}" \ + ${VM_SETUP_EXTRA:-} if [ ${USE_DHCP} = "true" ]; then # reduce the number of nodes in JSON file @@ -144,6 +135,16 @@ if [ ${USE_DHCP} = "true" ]; then && mv ${BAREMETAL_DATA_FILE}.new ${BAREMETAL_DATA_FILE} fi +if [ ${CLI_TEST} = "true" ]; then + # FIXME(dtantsur): bifrost-cli does not use opendev-provided repos. + ../bifrost-cli --debug install --release ${ZUUL_BRANCH:-master} --testenv + CLOUD_CONFIG+=" -e skip_install=true" + CLOUD_CONFIG+=" -e skip_package_install=true" + CLOUD_CONFIG+=" -e skip_bootstrap=true" + CLOUD_CONFIG+=" -e skip_start=true" + CLOUD_CONFIG+=" -e skip_migrations=true" +fi + set +e # Set BIFROST_INVENTORY_SOURCE diff --git a/zuul.d/bifrost-jobs.yaml b/zuul.d/bifrost-jobs.yaml index 1f947658d..89bd88875 100644 --- a/zuul.d/bifrost-jobs.yaml +++ b/zuul.d/bifrost-jobs.yaml @@ -99,6 +99,12 @@ vars: enable_keystone: true +- job: + name: bifrost-cli-ubuntu-bionic + parent: bifrost-integration-tinyipa-ubuntu-bionic + vars: + cli_test: true + - job: name: bifrost-integration-tinyipa-ubuntu-focal parent: bifrost-integration-tinyipa diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 1127fd1c6..026cabc3f 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -34,6 +34,8 @@ voting: false - bifrost-integration-dhcp-debian-buster: voting: false + - bifrost-cli-ubuntu-bionic: + voting: false gate: jobs: - bifrost-integration-tinyipa-ubuntu-bionic