From bfda49dd38e14c57c9a97dec06a0969991a2b05a Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 1 Feb 2022 20:44:56 +0000 Subject: [PATCH] Update to build using charmcraft Due to a build problem with the reactive plugin, this change falls back on overriding the steps and doing a manual build, but it also ensures the CI system builds the charm using charmcraft. Changes: - add a build-requirements.txt - modify charmcraft.yaml - modify osci.yaml -> indicate build with charmcraft - modify tox.ini -> tox -e build does charmcraft build/rename -> tox -e build-reactive does the reactive build - modify bundles to use the .charm artifact in tests. and fix deprecation warning re: prefix - tox inception to enable tox -e func-test in the CI Change-Id: Iafe802bd1b01c0447dd45ed697c0ea3d003218e5 --- build-requirements.txt | 7 +++++++ charmcraft.yaml | 18 +++++++++++++++--- osci.yaml | 1 + rename.sh | 13 +++++++++++++ src/test-requirements.txt | 3 +++ src/tests/bundles/focal-xena.yaml | 30 +++++++++++++++--------------- src/tests/bundles/focal-yoga.yaml | 30 +++++++++++++++--------------- src/tests/bundles/impish-xena.yaml | 30 +++++++++++++++--------------- src/tests/bundles/jammy-yoga.yaml | 30 +++++++++++++++--------------- src/tests/tests.yaml | 4 ++-- test-requirements.txt | 1 + tox.ini | 25 +++++++++++++++++++++++++ 12 files changed, 127 insertions(+), 65 deletions(-) create mode 100644 build-requirements.txt create mode 100755 rename.sh diff --git a/build-requirements.txt b/build-requirements.txt new file mode 100644 index 0000000..b6d2452 --- /dev/null +++ b/build-requirements.txt @@ -0,0 +1,7 @@ +# NOTES(lourot): +# * We don't install charmcraft via pip anymore because it anyway spins up a +# container and scp the system's charmcraft snap inside it. So the charmcraft +# snap is necessary on the system anyway. +# * `tox -e build` successfully validated with charmcraft 1.2.1 + +cffi==1.14.6; python_version < '3.6' # cffi 1.15.0 drops support for py35. diff --git a/charmcraft.yaml b/charmcraft.yaml index a850351..4968216 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -2,9 +2,21 @@ type: charm parts: charm: - source: src/ - plugin: reactive - build-snaps: [charm] + build-packages: + - tox + - git + - python3-dev + override-build: | + apt-get install ca-certificates -y + tox -e build-reactive + override-stage: | + echo "Copying charm to staging area: $CHARMCRAFT_STAGE" + NAME=$(ls $CHARMCRAFT_PART_BUILD/build/builds) + cp -r $CHARMCRAFT_PART_BUILD/build/builds/$NAME/* $CHARMCRAFT_STAGE/ + override-prime: | + # For some reason, the normal priming chokes on the fact that there's a + # hooks directory. + cp -r $CHARMCRAFT_STAGE/* . bases: - name: ubuntu diff --git a/osci.yaml b/osci.yaml index ff03fd5..a0cd125 100644 --- a/osci.yaml +++ b/osci.yaml @@ -7,3 +7,4 @@ vars: needs_charm_build: true charm_build_name: neutron-api-plugin-ironic + build_type: charmcraft diff --git a/rename.sh b/rename.sh new file mode 100755 index 0000000..d0c35c9 --- /dev/null +++ b/rename.sh @@ -0,0 +1,13 @@ +#!/bin/bash +charm=$(grep "charm_build_name" osci.yaml | awk '{print $2}') +echo "renaming ${charm}_*.charm to ${charm}.charm" +echo -n "pwd: " +pwd +ls -al +echo "Removing bad downloaded charm maybe?" +if [[ -e "${charm}.charm" ]]; +then + rm "${charm}.charm" +fi +echo "Renaming charm here." +mv ${charm}_*.charm ${charm}.charm diff --git a/src/test-requirements.txt b/src/test-requirements.txt index e771023..9c7afb7 100644 --- a/src/test-requirements.txt +++ b/src/test-requirements.txt @@ -4,6 +4,9 @@ # https://github.com/openstack-charmers/release-tools # +# Need tox to be available from tox... inception yes, but its a workaround for now +tox + # Functional Test Requirements (let Zaza's dependencies solve all dependencies here!) git+https://github.com/openstack-charmers/zaza.git#egg=zaza git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack diff --git a/src/tests/bundles/focal-xena.yaml b/src/tests/bundles/focal-xena.yaml index dc5646b..44f8739 100644 --- a/src/tests/bundles/focal-xena.yaml +++ b/src/tests/bundles/focal-xena.yaml @@ -177,7 +177,7 @@ services: worker-multiplier: 0.25 storage: block-devices: cinder,50G - channel: latest/edge + channel: yoga/edge ceph-radosgw: charm: ch:ceph-radosgw num_units: 1 @@ -185,7 +185,7 @@ services: options: source: *source namespace-tenants: True - channel: latest/edge + channel: quincy/edge ceph-mon: charm: ch:ceph-mon num_units: 3 @@ -193,7 +193,7 @@ services: options: expected-osd-count: 3 source: *source - channel: latest/edge + channel: quincy/edge ceph-osd: charm: ch:ceph-osd num_units: 3 @@ -202,7 +202,7 @@ services: source: *source storage: osd-devices: 'cinder,30G' - channel: latest/edge + channel: quincy/edge glance: charm: ch:glance num_units: 1 @@ -210,7 +210,7 @@ services: options: openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge keystone: charm: ch:keystone num_units: 1 @@ -218,7 +218,7 @@ services: options: openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge neutron-api: charm: ch:neutron-api num_units: 1 @@ -229,7 +229,7 @@ services: openstack-origin: *source manage-neutron-plugin-legacy-mode: false worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge neutron-gateway: charm: ch:neutron-gateway num_units: 1 @@ -239,7 +239,7 @@ services: enable-isolated-metadata: true enable-metadata-network: true bridge-mappings: physnet1:br-ex - channel: latest/edge + channel: yoga/edge nova-cloud-controller: charm: ch:nova-cloud-controller num_units: 1 @@ -248,7 +248,7 @@ services: network-manager: Neutron openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge nova-ironic: charm: ch:nova-compute num_units: 1 @@ -258,7 +258,7 @@ services: enable-resize: false openstack-origin: *source virt-type: ironic - channel: latest/edge + channel: yoga/edge placement: charm: ch:placement num_units: 1 @@ -266,7 +266,7 @@ services: options: openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge rabbitmq-server: charm: ch:rabbitmq-server num_units: 1 @@ -282,7 +282,7 @@ services: constraints: mem=2G options: openstack-origin: *source - channel: latest/edge + channel: yoga/edge ironic-conductor: charm: ch:ironic-conductor num_units: 1 @@ -293,15 +293,15 @@ services: disable-secure-erase: true use-ipxe: true enabled-network-interfaces: "flat, noop" - channel: latest/edge + channel: yoga/edge neutron-openvswitch: charm: ch:neutron-openvswitch num_units: 0 options: bridge-mappings: physnet1:br-ex - channel: latest/edge + channel: yoga/edge neutron-ironic-agent: - charm: ../../../neutron-api-plugin-ironic + charm: ../../../neutron-api-plugin-ironic.charm num_units: 0 options: openstack-origin: *source diff --git a/src/tests/bundles/focal-yoga.yaml b/src/tests/bundles/focal-yoga.yaml index 8a0b354..865ef6f 100644 --- a/src/tests/bundles/focal-yoga.yaml +++ b/src/tests/bundles/focal-yoga.yaml @@ -177,7 +177,7 @@ services: worker-multiplier: 0.25 storage: block-devices: cinder,50G - channel: latest/edge + channel: yoga/edge ceph-radosgw: charm: ch:ceph-radosgw num_units: 1 @@ -185,7 +185,7 @@ services: options: source: *source namespace-tenants: True - channel: latest/edge + channel: quincy/edge ceph-mon: charm: ch:ceph-mon num_units: 3 @@ -193,7 +193,7 @@ services: options: expected-osd-count: 3 source: *source - channel: latest/edge + channel: quincy/edge ceph-osd: charm: ch:ceph-osd num_units: 3 @@ -202,7 +202,7 @@ services: source: *source storage: osd-devices: 'cinder,30G' - channel: latest/edge + channel: quincy/edge glance: charm: ch:glance num_units: 1 @@ -210,7 +210,7 @@ services: options: openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge keystone: charm: ch:keystone num_units: 1 @@ -218,7 +218,7 @@ services: options: openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge neutron-api: charm: ch:neutron-api num_units: 1 @@ -229,7 +229,7 @@ services: openstack-origin: *source manage-neutron-plugin-legacy-mode: false worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge neutron-gateway: charm: ch:neutron-gateway num_units: 1 @@ -239,7 +239,7 @@ services: enable-isolated-metadata: true enable-metadata-network: true bridge-mappings: physnet1:br-ex - channel: latest/edge + channel: yoga/edge nova-cloud-controller: charm: ch:nova-cloud-controller num_units: 1 @@ -248,7 +248,7 @@ services: network-manager: Neutron openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge nova-ironic: charm: ch:nova-compute num_units: 1 @@ -258,7 +258,7 @@ services: enable-resize: false openstack-origin: *source virt-type: ironic - channel: latest/edge + channel: yoga/edge placement: charm: ch:placement num_units: 1 @@ -266,7 +266,7 @@ services: options: openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge rabbitmq-server: charm: ch:rabbitmq-server num_units: 1 @@ -282,7 +282,7 @@ services: constraints: mem=2G options: openstack-origin: *source - channel: latest/edge + channel: yoga/edge ironic-conductor: charm: ch:ironic-conductor num_units: 1 @@ -293,15 +293,15 @@ services: disable-secure-erase: true use-ipxe: true enabled-network-interfaces: "flat, noop" - channel: latest/edge + channel: yoga/edge neutron-openvswitch: charm: ch:neutron-openvswitch num_units: 0 options: bridge-mappings: physnet1:br-ex - channel: latest/edge + channel: yoga/edge neutron-ironic-agent: - charm: ../../../neutron-api-plugin-ironic + charm: ../../../neutron-api-plugin-ironic.charm num_units: 0 options: openstack-origin: *source diff --git a/src/tests/bundles/impish-xena.yaml b/src/tests/bundles/impish-xena.yaml index 496459a..78e3992 100644 --- a/src/tests/bundles/impish-xena.yaml +++ b/src/tests/bundles/impish-xena.yaml @@ -177,7 +177,7 @@ services: worker-multiplier: 0.25 storage: block-devices: cinder,50G - channel: latest/edge + channel: yoga/edge ceph-radosgw: charm: ch:ceph-radosgw num_units: 1 @@ -185,7 +185,7 @@ services: options: source: *source namespace-tenants: True - channel: latest/edge + channel: quincy/edge ceph-mon: charm: ch:ceph-mon num_units: 3 @@ -193,7 +193,7 @@ services: options: expected-osd-count: 3 source: *source - channel: latest/edge + channel: quincy/edge ceph-osd: charm: ch:ceph-osd num_units: 3 @@ -202,7 +202,7 @@ services: source: *source storage: osd-devices: 'cinder,30G' - channel: latest/edge + channel: quincy/edge glance: charm: ch:glance num_units: 1 @@ -210,7 +210,7 @@ services: options: openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge keystone: charm: ch:keystone num_units: 1 @@ -218,7 +218,7 @@ services: options: openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge neutron-api: charm: ch:neutron-api num_units: 1 @@ -229,7 +229,7 @@ services: openstack-origin: *source manage-neutron-plugin-legacy-mode: false worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge neutron-gateway: charm: ch:neutron-gateway num_units: 1 @@ -239,7 +239,7 @@ services: enable-isolated-metadata: true enable-metadata-network: true bridge-mappings: physnet1:br-ex - channel: latest/edge + channel: yoga/edge nova-cloud-controller: charm: ch:nova-cloud-controller num_units: 1 @@ -248,7 +248,7 @@ services: network-manager: Neutron openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge nova-ironic: charm: ch:nova-compute num_units: 1 @@ -258,7 +258,7 @@ services: enable-resize: false openstack-origin: *source virt-type: ironic - channel: latest/edge + channel: yoga/edge placement: charm: ch:placement num_units: 1 @@ -266,7 +266,7 @@ services: options: openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge rabbitmq-server: charm: ch:rabbitmq-server num_units: 1 @@ -282,7 +282,7 @@ services: constraints: mem=2G options: openstack-origin: *source - channel: latest/edge + channel: yoga/edge ironic-conductor: charm: ch:ironic-conductor num_units: 1 @@ -293,15 +293,15 @@ services: disable-secure-erase: true use-ipxe: true enabled-network-interfaces: "flat, noop" - channel: latest/edge + channel: yoga/edge neutron-openvswitch: charm: ch:neutron-openvswitch num_units: 0 options: bridge-mappings: physnet1:br-ex - channel: latest/edge + channel: yoga/edge neutron-ironic-agent: - charm: ../../../neutron-api-plugin-ironic + charm: ../../../neutron-api-plugin-ironic.charm num_units: 0 options: openstack-origin: *source diff --git a/src/tests/bundles/jammy-yoga.yaml b/src/tests/bundles/jammy-yoga.yaml index 26da530..34fb453 100644 --- a/src/tests/bundles/jammy-yoga.yaml +++ b/src/tests/bundles/jammy-yoga.yaml @@ -177,7 +177,7 @@ services: worker-multiplier: 0.25 storage: block-devices: cinder,50G - channel: latest/edge + channel: yoga/edge ceph-radosgw: charm: ch:ceph-radosgw num_units: 1 @@ -185,7 +185,7 @@ services: options: source: *source namespace-tenants: True - channel: latest/edge + channel: quincy/edge ceph-mon: charm: ch:ceph-mon num_units: 3 @@ -193,7 +193,7 @@ services: options: expected-osd-count: 3 source: *source - channel: latest/edge + channel: quincy/edge ceph-osd: charm: ch:ceph-osd num_units: 3 @@ -202,7 +202,7 @@ services: source: *source storage: osd-devices: 'cinder,30G' - channel: latest/edge + channel: quincy/edge glance: charm: ch:glance num_units: 1 @@ -210,7 +210,7 @@ services: options: openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge keystone: charm: ch:keystone num_units: 1 @@ -218,7 +218,7 @@ services: options: openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge neutron-api: charm: ch:neutron-api num_units: 1 @@ -229,7 +229,7 @@ services: openstack-origin: *source manage-neutron-plugin-legacy-mode: false worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge neutron-gateway: charm: ch:neutron-gateway num_units: 1 @@ -239,7 +239,7 @@ services: enable-isolated-metadata: true enable-metadata-network: true bridge-mappings: physnet1:br-ex - channel: latest/edge + channel: yoga/edge nova-cloud-controller: charm: ch:nova-cloud-controller num_units: 1 @@ -248,7 +248,7 @@ services: network-manager: Neutron openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge nova-ironic: charm: ch:nova-compute num_units: 1 @@ -258,7 +258,7 @@ services: enable-resize: false openstack-origin: *source virt-type: ironic - channel: latest/edge + channel: yoga/edge placement: charm: ch:placement num_units: 1 @@ -266,7 +266,7 @@ services: options: openstack-origin: *source worker-multiplier: 0.25 - channel: latest/edge + channel: yoga/edge rabbitmq-server: charm: ch:rabbitmq-server num_units: 1 @@ -282,7 +282,7 @@ services: constraints: mem=2G options: openstack-origin: *source - channel: latest/edge + channel: yoga/edge ironic-conductor: charm: ch:ironic-conductor num_units: 1 @@ -293,15 +293,15 @@ services: disable-secure-erase: true use-ipxe: true enabled-network-interfaces: "flat, noop" - channel: latest/edge + channel: yoga/edge neutron-openvswitch: charm: ch:neutron-openvswitch num_units: 0 options: bridge-mappings: physnet1:br-ex - channel: latest/edge + channel: yoga/edge neutron-ironic-agent: - charm: ../../../neutron-api-plugin-ironic + charm: ../../../neutron-api-plugin-ironic.charm num_units: 0 options: openstack-origin: *source diff --git a/src/tests/tests.yaml b/src/tests/tests.yaml index 82b4106..b955eda 100644 --- a/src/tests/tests.yaml +++ b/src/tests/tests.yaml @@ -17,10 +17,10 @@ smoke_bundles: target_deploy_status: vault: workload-status: blocked - workload-status-message: Vault needs to be initialized + workload-status-message-prefix: Vault needs to be initialized ironic-conductor: workload-status: blocked - workload-status-message: invalid enabled-deploy-interfaces config + workload-status-message-prefix: invalid enabled-deploy-interfaces config neutron-openvswitch: # NOTE(lourot): we deploy a bundle in which this subordinate charm has no # principle. diff --git a/test-requirements.txt b/test-requirements.txt index bb1307f..a11a7d0 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -51,3 +51,4 @@ pbr==5.6.0 # vault cryptography<3.4 # vault, keystone-saml-mellon lxml # keystone-saml-mellon hvac # vault, barbican-vault +psutil # cinder-lvm diff --git a/tox.ini b/tox.ini index faf6092..2d60b8a 100644 --- a/tox.ini +++ b/tox.ini @@ -37,11 +37,24 @@ setenv = VIRTUAL_ENV={envdir} passenv = http_proxy https_proxy INTERFACE_PATH LAYER_PATH JUJU_REPOSITORY install_command = {toxinidir}/pip.sh install {opts} {packages} +allowlist_externals = + charmcraft + bash + tox + rename.sh deps = -r{toxinidir}/requirements.txt [testenv:build] basepython = python3 +deps = -r{toxinidir}/build-requirements.txt +commands = + charmcraft clean + charmcraft -v build + {toxinidir}/rename.sh + +[testenv:build-reactive] +basepython = python3 commands = charm-build --log-level DEBUG --use-lock-file-branches -o {toxinidir}/build/builds src {posargs} @@ -86,6 +99,18 @@ deps = flake8==3.9.2 charm-tools==2.8.3 commands = flake8 {posargs} src unit_tests +[testenv:func-target] +# Hack to get functional tests working in the charmcraft +# world. We should fix this. +basepython = python3 +passenv = HOME TERM CS_* OS_* TEST_* +deps = -r{toxinidir}/src/test-requirements.txt +changedir = {toxinidir}/src +commands = + bash -c "if [ ! -f ../*.charm ]; then echo 'Charm does not exist. Run tox -e build'; exit 1; fi" + tox --version + tox -e func-target {posargs} + [testenv:cover] # Technique based heavily upon # https://github.com/openstack/nova/blob/master/tox.ini