From 10524b6ed3188e5a48060c6772c8425dcc8cbe6e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 21 Sep 2022 15:44:54 +0000 Subject: [PATCH] Build separately for each supported series and use binary builds Charms for OpenStack Yoga supports both Ubuntu Focal and Jammy which means Python 3.8 and Python 3.10. Managing dependencies across those two versions is non-trivial and we need to build the charm on the series the charm is supposed to support. Switch to using a binary build which allows pip's dependency resolution to work. This patch also updates manila.py to align with charm-interface-keystone commit 9ad5cade97e14b672ac859d34040353c3a40c7aa. This patch also drops the impish bundle. Change-Id: Ib3aea3fb26aa3e6282989b2758bdf4913ef58c28 --- bindep.txt | 4 + charmcraft.yaml | 56 ++- osci.yaml | 10 +- src/lib/charm/openstack/manila.py | 4 +- src/tests/bundles/focal-xena.yaml | 2 +- src/tests/bundles/focal-yoga.yaml | 2 +- src/tests/bundles/impish-xena.yaml | 342 ------------------ src/tests/bundles/jammy-yoga.yaml | 2 +- src/tests/tests.yaml | 4 - tox.ini | 20 +- unit_tests/test_lib_charm_openstack_manila.py | 25 +- 11 files changed, 66 insertions(+), 405 deletions(-) create mode 100644 bindep.txt delete mode 100644 src/tests/bundles/impish-xena.yaml diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000..17575d9 --- /dev/null +++ b/bindep.txt @@ -0,0 +1,4 @@ +libffi-dev [platform:dpkg] +libpq-dev [platform:dpkg] +libxml2-dev [platform:dpkg] +libxslt1-dev [platform:dpkg] diff --git a/charmcraft.yaml b/charmcraft.yaml index c9a0ed7..194893f 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -6,28 +6,52 @@ parts: - tox - git - python3-dev + - libffi-dev + - libssl-dev + - rustc + - cargo + build-environment: + - CHARM_INTERFACES_DIR: /root/project/interfaces/ + - CHARM_LAYERS_DIR: /root/project/layers/ + - MAKEFLAGS: -j$(nproc) override-build: | apt-get install ca-certificates -y tox -e build-reactive override-stage: | - echo "Copying charm to staging area: $CHARMCRAFT_STAGE" - NAME=$(ls $CHARMCRAFT_PART_BUILD/build/builds) - cp -r $CHARMCRAFT_PART_BUILD/build/builds/$NAME/* $CHARMCRAFT_STAGE/ + echo "Copying charm to staging area: $CRAFT_STAGE" + NAME=$(ls $CRAFT_PART_BUILD/build/builds) + cp -r $CRAFT_PART_BUILD/build/builds/$NAME/* $CRAFT_STAGE/ override-prime: | # For some reason, the normal priming chokes on the fact that there's a # hooks directory. - cp -r $CHARMCRAFT_STAGE/* . + cp -r $CRAFT_STAGE/* . + # Charmcraft looks for this specific entry point. + mkdir -p src + touch src/charm.py + chmod +x src/charm.py bases: - - build-on: - - name: ubuntu - channel: "20.04" - architectures: - - amd64 - run-on: - - name: ubuntu - channel: "20.04" - architectures: [amd64, s390x, ppc64el, arm64] - - name: ubuntu - channel: "22.04" - architectures: [amd64, s390x, ppc64el, arm64] + - name: ubuntu + channel: "20.04" + architectures: [amd64] + - name: ubuntu + channel: "20.04" + architectures: [arm64] + - name: ubuntu + channel: "20.04" + architectures: [ppc64el] + - name: ubuntu + channel: "20.04" + architectures: [s390x] + - name: ubuntu + channel: "22.04" + architectures: [amd64] + - name: ubuntu + channel: "22.04" + architectures: [arm64] + - name: ubuntu + channel: "22.04" + architectures: [ppc64el] + - name: ubuntu + channel: "22.04" + architectures: [s390x] diff --git a/osci.yaml b/osci.yaml index acd993d..75e81d3 100644 --- a/osci.yaml +++ b/osci.yaml @@ -11,14 +11,13 @@ - ganesha-focal-xena - ganesha-focal-yoga: voting: false - - ganesha-impish-xena: - voting: false - ganesha-jammy-yoga: voting: false vars: needs_charm_build: true charm_build_name: manila build_type: charmcraft + charmcraft_channel: 2.0/stable - job: name: ganesha-focal-xena @@ -37,13 +36,6 @@ - ganesha-focal-xena vars: tox_extra_args: ganesha:focal-yoga -- job: - name: ganesha-impish-xena - parent: func-target - dependencies: - - ganesha-focal-xena - vars: - tox_extra_args: ganesha:impish-xena - job: name: ganesha-jammy-yoga parent: func-target diff --git a/src/lib/charm/openstack/manila.py b/src/lib/charm/openstack/manila.py index 1191671..cdab48a 100644 --- a/src/lib/charm/openstack/manila.py +++ b/src/lib/charm/openstack/manila.py @@ -350,8 +350,8 @@ class ManilaCharm(charms_openstack.charm.HAOpenStackCharm): '{}_admin_url'.format(prefix): admin_url, '{}_region'.format(prefix): region, } - keystone.set_local(**relation_info) - keystone.set_remote(**relation_info) + for relation in keystone.relations: + relation.to_publish_raw.update(relation_info) @property def public_url(self): diff --git a/src/tests/bundles/focal-xena.yaml b/src/tests/bundles/focal-xena.yaml index a114aec..9bfc71f 100644 --- a/src/tests/bundles/focal-xena.yaml +++ b/src/tests/bundles/focal-xena.yaml @@ -118,7 +118,7 @@ services: channel: latest/edge manila: - charm: ../../../manila.charm + charm: ../../../manila_ubuntu-20.04-amd64.charm num_units: 3 options: default-share-backend: cephfsnfs1 diff --git a/src/tests/bundles/focal-yoga.yaml b/src/tests/bundles/focal-yoga.yaml index 925a47f..d8a9c85 100644 --- a/src/tests/bundles/focal-yoga.yaml +++ b/src/tests/bundles/focal-yoga.yaml @@ -118,7 +118,7 @@ services: channel: latest/edge manila: - charm: ../../../manila.charm + charm: ../../../manila_ubuntu-20.04-amd64.charm num_units: 3 options: default-share-backend: cephfsnfs1 diff --git a/src/tests/bundles/impish-xena.yaml b/src/tests/bundles/impish-xena.yaml deleted file mode 100644 index aace6d4..0000000 --- a/src/tests/bundles/impish-xena.yaml +++ /dev/null @@ -1,342 +0,0 @@ -variables: - openstack-origin: &openstack-origin distro - -local_overlay_enabled: False - -series: impish - -comment: -- 'machines section to decide order of deployment. database sooner = faster' -machines: - '0': - constraints: mem=3072M - '1': - constraints: mem=3072M - '2': - constraints: mem=3072M - '3': - '4': - '5': - '6': - '7': - '8': - '9': - '10': - '11': - '12': - '13': - '14': - '15': - # 16 and 17 are nova compute units - '16': - constraints: mem=8G - '17': - constraints: mem=8G - '18': - '19': - '20': - '21': - '22': - '23': - -services: - - manila-mysql-router: - charm: ch:mysql-router - channel: latest/edge - manila-ganesha-mysql-router: - charm: ch:mysql-router - channel: latest/edge - keystone-mysql-router: - charm: ch:mysql-router - channel: latest/edge - neutron-api-mysql-router: - charm: ch:mysql-router - channel: latest/edge - nova-cloud-controller-mysql-router: - charm: ch:mysql-router - channel: latest/edge - glance-mysql-router: - charm: ch:mysql-router - channel: latest/edge - placement-mysql-router: - charm: ch:mysql-router - channel: latest/edge - - mysql-innodb-cluster: - charm: ch:mysql-innodb-cluster - num_units: 3 - options: - source: *openstack-origin - to: - - '0' - - '1' - - '2' - channel: latest/edge - - manila-ganesha: - num_units: 1 - charm: ch:manila-ganesha - options: - openstack-origin: *openstack-origin - to: - - '3' - channel: latest/edge - - ceph-mon: - charm: ch:ceph-mon - num_units: 3 - options: - source: *openstack-origin - to: - - '4' - - '5' - - '6' - channel: latest/edge - - ceph-osd: - charm: ch:ceph-osd - num_units: 3 - options: - source: *openstack-origin - storage: - osd-devices: 'cinder,10G' - to: - - '7' - - '8' - - '9' - channel: latest/edge - - ceph-fs: - charm: ch:ceph-fs - num_units: 2 - options: - source: *openstack-origin - to: - - '10' - - '11' - channel: latest/edge - - manila: - charm: ../../../manila.charm - num_units: 3 - options: - default-share-backend: cephfsnfs1 - share-protocols: NFS - openstack-origin: *openstack-origin - to: - - '12' - - '13' - - '14' - - nova-cloud-controller: - charm: ch:nova-cloud-controller - num_units: 1 - options: - network-manager: Neutron - openstack-origin: *openstack-origin - to: - - '15' - channel: latest/edge - - nova-compute: - charm: ch:nova-compute - num_units: 2 - storage: - ephemeral-device: '40G' - options: - config-flags: default_ephemeral_format=ext4 - enable-live-migration: true - enable-resize: true - migration-auth-type: ssh - openstack-origin: *openstack-origin - to: - - '16' - - '17' - channel: latest/edge - - glance: - charm: ch:glance - num_units: 1 - options: - openstack-origin: *openstack-origin - to: - - '18' - channel: latest/edge - - neutron-api: - charm: ch:neutron-api - num_units: 1 - options: - manage-neutron-plugin-legacy-mode: true - neutron-plugin: ovs - flat-network-providers: physnet1 - neutron-security-groups: true - openstack-origin: *openstack-origin - to: - - '19' - channel: latest/edge - - neutron-openvswitch: - charm: ch:neutron-openvswitch - channel: latest/edge - - neutron-gateway: - charm: ch:neutron-gateway - num_units: 1 - options: - bridge-mappings: physnet1:br-ex - openstack-origin: *openstack-origin - to: - - '20' - channel: latest/edge - - keystone: - charm: ch:keystone - num_units: 1 - options: - openstack-origin: *openstack-origin - to: - - '21' - channel: latest/edge - - rabbitmq-server: - charm: ch:rabbitmq-server - num_units: 1 - options: - source: *openstack-origin - to: - - '22' - channel: latest/edge - - placement: - charm: ch:placement - num_units: 1 - options: - openstack-origin: *openstack-origin - to: - - '23' - channel: latest/edge - - nrpe: - charm: cs:nrpe - -relations: - - - 'ceph-mon' - - 'ceph-osd' - - - - 'ceph-mon' - - 'ceph-fs' - - - - 'ceph-mon' - - 'manila-ganesha' - - - - 'manila:shared-db' - - 'manila-mysql-router:shared-db' - - - 'manila-mysql-router:db-router' - - 'mysql-innodb-cluster:db-router' - - - - 'manila-ganesha' - - 'rabbitmq-server' - - - - 'manila-ganesha' - - 'keystone' - - - - 'manila' - - 'manila-ganesha' - - - - 'manila-ganesha:shared-db' - - 'manila-ganesha-mysql-router:shared-db' - - - 'manila-ganesha-mysql-router:db-router' - - 'mysql-innodb-cluster:db-router' - - - - 'manila' - - 'rabbitmq-server' - - - - 'manila' - - 'keystone' - - - - 'keystone:shared-db' - - 'keystone-mysql-router:shared-db' - - - 'keystone-mysql-router:db-router' - - 'mysql-innodb-cluster:db-router' - - - - 'neutron-api:shared-db' - - 'neutron-api-mysql-router:shared-db' - - - 'neutron-api-mysql-router:db-router' - - 'mysql-innodb-cluster:db-router' - - - - 'neutron-api:amqp' - - 'rabbitmq-server:amqp' - - - - 'neutron-api:neutron-api' - - 'nova-cloud-controller:neutron-api' - - - - 'neutron-api:neutron-plugin-api' - - 'neutron-gateway:neutron-plugin-api' - - - - 'neutron-api:identity-service' - - 'keystone:identity-service' - - - - 'nova-compute:neutron-plugin' - - 'neutron-openvswitch:neutron-plugin' - - - - 'nova-cloud-controller:shared-db' - - 'nova-cloud-controller-mysql-router:shared-db' - - - 'nova-cloud-controller-mysql-router:db-router' - - 'mysql-innodb-cluster:db-router' - - - - 'neutron-gateway:amqp' - - 'rabbitmq-server:amqp' - - - - 'nova-cloud-controller:amqp' - - 'rabbitmq-server:amqp' - - - - 'nova-compute:amqp' - - 'rabbitmq-server:amqp' - - - - 'neutron-openvswitch:amqp' - - 'rabbitmq-server:amqp' - - - - 'nova-cloud-controller:identity-service' - - 'keystone:identity-service' - - - - 'nova-cloud-controller:cloud-compute' - - 'nova-compute:cloud-compute' - - - - 'glance:identity-service' - - 'keystone:identity-service' - - - - 'glance:shared-db' - - 'glance-mysql-router:shared-db' - - - 'glance-mysql-router:db-router' - - 'mysql-innodb-cluster:db-router' - - - - 'glance:amqp' - - 'rabbitmq-server:amqp' - - - - 'nova-compute:image-service' - - 'glance:image-service' - - - - 'nova-cloud-controller:image-service' - - 'glance:image-service' - - - - 'nova-cloud-controller:quantum-network-service' - - 'neutron-gateway:quantum-network-service' - - - - 'placement:placement' - - 'nova-cloud-controller:placement' - - - - 'placement:amqp' - - 'rabbitmq-server:amqp' - - - - 'placement:shared-db' - - 'placement-mysql-router:shared-db' - - - 'placement-mysql-router:db-router' - - 'mysql-innodb-cluster:db-router' - - - - 'placement:identity-service' - - 'keystone:identity-service' - - - - 'nrpe:nrpe-external-master' - - 'manila:nrpe-external-master' diff --git a/src/tests/bundles/jammy-yoga.yaml b/src/tests/bundles/jammy-yoga.yaml index 3922ec4..a422aa4 100644 --- a/src/tests/bundles/jammy-yoga.yaml +++ b/src/tests/bundles/jammy-yoga.yaml @@ -118,7 +118,7 @@ services: channel: latest/edge manila: - charm: ../../../manila.charm + charm: ../../../manila_ubuntu-22.04-amd64.charm num_units: 3 options: default-share-backend: cephfsnfs1 diff --git a/src/tests/tests.yaml b/src/tests/tests.yaml index 0e239f7..aa73328 100644 --- a/src/tests/tests.yaml +++ b/src/tests/tests.yaml @@ -9,7 +9,6 @@ dev_bundles: # handle - ganesha: focal-xena - ganesha: focal-yoga -- ganesha: impish-xena - ganesha: jammy-yoga smoke_bundles: @@ -41,7 +40,4 @@ configure_options: tests_options: force_deploy: - # NOTE(lourot): this is needed because the NRPE charm isn't available on - # non-LTS Ubuntu series. See lp:1933643 - - impish-xena - jammy-yoga diff --git a/tox.ini b/tox.ini index e22fe48..e649e6a 100644 --- a/tox.ini +++ b/tox.ini @@ -11,18 +11,6 @@ envlist = pep8,py3 sitepackages = False # NOTE: Avoid false positives by not skipping missing interpreters. skip_missing_interpreters = False -# NOTES: -# * We avoid the new dependency resolver by pinning pip < 20.3, see -# https://github.com/pypa/pip/issues/9187 -# * Pinning dependencies requires tox >= 3.2.0, see -# https://tox.readthedocs.io/en/latest/config.html#conf-requires -# * It is also necessary to pin virtualenv as a newer virtualenv would still -# lead to fetching the latest pip in the func* tox targets, see -# https://stackoverflow.com/a/38133283 -requires = - pip < 20.3 - virtualenv < 20.0 - setuptools<50.0.0 # NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci minversion = 3.18.0 @@ -41,7 +29,6 @@ allowlist_externals = charmcraft bash tox - rename.sh deps = -r{toxinidir}/requirements.txt @@ -50,13 +37,12 @@ basepython = python3 deps = -r{toxinidir}/build-requirements.txt commands = charmcraft clean - charmcraft -v build - {toxinidir}/rename.sh + charmcraft -v pack [testenv:build-reactive] basepython = python3 commands = - charm-build --log-level DEBUG --use-lock-file-branches -o {toxinidir}/build/builds src {posargs} + charm-build --log-level DEBUG --use-lock-file-branches --binary-wheels-from-source -o {toxinidir}/build/builds src {posargs} [testenv:add-build-lock-file] basepython = python3 @@ -91,7 +77,7 @@ commands = stestr run --slowest {posargs} [testenv:pep8] basepython = python3 deps = flake8==3.9.2 - charm-tools==2.8.3 + git+https://github.com/juju/charm-tools.git commands = flake8 {posargs} src unit_tests [testenv:func-target] diff --git a/unit_tests/test_lib_charm_openstack_manila.py b/unit_tests/test_lib_charm_openstack_manila.py index f325f32..a4602c7 100644 --- a/unit_tests/test_lib_charm_openstack_manila.py +++ b/unit_tests/test_lib_charm_openstack_manila.py @@ -167,6 +167,8 @@ class TestManilaCharm(Helper): # note that this also tests _custom_register_endpoints() indirectly, # which means it doesn't require a separate test. keystone = mock.MagicMock() + relation = mock.MagicMock() + keystone.relations.__iter__.return_value = [relation] config = { 'region': 'the_region', } @@ -190,19 +192,18 @@ class TestManilaCharm(Helper): self.internal_url_v2.return_value = 'i2' self.admin_url_v2.return_value = 'a2' c.register_endpoints(keystone) - v1 = mock.call(v1_admin_url='a1', - v1_internal_url='i1', - v1_public_url='p1', - v1_region='the_region', - v1_service='manila') - v2 = mock.call(v2_admin_url='a2', - v2_internal_url='i2', - v2_public_url='p2', - v2_region='the_region', - v2_service='manilav2') + v1 = mock.call({'v1_admin_url': 'a1', + 'v1_internal_url': 'i1', + 'v1_public_url': 'p1', + 'v1_region': 'the_region', + 'v1_service': 'manila'}) + v2 = mock.call({'v2_admin_url': 'a2', + 'v2_internal_url': 'i2', + 'v2_public_url': 'p2', + 'v2_region': 'the_region', + 'v2_service': 'manilav2'}) calls = [v1, v2] - keystone.set_local.assert_has_calls(calls) - keystone.set_remote.assert_has_calls(calls) + relation.to_publish_raw.update.assert_has_calls(calls) def test_url_endpoints_creation(self): # Tests that the endpoint functions call through to the baseclass