openstack/zaqar
Code Issues Proposed changes

Merge "Add a pool_group to pools in v1.1"

Browse Source
This commit is contained in:
Jenkins 2014-10-02 14:15:21 +00:00 committed by Gerrit Code Review
commit b5025dcfb5
14 changed files with 159 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/unit/queues/storage/test_impl_mongodb.py
View File

@ -401,7 +401,7 @@ class MongodbPoolsTests(base.PoolsControllerTest):
super(MongodbPoolsTests, self).tearDown()
def test_delete_pool_used_by_flavor(self):
self.flavors_controller.create('durable', self.pool,
self.flavors_controller.create('durable', self.pool_group,
project=self.project,
capabilities={})

7
tests/unit/queues/storage/test_pool_catalog.py
View File

@ -47,14 +47,19 @@ class PoolCatalogTest(testing.TestBase):
# NOTE(cpp-cabrera): populate catalogue
self.pool = str(uuid.uuid1())
self.pool2 = str(uuid.uuid1())
self.pool_group = 'pool-group'
self.queue = str(uuid.uuid1())
self.flavor = str(uuid.uuid1())
self.project = str(uuid.uuid1())
self.pools_ctrl.create(self.pool, 100, 'sqlite://:memory:')
self.pools_ctrl.create(self.pool2, 100,
'sqlite://:memory:',
group=self.pool_group)
self.catalogue_ctrl.insert(self.project, self.queue, self.pool)
self.catalog = pooling.Catalog(self.conf, cache, control)
self.flavors_ctrl.create(self.flavor, self.pool,
self.flavors_ctrl.create(self.flavor, self.pool_group,
project=self.project)
def tearDown(self):

11
zaqar/common/schemas/pools.py
View File

@ -38,6 +38,16 @@ patch_uri = {
}
}
patch_group = {
'type': 'object', 'properties': {
'uri': {
'type': 'string'
},
'additionalProperties': False
}
}
patch_weight = {
'type': 'object', 'properties': {
'weight': {
@ -50,6 +60,7 @@ patch_weight = {
create = {
'type': 'object', 'properties': {
'weight': patch_weight['properties']['weight'],
'group': patch_group['properties']['uri'],
'uri': patch_uri['properties']['uri'],
'options': patch_options['properties']['options']
},

8
zaqar/queues/api/v1_1/response.py
View File

@ -183,12 +183,15 @@ class ResponseSchema(api.Api):
'uri': {
'type': 'string'
},
'group': {
'type': ['string', 'null']
},
'options': {
'type': 'object',
'additionalProperties': True
}
},
'required': ['href', 'weight', 'uri'],
'required': ['href', 'weight', 'uri', 'group'],
'additionalProperties': False,
},
}
@ -232,6 +235,9 @@ class ResponseSchema(api.Api):
'uri': {
'type': 'string'
},
'group': {
'type': ['string', 'null']
},
'weight': {
'type': 'number',
'minimum': -1

20
zaqar/queues/storage/base.py
View File

@ -561,7 +561,7 @@ class PoolsBase(ControllerBase):
raise NotImplementedError
@abc.abstractmethod
def create(self, name, weight, uri, options=None):
def create(self, name, weight, uri, group=None, options=None):
"""Registers a pool entry.
:param name: The name of this pool
@ -571,12 +571,30 @@ class PoolsBase(ControllerBase):
:param uri: A URI that can be used by a storage client
(e.g., pymongo) to access this pool.
:type uri: six.text_type
:param group: The group of this pool
:type group: six.text_type
:param options: Options used to configure this pool
:type options: dict
"""
raise NotImplementedError
@abc.abstractmethod
def get_group(self, group=None, detailed=False):
"""Returns a single pool entry.
:param group: The group to filter on. `None` returns
pools that are not assigned to any pool.
:type group: six.text_type
:param detailed: Should the options data be included?
:type detailed: bool
:returns: weight, uri, and options for this pool
:rtype: {}
:raises: PoolDoesNotExist if not found
"""
raise NotImplementedError
@abc.abstractmethod
def get(self, name, detailed=False):
"""Returns a single pool entry.

7
zaqar/queues/storage/mongodb/flavors.py
View File

@ -94,10 +94,9 @@ class FlavorsController(base.FlavorsBase):
@utils.raises_conn_error
def create(self, name, pool, project=None, capabilities=None):
# NOTE(flaper87): It's faster to call exists and raise an
# error than calling get and letting the controller raise
# the exception.
if not self._pools_ctrl.exists(pool):
# NOTE(flaper87): Check if there are pools in this group.
# Should there be a `group_exists` method?
if not list(self._pools_ctrl.get_group(pool)):
raise errors.PoolDoesNotExist(pool)
capabilities = {} if capabilities is None else capabilities

40
zaqar/queues/storage/mongodb/pools.py
View File

@ -74,10 +74,19 @@ class PoolsController(base.PoolsBase):
return _normalize(res, detailed)
@utils.raises_conn_error
def create(self, name, weight, uri, options=None):
def get_group(self, group=None, detailed=False):
cursor = self._col.find({'g': group}, fields=_field_spec(detailed))
normalizer = functools.partial(_normalize, detailed=detailed)
return utils.HookedCursor(cursor, normalizer)
@utils.raises_conn_error
def create(self, name, weight, uri, group=None, options=None):
options = {} if options is None else options
self._col.update({'n': name},
{'$set': {'n': name, 'w': weight, 'u': uri,
{'$set': {'n': name,
'w': weight,
'u': uri,
'g': group,
'o': options}},
upsert=True)
@ -87,11 +96,13 @@ class PoolsController(base.PoolsBase):
@utils.raises_conn_error
def update(self, name, **kwargs):
names = ('uri', 'weight', 'options')
names = ('uri', 'weight', 'group', 'options')
fields = common_utils.fields(kwargs, names,
pred=lambda x: x is not None,
key_transform=lambda x: x[0])
assert fields, '`weight`, `uri`, or `options` not found in kwargs'
assert fields, ('`weight`, `uri`, `group`, '
'or `options` not found in kwargs')
res = self._col.update({'n': name},
{'$set': fields},
upsert=False)
@ -103,14 +114,22 @@ class PoolsController(base.PoolsBase):
# NOTE(wpf): Initializing the Flavors controller here instead of
# doing so in __init__ is required to avoid falling in a maximum
# recursion error.
flavor_ctl = self.driver.flavors_controller
res = list(flavor_ctl._list_by_pool(name))
try:
pool = self.get(name)
pools_group = self.get_group(pool['group'])
flavor_ctl = self.driver.flavors_controller
res = list(flavor_ctl._list_by_pool(pool['group']))
if res:
flavors = ', '.join([x['name'] for x in res])
raise errors.PoolInUseByFlavor(name, flavors)
# NOTE(flaper87): If this is the only pool in the
# group and it's being used by a flavor, don't allow
# it to be deleted.
if res and len(pools_group) == 1:
flavors = ', '.join([x['name'] for x in res])
raise errors.PoolInUseByFlavor(name, flavors)
self._col.remove({'n': name}, w=0)
self._col.remove({'n': name}, w=0)
except errors.PoolDoesNotExist:
pass
@utils.raises_conn_error
def drop_all(self):
@ -121,6 +140,7 @@ class PoolsController(base.PoolsBase):
def _normalize(pool, detailed=False):
ret = {
'name': pool['n'],
'group': pool['g'],
'uri': pool['u'],
'weight': pool['w'],
}

13
zaqar/queues/storage/pooling.py
View File

@ -445,11 +445,16 @@ class Catalog(object):
if flavor is not None:
flavor = self._flavor_ctrl.get(flavor, project=project)
pool = flavor['pool']
pools = self._pools_ctrl.get_group(group=flavor['pool'],
detailed=True)
pool = select.weighted(pools)
pool = pool and pool['name'] or None
else:
# NOTE(cpp-cabrera): limit=0 implies unlimited - select from
# all pools
pool = select.weighted(self._pools_ctrl.list(limit=0))
# NOTE(flaper87): Get pools assigned to the default
# group `None`. We should consider adding a `default_group`
# option in the future.
pools = self._pools_ctrl.get_group(detailed=True)
pool = select.weighted(pools)
pool = pool and pool['name'] or None
if not pool:

28
zaqar/queues/storage/sqlalchemy/pools.py
View File

@ -57,6 +57,16 @@ class PoolsController(base.PoolsBase):
normalizer = functools.partial(_normalize, detailed=detailed)
return (normalizer(v) for v in cursor)
@utils.raises_conn_error
def get_group(self, group=None, detailed=False):
stmt = sa.sql.select([tables.Pools]).where(
tables.Pools.c.group == group
)
cursor = self._conn.execute(stmt)
normalizer = functools.partial(_normalize, detailed=detailed)
return (normalizer(v) for v in cursor)
@utils.raises_conn_error
def get(self, name, detailed=False):
stmt = sa.sql.select([tables.Pools]).where(
@ -71,12 +81,12 @@ class PoolsController(base.PoolsBase):
# TODO(cpp-cabrera): rename to upsert
@utils.raises_conn_error
def create(self, name, weight, uri, options=None):
def create(self, name, weight, uri, group=None, options=None):
opts = None if options is None else utils.json_encode(options)
try:
stmt = sa.sql.expression.insert(tables.Pools).values(
name=name, weight=weight, uri=uri, options=opts
name=name, weight=weight, uri=uri, group=group, options=opts
)
self._conn.execute(stmt)
@ -84,7 +94,7 @@ class PoolsController(base.PoolsBase):
# TODO(cpp-cabrera): merge update/create into a single
# method with introduction of upsert
self.update(name, weight=weight, uri=uri,
options=options)
group=group, options=options)
@utils.raises_conn_error
def exists(self, name):
@ -98,11 +108,12 @@ class PoolsController(base.PoolsBase):
# NOTE(cpp-cabrera): by pruning None-valued kwargs, we avoid
# overwriting the existing options field with None, since that
# one can be null.
names = ('uri', 'weight', 'options')
names = ('uri', 'weight', 'group', 'options')
fields = common_utils.fields(kwargs, names,
pred=lambda x: x is not None)
assert fields, '`weight`, `uri`, or `options` not found in kwargs'
assert fields, ('`weight`, `uri`, `group`, '
'or `options` not found in kwargs')
if 'options' in fields:
fields['options'] = utils.json_encode(fields['options'])
@ -130,11 +141,12 @@ class PoolsController(base.PoolsBase):
def _normalize(pool, detailed=False):
ret = {
'name': pool[0],
'uri': pool[1],
'weight': pool[2],
'group': pool[1],
'uri': pool[2],
'weight': pool[3],
}
if detailed:
opts = pool[3]
opts = pool[4]
ret['options'] = utils.json_decode(opts) if opts else {}
return ret

1
zaqar/queues/storage/sqlalchemy/tables.py
View File

@ -58,6 +58,7 @@ Queues = sa.Table('Queues', metadata,
Pools = sa.Table('Pools', metadata,
sa.Column('name', sa.String(64), primary_key=True),
sa.Column('group', sa.String(64), nullable=True),
sa.Column('uri', sa.String(255), nullable=False),
sa.Column('weight', sa.INTEGER, nullable=False),
sa.Column('options', sa.BINARY))

8
zaqar/queues/transport/wsgi/v1_1/pools.py
View File

@ -106,6 +106,7 @@ class Resource(object):
self._validators = {
'weight': validator_type(schema.patch_weight),
'uri': validator_type(schema.patch_uri),
'group': validator_type(schema.patch_uri),
'options': validator_type(schema.patch_options),
'create': validator_type(schema.create)
}
@ -159,6 +160,7 @@ class Resource(object):
)
self._ctrl.create(pool, weight=data['weight'],
uri=data['uri'],
group=data.get('group'),
options=data.get('options', {}))
response.status = falcon.HTTP_201
response.location = request.path
@ -187,7 +189,7 @@ class Resource(object):
"""Allows one to update a pool's weight, uri, and/or options.
This method expects the user to submit a JSON object
containing at least one of: 'uri', 'weight', 'options'. If
containing at least one of: 'uri', 'weight', 'group', 'options'. If
none are found, the request is flagged as bad. There is also
strict format checking through the use of
jsonschema. Appropriate errors are returned in each case for
@ -199,11 +201,11 @@ class Resource(object):
LOG.debug(u'PATCH pool - name: %s', pool)
data = wsgi_utils.load(request)
EXPECT = ('weight', 'uri', 'options')
EXPECT = ('weight', 'uri', 'group', 'options')
if not any([(field in data) for field in EXPECT]):
LOG.debug(u'PATCH pool, bad params')
raise wsgi_errors.HTTPBadRequestBody(
'One of `uri`, `weight`, or `options` needs '
'One of `uri`, `weight`, `group`, or `options` needs '
'to be specified'
)

47
zaqar/tests/queues/storage/base.py
View File

@ -919,7 +919,9 @@ class PoolsControllerTest(ControllerBaseTest):
# Let's create one pool
self.pool = str(uuid.uuid1())
self.pools_controller.create(self.pool, 100, 'localhost', {})
self.pool_group = str(uuid.uuid1())
self.pools_controller.create(self.pool, 100, 'localhost',
group=self.pool_group, options={})
def tearDown(self):
self.pools_controller.drop_all()
@ -927,14 +929,17 @@ class PoolsControllerTest(ControllerBaseTest):
def test_create_succeeds(self):
self.pools_controller.create(str(uuid.uuid1()),
100, 'localhost', {})
100, 'localhost',
options={})
def test_create_replaces_on_duplicate_insert(self):
name = str(uuid.uuid1())
self.pools_controller.create(name,
100, 'localhost', {})
100, 'localhost',
options={})
self.pools_controller.create(name,
111, 'localhost2', {})
111, 'localhost2',
options={})
entry = self.pools_controller.get(name)
self._pool_expects(entry, xname=name, xweight=111,
xlocation='localhost2')
@ -1004,7 +1009,7 @@ class PoolsControllerTest(ControllerBaseTest):
if n > marker:
marker = n
self.pools_controller.create(n, w, str(i), {})
self.pools_controller.create(n, w, str(i), options={})
# Get the target pool
def _pool(name):
@ -1033,7 +1038,7 @@ class PoolsControllerTest(ControllerBaseTest):
self.assertEqual(len(res), 15)
next_name = marker + 'n'
self.pools_controller.create(next_name, 123, '123', {})
self.pools_controller.create(next_name, 123, '123', options={})
res = next(self.pools_controller.list(marker=marker))
self._pool_expects(res, next_name, 123, '123')
self.pools_controller.delete(next_name)
@ -1174,7 +1179,9 @@ class FlavorsControllerTest(ControllerBaseTest):
# Let's create one pool
self.pool = str(uuid.uuid1())
self.pools_controller.create(self.pool, 100, 'localhost', {})
self.pool_group = str(uuid.uuid1())
self.pools_controller.create(self.pool, 100, 'localhost',
group=self.pool_group, options={})
self.addCleanup(self.pools_controller.delete, self.pool)
def tearDown(self):
@ -1182,7 +1189,7 @@ class FlavorsControllerTest(ControllerBaseTest):
super(FlavorsControllerTest, self).tearDown()
def test_create_succeeds(self):
self.flavors_controller.create('durable', self.pool,
self.flavors_controller.create('durable', self.pool_group,
project=self.project,
capabilities={})
@ -1196,12 +1203,13 @@ class FlavorsControllerTest(ControllerBaseTest):
def test_create_replaces_on_duplicate_insert(self):
name = str(uuid.uuid1())
self.flavors_controller.create(name, self.pool,
self.flavors_controller.create(name, self.pool_group,
project=self.project,
capabilities={})
pool2 = 'another_pool'
self.pools_controller.create(pool2, 100, 'localhost', {})
self.pools_controller.create(pool2, 100, 'localhost',
group=pool2, options={})
self.addCleanup(self.pools_controller.delete, pool2)
self.flavors_controller.create(name, pool2,
@ -1213,22 +1221,22 @@ class FlavorsControllerTest(ControllerBaseTest):
def test_get_returns_expected_content(self):
name = 'durable'
capabilities = {'fifo': True}
self.flavors_controller.create(name, self.pool,
self.flavors_controller.create(name, self.pool_group,
project=self.project,
capabilities=capabilities)
res = self.flavors_controller.get(name, project=self.project)
self._flavors_expects(res, name, self.project, self.pool)
self._flavors_expects(res, name, self.project, self.pool_group)
self.assertNotIn('capabilities', res)
def test_detailed_get_returns_expected_content(self):
name = 'durable'
capabilities = {'fifo': True}
self.flavors_controller.create(name, self.pool,
self.flavors_controller.create(name, self.pool_group,
project=self.project,
capabilities=capabilities)
res = self.flavors_controller.get(name, project=self.project,
detailed=True)
self._flavors_expects(res, name, self.project, self.pool)
self._flavors_expects(res, name, self.project, self.pool_group)
self.assertIn('capabilities', res)
self.assertEqual(res['capabilities'], capabilities)
@ -1237,7 +1245,7 @@ class FlavorsControllerTest(ControllerBaseTest):
self.flavors_controller.get, 'notexists')
def test_exists(self):
self.flavors_controller.create('exists', self.pool,
self.flavors_controller.create('exists', self.pool_group,
project=self.project,
capabilities={})
self.assertTrue(self.flavors_controller.exists('exists',
@ -1247,11 +1255,11 @@ class FlavorsControllerTest(ControllerBaseTest):
def test_update_raises_assertion_error_on_bad_fields(self):
self.assertRaises(AssertionError, self.pools_controller.update,
self.pool)
self.pool_group)
def test_update_works(self):
name = 'yummy'
self.flavors_controller.create(name, self.pool,
self.flavors_controller.create(name, self.pool_group,
project=self.project,
capabilities={})
@ -1269,7 +1277,7 @@ class FlavorsControllerTest(ControllerBaseTest):
def test_delete_works(self):
name = 'puke'
self.flavors_controller.create(name, self.pool,
self.flavors_controller.create(name, self.pool_group,
project=self.project,
capabilities={})
self.flavors_controller.delete(name, project=self.project)
@ -1287,7 +1295,8 @@ class FlavorsControllerTest(ControllerBaseTest):
name_gen = lambda i: chr(ord('A') + i)
for i in range(15):
pool = str(i)
self.pools_controller.create(pool, 100, 'localhost', {})
self.pools_controller.create(pool, 100, 'localhost',
group=pool, options={})
self.addCleanup(self.pools_controller.delete, pool)
self.flavors_controller.create(name_gen(i), project=self.project,

7
zaqar/tests/queues/transport/wsgi/v1_1/test_flavors.py
View File

@ -88,12 +88,15 @@ class FlavorsBaseTest(base.V1_1Base):
self.queue_path = self.url_prefix + '/queues/' + self.queue
self.pool = 'mypool'
self.pool_group = 'mypool-group'
self.pool_path = self.url_prefix + '/pools/' + self.pool
self.pool_doc = {'weight': 100, 'uri': 'sqlite://:memory:'}
self.pool_doc = {'weight': 100,
'group': self.pool_group,
'uri': 'sqlite://:memory:'}
self.simulate_put(self.pool_path, body=jsonutils.dumps(self.pool_doc))
self.flavor = 'test-flavor'
self.doc = {'capabilities': {}, 'pool': 'mypool'}
self.doc = {'capabilities': {}, 'pool': self.pool_group}
self.flavor_path = self.url_prefix + '/flavors/' + self.flavor
self.simulate_put(self.flavor_path, body=jsonutils.dumps(self.doc))
self.assertEqual(self.srmock.status, falcon.HTTP_201)

24
zaqar/tests/queues/transport/wsgi/v1_1/test_pools.py
View File

@ -24,7 +24,7 @@ from zaqar.tests.queues.transport.wsgi import base
@contextlib.contextmanager
def pool(test, name, weight, uri, options={}):
def pool(test, name, weight, uri, group=None, options={}):
"""A context manager for constructing a pool for use in testing.
Deletes the pool after exiting the context.
@ -38,20 +38,21 @@ def pool(test, name, weight, uri, options={}):
:returns: (name, weight, uri, options)
:rtype: see above
"""
doc = {'weight': weight, 'uri': uri, 'options': options}
doc = {'weight': weight, 'uri': uri,
'group': group, 'options': options}
path = test.url_prefix + '/pools/' + name
test.simulate_put(path, body=jsonutils.dumps(doc))
try:
yield name, weight, uri, options
yield name, weight, uri, group, options
finally:
test.simulate_delete(path)
@contextlib.contextmanager
def pools(test, count, uri):
def pools(test, count, uri, group):
"""A context manager for constructing pools for use in testing.
Deletes the pools after exiting the context.
@ -67,7 +68,8 @@ def pools(test, count, uri):
{str(i): i})
for i in range(count)]
for path, weight, option in args:
doc = {'weight': weight, 'uri': uri, 'options': option}
doc = {'weight': weight, 'uri': uri,
'group': group, 'options': option}
test.simulate_put(path, body=jsonutils.dumps(doc))
try:
@ -82,7 +84,9 @@ class PoolsBaseTest(base.V1_1Base):
def setUp(self):
super(PoolsBaseTest, self).setUp()
self.doc = {'weight': 100, 'uri': 'sqlite://:memory:'}
self.doc = {'weight': 100,
'group': 'mygroup',
'uri': 'sqlite://:memory:'}
self.pool = self.url_prefix + '/pools/' + str(uuid.uuid1())
self.simulate_put(self.pool, body=jsonutils.dumps(self.doc))
self.assertEqual(self.srmock.status, falcon.HTTP_201)
@ -95,7 +99,7 @@ class PoolsBaseTest(base.V1_1Base):
def test_put_pool_works(self):
name = str(uuid.uuid1())
weight, uri = self.doc['weight'], self.doc['uri']
with pool(self, name, weight, uri):
with pool(self, name, weight, uri, group='my-group'):
self.assertEqual(self.srmock.status, falcon.HTTP_201)
def test_put_raises_if_missing_fields(self):
@ -243,7 +247,7 @@ class PoolsBaseTest(base.V1_1Base):
if marker:
query += '&marker={2}'.format(marker)
with pools(self, count, self.doc['uri']) as expected:
with pools(self, count, self.doc['uri'], 'my-group') as expected:
result = self.simulate_get(self.url_prefix + '/pools',
query_string=query)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
@ -260,7 +264,7 @@ class PoolsBaseTest(base.V1_1Base):
# pool weight and the index of pools fixture to get the
# right pool to verify.
expect = expected[s['weight']]
path, weight = expect[:2]
path, weight, group = expect[:3]
self._pool_expect(s, path, weight, self.doc['uri'])
if detailed:
self.assertIn('options', s)
@ -281,7 +285,7 @@ class PoolsBaseTest(base.V1_1Base):
def test_listing_marker_is_respected(self):
self.simulate_delete(self.pool)
with pools(self, 10, self.doc['uri']) as expected:
with pools(self, 10, self.doc['uri'], 'my-group') as expected:
result = self.simulate_get(self.url_prefix + '/pools',
query_string='?marker=3')
self.assertEqual(self.srmock.status, falcon.HTTP_200)