openstack/zun
Code Issues Proposed changes

Add Quota DB etcd

Browse Source 
Change-Id: Iff9593ab3639342cb4b242facb865fe185766521
Partial-Implements: blueprint quota-support
This commit is contained in:
Kien Nguyen 2018-04-19 17:31:12 +07:00
parent eb5608c1cc
commit 5a506ee93c
11 changed files with 450 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
zun/common/consts.py
View File

@ -58,3 +58,5 @@ SANDBOX_NAME_PREFIX = 'zun-sandbox-'
# Storage drivers that support disk quota feature
SUPPORTED_STORAGE_DRIVERS = \
['devicemapper', 'overlay2', 'windowfilter', 'zfs', 'btrfs']
DEFAULT_QUOTA_CLASS_NAME = 'default'

163
zun/db/etcd/api.py
View File

@ -24,6 +24,7 @@ from oslo_utils import timeutils
from oslo_utils import uuidutils
import six
from zun.common import consts
from zun.common import exception
from zun.common.i18n import _
from zun.common import singleton
@ -88,6 +89,10 @@ def translate_etcd_result(etcd_result, model_type):
ret = models.ContainerAction(data)
elif model_type == 'container_action_event':
ret = models.ContainerActionEvent(data)
elif model_type == 'quota':
ret = models.Quota(data)
elif model_type == 'quota_class':
ret = models.QuotaClass(data)
else:
raise exception.InvalidParameterValue(
_('The model_type value: %s is invalid.'), model_type)
@ -1151,3 +1156,161 @@ class EtcdAPI(object):
def action_events_get(self, context, action_id):
events = self._action_events_get(context, action_id)
return events
def quota_get(self, context, project_id, resource):
try:
res = self.client.read(
'/quotas/{}/{}'. format(project_id, resource))
if res.value is not None:
return translate_etcd_result(res, 'quota')
else:
raise exception.QuotaNotFound()
except etcd.EtcdKeyNotFound:
raise exception.QuotaNotFound()
except Exception as e:
LOG.error('Error occurred while retrieving quota: %s',
six.text_type(e))
raise
def quota_get_all_by_project(self, context, project_id):
try:
res = getattr(self.client.read('/quotas/{}'. format(project_id)),
'children', None)
quotas = []
for q in res:
if q.value is not None:
quotas.append(translate_etcd_result(q, 'quota'))
return quotas
except etcd.EtcdKeyNotFound:
return []
except Exception as e:
LOG.error('Error occurred while retrieving quota: %s',
six.text_type(e))
raise
@lockutils.synchronized('etcd_quota')
def quota_create(self, context, project_id, resource, limit):
quota_data = {
'project_id': project_id,
'resource': resource,
'hard_limit': limit,
'created_at': datetime.isoformat(timeutils.utcnow()),
'uuid': uuidutils.generate_uuid()
}
quota = models.Quota(quota_data)
try:
quota.save()
except Exception:
raise
return quota
@lockutils.synchronized('etcd_quota')
def quota_update(self, context, project_id, resource, limit):
quota_data = {
'project_id': project_id,
'resource': resource,
'hard_limit': limit,
}
try:
target = self.client.read(
'/quotas/{}/{}' . format(project_id, resource))
target_value = json.loads(target.value)
quota_data['updated_at'] = datetime.isoformat(timeutils.utcnow())
target_value.update(quota_data)
target.value = json.dump_as_bytes(target_value)
self.client.update(target)
except etcd.EtcdKeyNotFound:
raise exception.QuotaNotFound()
except Exception as e:
LOG.error('Error occurred while updating quota: %s',
six.text_type(e))
raise
@lockutils.synchronized('etcd_quota')
def quota_destroy(self, context, project_id, resource):
self.client.delete('/quotas/{}/{}' . format(project_id, resource))
def quota_class_create(self, context, class_name, resource, limit):
quota_class_data = {
'class_name': class_name,
'resource': resource,
'hard_limit': limit,
'created_at': datetime.isoformat(timeutils.utcnow()),
'uuid': uuidutils.generate_uuid()
}
quota_class = models.QuotaClass(quota_class_data)
try:
quota_class.save()
except Exception:
raise
return quota_class
def quota_class_get(self, context, class_name, resource):
try:
res = self.client.read(
'/quota_classes/{}/{}'. format(class_name,
resource))
if res.value is not None:
return translate_etcd_result(res, 'quota_class')
else:
raise exception.QuotaClassNotFound()
except etcd.EtcdKeyNotFound:
raise exception.QuotaClassNotFound()
except Exception as e:
LOG.error('Error occurred while retrieving quota class: %s',
six.text_type(e))
raise
def _quota_class_get_all_by_name(self, context, class_name=None):
if class_name is None or class_name == 'default':
class_name = consts.DEFAULT_QUOTA_CLASS_NAME
try:
res = getattr(self.client.read(
'/quota_classes/{}' . format(class_name)),
'children', None)
quota_classes = []
for qc in res:
if qc.value is not None:
quota_classes.append(translate_etcd_result(
qc, 'quota_class'))
return quota_classes
except etcd.EtcdKeyNotFound:
return []
except Exception as e:
LOG.error('Error occurred while retrieving quota class: %s',
six.text_type(e))
raise
def quota_class_get_default(self, context):
return self._quota_class_get_all_by_name(context)
def quota_class_get_all_by_name(self, context, class_name):
return self._quota_class_get_all_by_name(
context, class_name=class_name)
def quota_class_update(self, context, class_name, resource, limit):
quota_class_data = {
'class_name': class_name,
'resource': resource,
'hard_limit': limit,
}
try:
target = self.client.read(
'/quota_classes/{}/{}' . format(class_name, resource))
target_value = json.loads(target.value)
quota_class_data['updated_at'] = datetime.isoformat(
timeutils.utcnow())
target_value.update(quota_class_data)
target.value = json.dump_as_bytes(target_value)
self.client.update(target)
except etcd.EtcdKeyNotFound:
raise exception.QuotaClassNotFound()
except Exception as e:
LOG.error('Error occurred while updating quota class: %s',
six.text_type(e))
raise

59
zun/db/etcd/models.py
View File

@ -372,3 +372,62 @@ class ContainerActionEvent(Base):
@classmethod
def fields(cls):
return cls._fields
class Quota(Base):
"""Represents a Quota."""
_path = '/quotas'
_fields = list(objects.Quota.fields) + ['uuid']
def __init__(self, quota_data):
self.path = Quota.path(project_id=quota_data.get('class_name'),
resource=quota_data.get('resource'))
for f in Quota.fields():
setattr(self, f, None)
self.id = 1
self.update(quota_data)
@classmethod
def path(cls, project_id, resource=None):
if resource is not None:
path = '{}/{}/{}' . format(cls._path, project_id, resource)
else:
path = '{}/{}' . format(cls._path, project_id)
return path
@classmethod
def fields(cls):
return cls._fields
class QuotaClass(Base):
"""Represents a QuotaClass."""
_path = '/quota_classes'
_fields = list(objects.QuotaClass.fields) + ['uuid']
def __init__(self, quota_class_data):
self.path = QuotaClass.path(
class_name=quota_class_data.get('class_name'),
resource=quota_class_data.get('resource'))
for f in Quota.fields():
setattr(self, f, None)
self.id = 1
self.update(quota_class_data)
@classmethod
def path(cls, class_name, resource=None):
if resource is not None:
path = '{}/{}/{}' . format(cls._path, class_name, resource)
else:
path = '{}/{}' . format(cls._path, class_name)
return path
@classmethod
def fields(cls):
return cls._fields

6
zun/db/sqlalchemy/api.py
View File

@ -40,8 +40,6 @@ CONF = zun.conf.CONF
_FACADE = None
_DEFAULT_QUOTA_NAME = 'default'
def _create_facade_lazily():
global _FACADE
@ -1127,10 +1125,10 @@ class Connection(object):
session = get_session()
with session.begin():
rows = model_query(models.QuotaClass, session=session).\
filter_by(class_name=_DEFAULT_QUOTA_NAME).\
filter_by(class_name=consts.DEFAULT_QUOTA_CLASS_NAME).\
all()
result = {'class_name': _DEFAULT_QUOTA_NAME}
result = {'class_name': consts.DEFAULT_QUOTA_CLASS_NAME}
for row in rows:
result[row.resource] = row.hard_limit

8
zun/objects/quota.py
View File

@ -19,13 +19,17 @@ from zun.objects import base
@base.ZunObjectRegistry.register
class Quota(base.ZunPersistentObject, base.ZunObject):
# Version 1.0: Initial version
VERSION = '1.0'
# Version 1.1: Add uuid column
VERSION = '1.1'
fields = {
'id': fields.IntegerField(),
'project_id': fields.StringField(nullable=True),
'resource': fields.StringField(),
'hard_limit': fields.IntegerField(nullable=True)
'hard_limit': fields.IntegerField(nullable=True),
# NOTE(kiennt): By now, this field is only used for etcd. If using sql,
# this field will be None.
'uuid': fields.StringField(nullable=True),
}
@staticmethod

8
zun/objects/quota_class.py
View File

@ -19,13 +19,17 @@ from zun.objects import base
@base.ZunObjectRegistry.register
class QuotaClass(base.ZunPersistentObject, base.ZunObject):
# Version 1.0: Initial version
VERSION = '1.0'
# Version 1.1: Add uuid column
VERSION = '1.1'
fields = {
'id': fields.IntegerField(),
'class_name': fields.StringField(nullable=True),
'resource': fields.StringField(nullable=True),
'hard_limit': fields.IntegerField(nullable=True)
'hard_limit': fields.IntegerField(nullable=True),
# NOTE(kiennt): By now, this field is only used for etcd. If using sql,
# this field will be None.
'uuid': fields.StringField(nullable=True),
}
@staticmethod

109
zun/tests/unit/db/test_quota_classes.py
View File

@ -11,10 +11,19 @@
# under the License.
"""Tests for manipulating Quota via the DB API"""
import json
import etcd
from etcd import Client as etcd_client
import mock
from oslo_config import cfg
from zun.common import consts
from zun.common import context
from zun.common import exception
import zun.conf
from zun.db import api as dbapi
from zun.db.sqlalchemy import api as sqlalchemy_dbapi
from zun.db.etcd import api as etcdapi
from zun.tests.unit.db import base
from zun.tests.unit.db import utils
@ -54,19 +63,19 @@ class DBQuotaClassesTestCase(base.DbTestCase):
def test_get_default_quota_class(self):
default_quota_class_resource_1 = utils.create_test_quota_class(
context=self.ctx,
class_name=sqlalchemy_dbapi._DEFAULT_QUOTA_NAME,
class_name=consts.DEFAULT_QUOTA_CLASS_NAME,
resource='resource_1',
limit=10)
default_quota_class_resource_2 = utils.create_test_quota_class(
context=self.ctx,
class_name=sqlalchemy_dbapi._DEFAULT_QUOTA_NAME,
class_name=consts.DEFAULT_QUOTA_CLASS_NAME,
resource='resource_2',
limit=20)
res = dbapi.quota_class_get_default(self.ctx)
self.assertEqual(res['class_name'],
sqlalchemy_dbapi._DEFAULT_QUOTA_NAME)
consts.DEFAULT_QUOTA_CLASS_NAME)
self.assertEqual(res[default_quota_class_resource_1.resource],
default_quota_class_resource_1.hard_limit)
self.assertEqual(res[default_quota_class_resource_2.resource],
@ -105,3 +114,95 @@ class DBQuotaClassesTestCase(base.DbTestCase):
self.ctx, quota_class.class_name,
quota_class.resource)
self.assertEqual(updated_quota_class.hard_limit, 200)
class EtcdDbQuotaClassTestCase(base.DbTestCase):
def setUp(self):
cfg.CONF.set_override('backend', 'etcd', 'database')
super(EtcdDbQuotaClassTestCase, self).setUp()
@mock.patch.object(etcd_client, 'read')
@mock.patch.object(etcd_client, 'write')
def test_create_quota_class(self, mock_write, mock_read):
mock_read.side_effect = etcd.EtcdKeyNotFound
utils.create_test_quota_class(context=self.context)
mock_read.side_effect = lambda *args: None
self.assertRaises(exception.ResourceExists,
utils.create_test_quota_class,
context=self.context)
@mock.patch.object(etcd_client, 'read')
@mock.patch.object(etcd_client, 'write')
@mock.patch.object(dbapi, '_get_dbdriver_instance')
def test_get_quota_class(self, mock_db_inst,
mock_write, mock_read):
mock_db_inst.return_value = etcdapi.get_backend()
mock_read.side_effect = etcd.EtcdKeyNotFound
quota_class = utils.create_test_quota_class(context=self.context)
mock_read.side_effect = lambda *args: utils.FakeEtcdResult(
quota_class.as_dict())
res = dbapi.quota_class_get(self.context, quota_class.class_name,
quota_class.resource)
self.assertEqual(quota_class.hard_limit, res.hard_limit)
@mock.patch.object(etcd_client, 'read')
@mock.patch.object(etcd_client, 'write')
@mock.patch.object(dbapi, '_get_dbdriver_instance')
def test_get_quota_class_by_default(self, mock_db_inst,
mock_write, mock_read):
mock_db_inst.return_value = etcdapi.get_backend()
mock_read.side_effect = etcd.EtcdKeyNotFound
quota_class_1 = utils.create_test_quota_class(
context=self.context, resource='fake_resource_1', hard_limit=10)
quota_class_2 = utils.create_test_quota_class(
context=self.context, resource='fake_resource_2', hard_limit=10)
quota_classes = [quota_class_1, quota_class_2]
mock_read.side_effect = lambda *args: utils.FakeEtcdMultipleResult(
[quota_class_1.as_dict(), quota_class_2.as_dict()])
res = dbapi.quota_class_get_default(self.context)
self.assertEqual([qc.resource for qc in res],
[qc.resource for qc in quota_classes])
self.assertEqual([q.hard_limit for q in res],
[q.hard_limit for q in quota_classes])
@mock.patch.object(etcd_client, 'read')
@mock.patch.object(etcd_client, 'write')
@mock.patch.object(dbapi, '_get_dbdriver_instance')
def test_get_quota_class_by_cls_name(self, mock_db_inst,
mock_write, mock_read):
cls_name = 'fake_class_name'
mock_db_inst.return_value = etcdapi.get_backend()
mock_read.side_effect = etcd.EtcdKeyNotFound
quota_class_1 = utils.create_test_quota_class(
context=self.context, class_name=cls_name,
resource='fake_resource_1', hard_limit=10)
quota_class_2 = utils.create_test_quota_class(
context=self.context, class_name=cls_name,
resource='fake_resource_2', hard_limit=10)
quota_classes = [quota_class_1, quota_class_2]
mock_read.side_effect = lambda *args: utils.FakeEtcdMultipleResult(
[quota_class_1.as_dict(), quota_class_2.as_dict()])
res = dbapi.quota_class_get_all_by_name(self.context, cls_name)
self.assertEqual([qc.resource for qc in res],
[qc.resource for qc in quota_classes])
self.assertEqual([q.hard_limit for q in res],
[q.hard_limit for q in quota_classes])
@mock.patch.object(etcd_client, 'read')
@mock.patch.object(etcd_client, 'write')
@mock.patch.object(etcd_client, 'update')
@mock.patch.object(dbapi, '_get_dbdriver_instance')
def test_update_quota_class(self, mock_db_inst, mock_update,
mock_write, mock_read):
mock_db_inst.return_value = etcdapi.get_backend()
mock_read.side_effect = etcd.EtcdKeyNotFound
quota_class = utils.create_test_quota_class(context=self.context)
new_hard_limit = 60
mock_read.side_effect = lambda *args: utils.FakeEtcdResult(
quota_class.as_dict())
dbapi.quota_class_update(self.context, quota_class.class_name,
quota_class.resource, new_hard_limit)
self.assertEqual(new_hard_limit,
json.loads(mock_update.call_args_list[0][0][0].
value.decode('utf-8'))['hard_limit'])

100
zun/tests/unit/db/test_quotas.py
View File

@ -11,10 +11,18 @@
# under the License.
"""Tests for manipulating Quota via the DB API"""
import json
import etcd
from etcd import Client as etcd_client
import mock
from oslo_config import cfg
from zun.common import context
from zun.common import exception
import zun.conf
from zun.db import api as dbapi
from zun.db.etcd import api as etcdapi
from zun.tests.unit.db import base
from zun.tests.unit.db import utils
@ -53,7 +61,7 @@ class DBQuotaTestCase(base.DbTestCase):
def test_get_all_project_quota(self):
quota_1 = utils.create_test_quota(context=self.ctx,
project_d=self.project_id,
project_id=self.project_id,
resource='resource_1',
limit=10)
quota_2 = utils.create_test_quota(context=self.ctx,
@ -84,3 +92,93 @@ class DBQuotaTestCase(base.DbTestCase):
updated_quota = dbapi.quota_get(self.ctx, quota.project_id,
quota.resource)
self.assertEqual(updated_quota.hard_limit, 200)
class EtcdDbQuotaTestCase(base.DbTestCase):
def setUp(self):
cfg.CONF.set_override('backend', 'etcd', 'database')
super(EtcdDbQuotaTestCase, self).setUp()
@mock.patch.object(etcd_client, 'read')
@mock.patch.object(etcd_client, 'write')
def test_create_quota(self, mock_write, mock_read):
mock_read.side_effect = etcd.EtcdKeyNotFound
utils.create_test_quota(context=self.context)
mock_read.side_effect = lambda *args: None
self.assertRaises(exception.ResourceExists,
utils.create_test_quota,
context=self.context)
@mock.patch.object(etcd_client, 'read')
@mock.patch.object(etcd_client, 'write')
@mock.patch.object(dbapi, '_get_dbdriver_instance')
def test_get_quota(self, mock_db_inst,
mock_write, mock_read):
mock_db_inst.return_value = etcdapi.get_backend()
mock_read.side_effect = etcd.EtcdKeyNotFound
quota = utils.create_test_quota(context=self.context)
mock_read.side_effect = lambda *args: utils.FakeEtcdResult(
quota.as_dict())
res = dbapi.quota_get(self.context, quota.project_id,
quota.resource)
self.assertEqual(quota.hard_limit, res.hard_limit)
@mock.patch.object(etcd_client, 'read')
@mock.patch.object(etcd_client, 'write')
@mock.patch.object(dbapi, '_get_dbdriver_instance')
def test_get_all_project_quota(self, mock_db_inst,
mock_write, mock_read):
prj_id = 'fake_project_id'
resources = ['fake_resource_1', 'fake_resource_2']
hard_limits = [10, 20]
mock_db_inst.return_value = etcdapi.get_backend()
mock_read.side_effect = etcd.EtcdKeyNotFound
quota_1 = utils.create_test_quota(
context=self.context, project_id=prj_id,
resource=resources[0], hard_limit=hard_limits[0])
quota_2 = utils.create_test_quota(
context=self.context, project_id=prj_id,
resource=resources[1], hard_limit=hard_limits[1])
quotas = [quota_1, quota_2]
mock_read.side_effect = lambda *args: utils.FakeEtcdMultipleResult(
[quota_1.as_dict(), quota_2.as_dict()])
res = dbapi.quota_get_all_by_project(self.context, prj_id)
self.assertEqual([q.resource for q in res],
[q.resource for q in quotas])
self.assertEqual([q.hard_limit for q in res],
[q.hard_limit for q in quotas])
@mock.patch.object(etcd_client, 'read')
@mock.patch.object(etcd_client, 'write')
@mock.patch.object(etcd_client, 'delete')
@mock.patch.object(dbapi, '_get_dbdriver_instance')
def test_destroy_quota(self, mock_db_inst, mock_delete,
mock_write, mock_read):
mock_db_inst.return_value = etcdapi.get_backend()
mock_read.side_effect = etcd.EtcdKeyNotFound
quota = utils.create_test_quota(context=self.context)
mock_read.side_effect = lambda *args: utils.FakeEtcdResult(
quota.as_dict())
dbapi.quota_destroy(
self.context, quota.project_id, quota.resource)
mock_delete.assert_called_once_with(
'/quotas/{}/{}' . format(quota.project_id, quota.resource))
@mock.patch.object(etcd_client, 'read')
@mock.patch.object(etcd_client, 'write')
@mock.patch.object(etcd_client, 'update')
@mock.patch.object(dbapi, '_get_dbdriver_instance')
def test_update_quota(self, mock_db_inst, mock_update,
mock_write, mock_read):
mock_db_inst.return_value = etcdapi.get_backend()
mock_read.side_effect = etcd.EtcdKeyNotFound
quota = utils.create_test_quota(context=self.context)
new_hard_limit = 60
mock_read.side_effect = lambda *args: utils.FakeEtcdResult(
quota.as_dict())
dbapi.quota_update(self.context, quota.project_id, quota.resource,
new_hard_limit)
self.assertEqual(new_hard_limit,
json.loads(mock_update.call_args_list[0][0][0].
value.decode('utf-8'))['hard_limit'])

6
zun/tests/unit/db/utils.py
View File

@ -522,7 +522,8 @@ def get_test_quota_value(**kwargs):
'id': kwargs.get('id', 123),
'project_id': kwargs.get('project_id', 'fake_project_id'),
'resource': kwargs.get('resource', 'container'),
'hard_limit': kwargs.get('hard_limit', 20)
'hard_limit': kwargs.get('hard_limit', 20),
'uuid': kwargs.get('uuid', 'z2b96c5f-242a-41a0-a736-b6e1fada071b'),
}
return quota_values
@ -544,7 +545,8 @@ def get_test_quota_class_value(**kwargs):
'id': kwargs.get('id', 123),
'class_name': kwargs.get('class_name', 'fake_class_name'),
'resource': kwargs.get('resource', 'container'),
'hard_limit': kwargs.get('hard_limit', 20)
'hard_limit': kwargs.get('hard_limit', 20),
'uuid': kwargs.get('uuid', 'z2b96c5b-242a-41a0-a736-b6e1fada071b'),
}
return quota_values

4
zun/tests/unit/objects/test_objects.py
View File

@ -358,8 +358,8 @@ object_data = {
'ComputeNode': '1.11-08be22db017745f4f0bc8f873eca7db0',
'PciDevicePool': '1.0-3f5ddc3ff7bfa14da7f6c7e9904cc000',
'PciDevicePoolList': '1.0-15ecf022a68ddbb8c2a6739cfc9f8f5e',
'Quota': '1.0-4daf54427ac19cb23182cad82fcde751',
'QuotaClass': '1.0-4739583a70891fbc145031228fb8001e',
'Quota': '1.1-627da82e1289d7a4df8db759e1df132b',
'QuotaClass': '1.1-239ae335b32036b86504684d3fdbeb7f',
'ContainerPCIRequest': '1.0-b060f9f9f734bedde79a71a4d3112ee0',
'ContainerPCIRequests': '1.0-7b8f7f044661fe4e24e6949c035af2c4',
'ContainerAction': '1.1-b0c721f9e10c6c0d1e41e512c49eb877',

4
zun/tests/unit/objects/test_quota_class.py
View File

@ -12,7 +12,7 @@
import mock
from zun.db.sqlalchemy import api
from zun.common import consts
from zun import objects
from zun.tests.unit.db import base
from zun.tests.unit.db import utils
@ -37,7 +37,7 @@ class TestQuotaClassObject(base.DbTestCase):
self.assertEqual(self.context, quota_class._context)
def test_get_all_with_default(self):
class_name = api._DEFAULT_QUOTA_NAME
class_name = consts.DEFAULT_QUOTA_CLASS_NAME
with mock.patch.object(self.dbapi, 'quota_class_get_default',
autospec=True) as mock_get_all:
mock_get_all.return_value = {