openstack/zaqar
Code Issues Proposed changes

CONF file structure refactor

Browse Source 
This is the openstack governance goal that the config files should
be merged into one folder to make the code to be better understood
and managed.

Change-Id: I33dd85145d9f5b0384323ffa51b4d68de0aeb5b7
This commit is contained in:
xywang 2018-04-28 15:36:38 +08:00
parent c214c608e3
commit a7df08deb9
55 changed files with 1573 additions and 732 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
doc/source/conf.py
View File

@ -48,10 +48,9 @@ extensions = ['sphinx.ext.coverage',
'openstackdocstheme',
]
config_generator_config_file = [
('../../etc/oslo-config-generator/zaqar.conf',
'_static/zaqar'),
]
config_generator_config_file = '../../etc/oslo-config-generator/zaqar.conf'
sample_config_basename = '_static/zaqar'
# autodoc generation is a bit aggressive and a nuisance
# when doing heavy text edit cycles. Execute "export SPHINX_DEBUG=1"

1
doc/source/configuration/zaqar.rst
View File

@ -6,4 +6,3 @@ zaqar.conf
.. show-options::
:config-file: etc/oslo-config-generator/zaqar.conf

2
doc/source/install/install-obs.rst
View File

@ -278,7 +278,7 @@ Install and configure ``memcached``, ``uWSGI`` and Messaging on the web server
.. code-block:: console
# oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf
# oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf
# cp etc/zaqar.policy.yaml.sample /etc/zaqar/policy.yaml
Edit any item as needed in policy.yaml.

2
doc/source/install/install-rdo.rst
View File

@ -278,7 +278,7 @@ Install and configure ``memcached``, ``uWSGI`` and Messaging on the web server
.. code-block:: console
# oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf
# oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf
# cp etc/zaqar.policy.yaml.sample /etc/zaqar/policy.yaml
Edit any item as needed in policy.yaml.

2
doc/source/install/install-ubuntu.rst
View File

@ -266,7 +266,7 @@ Install and configure ``memcached``, ``uWSGI`` and Messaging on the web server
.. code-block:: console
# oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf
# oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf
# cp etc/zaqar.policy.yaml.sample /etc/zaqar/policy.yaml
Edit any item as needed in policy.yaml.

2
etc/README-policy.json.sample
View File

@ -3,6 +3,6 @@ level of the zaqar directory:
tox -egenpolicy
Or run the command directly:
oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf
oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf
The output file will be in etc folder, named "zaqar.policy.yaml.sample" by
default.

15
etc/oslo-config-generator/zaqar.conf
View File

@ -1,18 +1,7 @@
[DEFAULT]
wrap_width = 80
wrap_width = 79
output_file = etc/zaqar.conf.sample
namespace = zaqar.common.auth
namespace = zaqar.common.configs
namespace = zaqar.storage.pipeline
namespace = zaqar.storage.pooling
namespace = zaqar.storage.mongodb
namespace = zaqar.storage.redis
namespace = zaqar.storage.sqlalchemy
namespace = zaqar.storage.swift
namespace = zaqar.transport.wsgi
namespace = zaqar.transport.websocket
namespace = zaqar.transport.base
namespace = zaqar.transport.validation
namespace = zaqar
namespace = keystonemiddleware.auth_token
namespace = oslo.cache
namespace = oslo.log

0
etc/oslo-config-generator/zaqar-policy-generator.conf → etc/zaqar-policy-generator.conf
View File

6
releasenotes/notes/configuration-refactor-0ff219ac59c96347.yaml Normal file
View File

@ -0,0 +1,6 @@
other:
- |
The code structure for configuration files are changed. This is insensitvie
for end users, but the persons who work for downstream changes should pay
attention. Please refactor your private configurations to ``zaqar/conf/``
folder as well.

13
setup.cfg
View File

@ -46,18 +46,7 @@ zaqar.transport =
websocket = zaqar.transport.websocket.driver:Driver
oslo.config.opts =
zaqar.common.auth = zaqar.common.auth:_config_options
zaqar.common.configs = zaqar.common.configs:_config_options
zaqar.storage.pipeline = zaqar.storage.pipeline:_config_options
zaqar.storage.pooling = zaqar.storage.pooling:_config_options
zaqar.storage.mongodb = zaqar.storage.mongodb.options:_config_options
zaqar.storage.redis = zaqar.storage.redis.options:_config_options
zaqar.storage.sqlalchemy = zaqar.storage.sqlalchemy.options:_config_options
zaqar.storage.swift = zaqar.storage.swift.options:_config_options
zaqar.transport.wsgi = zaqar.transport.wsgi.driver:_config_options
zaqar.transport.websocket = zaqar.transport.websocket.driver:_config_options
zaqar.transport.base = zaqar.transport.base:_config_options
zaqar.transport.validation = zaqar.transport.validation:_config_options
zaqar = zaqar.conf.opts:list_opts
zaqar.storage.stages =
zaqar.notification.notifier = zaqar.notification.notifier:NotifierDriver

2
tox.ini
View File

@ -32,7 +32,7 @@ commands =
[testenv:genpolicy]
commands =
oslopolicy-sample-generator --config-file etc/oslo-config-generator/zaqar-policy-generator.conf
oslopolicy-sample-generator --config-file etc/zaqar-policy-generator.conf
[testenv:cover]
commands =

7
zaqar/bootstrap.py
View File

@ -22,10 +22,11 @@ from stevedore import driver
from zaqar.api import handler
from zaqar.common import cache as oslo_cache
from zaqar.common import configs
from zaqar.common import consts
from zaqar.common import decorators
from zaqar.common import errors
from zaqar.conf import drivers as driver_opts
from zaqar.conf import opts as opts_tool
from zaqar.storage import pipeline
from zaqar.storage import pooling
from zaqar.storage import utils as storage_utils
@ -46,7 +47,7 @@ class Bootstrap(object):
def __init__(self, conf):
self.conf = conf
for group, opts in configs._config_options():
for group, opts in opts_tool.list_opts_by_group():
self.conf.register_opts(opts, group=group)
profiler_opts.set_defaults(self.conf)
@ -54,7 +55,7 @@ class Bootstrap(object):
# wsgi. Websocket part will be added in the future.
profile.setup(self.conf, 'Zaqar-server', socket.gethostname())
self.driver_conf = self.conf[configs._DRIVER_GROUP]
self.driver_conf = self.conf[driver_opts.GROUP_NAME]
@decorators.lazy_property(write=False)
def api(self):

4
zaqar/cmd/server.py
View File

@ -21,12 +21,12 @@ from oslo_reports import opts as gmr_opts
from zaqar import bootstrap
from zaqar.common import cli
from zaqar.common import configs
from zaqar.conf import default
from zaqar import version
# NOTE(eggmaster): define command line options for zaqar-server
_CLI_OPTIONS = (
configs._ADMIN_MODE_OPT,
default.admin_mode,
cfg.BoolOpt('daemon', default=False,
help='Run Zaqar server in the background.'),
)

129
zaqar/common/configs.py
View File

@ -1,129 +0,0 @@
# Copyright (c) 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
_ADMIN_MODE_OPT = cfg.BoolOpt('admin_mode', default=False,
help='Activate privileged endpoints.')
_GENERAL_OPTIONS = (
_ADMIN_MODE_OPT,
cfg.BoolOpt('pooling', default=False,
help=('Enable pooling across multiple storage backends. '
'If pooling is enabled, the storage driver '
'configuration is used to determine where the '
'catalogue/control plane data is kept.'),
deprecated_opts=[cfg.DeprecatedOpt('sharding')]),
cfg.BoolOpt('unreliable', default=False,
help='Disable all reliability constraints.'),
cfg.ListOpt('enable_deprecated_api_versions', default=[],
item_type=cfg.types.List(item_type=cfg.types.String(
choices=('1', '1.1'))),
help='List of deprecated API versions to enable.'),
cfg.BoolOpt('enable_checksum', default=False,
help='Enable a checksum for message body. The default value '
'is False.'),
)
_DRIVER_OPTIONS = (
cfg.StrOpt('transport', default='wsgi',
help='Transport driver to use.'),
cfg.StrOpt('message_store', default='mongodb',
deprecated_opts=[cfg.DeprecatedOpt('storage')],
help='Storage driver to use as the messaging store.'),
cfg.StrOpt('management_store', default='mongodb',
help='Storage driver to use as the management store.'),
)
_DRIVER_GROUP = 'drivers'
_SIGNED_URL_OPTIONS = (
cfg.StrOpt('secret_key',
help=('Secret key used to encrypt pre-signed URLs.')),
)
_SIGNED_URL_GROUP = 'signed_url'
_NOTIFICATION_OPTIONS = (
cfg.StrOpt('smtp_command', default='/usr/sbin/sendmail -t -oi',
help=('The command of smtp to send email. The format is '
'"command_name arg1 arg2".')),
cfg.IntOpt('max_notifier_workers', default=10,
help='The max amount of the notification workers.'),
cfg.BoolOpt('require_confirmation', default=False,
help='Whether the http/https/email subscription need to be '
'confirmed before notification.'),
cfg.StrOpt('external_confirmation_url',
help='The confirmation page url that will be used in email '
'subscription confirmation before notification.'),
cfg.DictOpt("subscription_confirmation_email_template",
default={'topic': 'Zaqar Notification - Subscription '
'Confirmation',
'body': 'You have chosen to subscribe to the '
'queue: {0}. This queue belongs to '
'project: {1}. '
'To confirm this subscription, '
'click or visit this link below: {2}',
'sender': 'Zaqar Notifications '
'<no-reply@openstack.org>'},
help="Defines the set of subscription confirmation email "
"content, including topic, body and sender. There is "
"a mapping is {0} -> queue name, {1} ->project id, "
"{2}-> confirm url in body string. User can use any of "
"the three value. But they can't use more than three."),
cfg.DictOpt("unsubscribe_confirmation_email_template",
default={'topic': 'Zaqar Notification - '
'Unsubscribe Confirmation',
'body': 'You have unsubscribed successfully to the '
'queue: {0}. This queue belongs to '
'project: {1}. '
'To resubscribe this subscription, '
'click or visit this link below: {2}',
'sender': 'Zaqar Notifications '
'<no-reply@openstack.org>'},
help="Defines the set of unsubscribe confirmation email "
"content, including topic, body and sender. There is "
"a mapping is {0} -> queue name, {1} ->project id, "
"{2}-> confirm url in body string. User can use any of "
"the three value. But they can't use more than three."),
)
_NOTIFICATION_GROUP = 'notification'
_PROFILER_OPTIONS = [
cfg.BoolOpt("trace_wsgi_transport", default=False,
help="If False doesn't trace any transport requests."
"Please note that it doesn't work for websocket now."),
cfg.BoolOpt("trace_message_store", default=False,
help="If False doesn't trace any message store requests."),
cfg.BoolOpt("trace_management_store", default=False,
help="If False doesn't trace any management store requests.")
]
_PROFILER_GROUP = "profiler"
def _config_options():
return [(None, _GENERAL_OPTIONS),
(_DRIVER_GROUP, _DRIVER_OPTIONS),
(_SIGNED_URL_GROUP, _SIGNED_URL_OPTIONS),
(_NOTIFICATION_GROUP, _NOTIFICATION_OPTIONS),
(_PROFILER_GROUP, _PROFILER_OPTIONS)]

74
zaqar/conf/__init__.py Normal file
View File

@ -0,0 +1,74 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from oslo_cache import core as cache
from oslo_config import cfg
from oslo_log import log
from zaqar.conf import default
from zaqar.conf import drivers
from zaqar.conf import drivers_management_store_mongodb
from zaqar.conf import drivers_management_store_redis
from zaqar.conf import drivers_management_store_sqlalchemy
from zaqar.conf import drivers_message_store_mongodb
from zaqar.conf import drivers_message_store_redis
from zaqar.conf import drivers_message_store_swift
from zaqar.conf import drivers_transport_websocket
from zaqar.conf import drivers_transport_wsgi
from zaqar.conf import notification
from zaqar.conf import pooling_catalog
from zaqar.conf import profiler
from zaqar.conf import signed_url
from zaqar.conf import storage
from zaqar.conf import transport
CONF = cfg.CONF
conf_modules = [
default,
drivers,
drivers_management_store_mongodb,
drivers_management_store_redis,
drivers_management_store_sqlalchemy,
drivers_message_store_mongodb,
drivers_message_store_redis,
drivers_message_store_swift,
drivers_transport_websocket,
drivers_transport_wsgi,
notification,
pooling_catalog,
profiler,
signed_url,
storage,
transport
]
def setup_logging():
"""Set up logging for the keystone package."""
log.setup(CONF, 'zaqar')
logging.captureWarnings(True)
def configure(conf=None):
if conf is None:
conf = CONF
for module in conf_modules:
module.register_opts(conf)
# add oslo.cache related config options
cache.configure(conf)

70
zaqar/conf/default.py Normal file
View File

@ -0,0 +1,70 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
admin_mode = cfg.BoolOpt(
'admin_mode', default=False,
help='Activate privileged endpoints.')
pooling = cfg.BoolOpt(
'pooling', default=False,
help=('Enable pooling across multiple storage backends. '
'If pooling is enabled, the storage driver '
'configuration is used to determine where the '
'catalogue/control plane data is kept.'),
deprecated_opts=[cfg.DeprecatedOpt('sharding')])
unreliable = cfg.BoolOpt(
'unreliable', default=False,
help='Disable all reliability constraints.')
enable_deprecated_api_versions = cfg.ListOpt(
'enable_deprecated_api_versions', default=[],
item_type=cfg.types.List(item_type=cfg.types.String(choices=('1', '1.1'))),
help='List of deprecated API versions to enable.')
enable_checksum = cfg.BoolOpt(
'enable_checksum', default=False,
help='Enable a checksum for message body. The default value is False.')
auth_strategy = cfg.StrOpt(
'auth_strategy', default='',
help=('Backend to use for authentication. '
'For no auth, keep it empty. '
'Existing strategies: keystone. '
'See also the keystone_authtoken section below'))
GROUP_NAME = 'DEFAULT'
ALL_OPTS = [
admin_mode,
pooling,
unreliable,
enable_deprecated_api_versions,
enable_checksum,
auth_strategy
]
def register_opts(conf):
conf.register_opts(ALL_OPTS)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

46
zaqar/conf/drivers.py Normal file
View File

@ -0,0 +1,46 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
transport = cfg.StrOpt(
'transport', default='wsgi',
help='Transport driver to use.')
message_store = cfg.StrOpt(
'message_store', default='mongodb',
deprecated_opts=[cfg.DeprecatedOpt('storage')],
help='Storage driver to use as the messaging store.')
management_store = cfg.StrOpt(
'management_store', default='mongodb',
help='Storage driver to use as the management store.')
GROUP_NAME = 'drivers'
ALL_OPTS = [
transport,
message_store,
management_store
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

157
zaqar/conf/drivers_management_store_mongodb.py Normal file
View File

@ -0,0 +1,157 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
_deprecated_group = 'drivers:storage:mongodb'
ssl_keyfile = cfg.StrOpt(
'ssl_keyfile',
deprecated_opts=[cfg.DeprecatedOpt(
'ssl_keyfile',
group=_deprecated_group), ],
help='The private keyfile used to identify the local '
'connection against mongod. If included with '
'the ``certifle`` then only the ``ssl_certfile``'
' is needed.')
ssl_certfile = cfg.StrOpt(
'ssl_certfile',
deprecated_opts=[cfg.DeprecatedOpt(
'ssl_certfile',
group=_deprecated_group), ],
help='The certificate file used to identify the '
'local connection against mongod.')
ssl_cert_reqs = cfg.StrOpt(
'ssl_cert_reqs', default='CERT_REQUIRED',
deprecated_opts=[cfg.DeprecatedOpt(
'ssl_cert_reqs',
group=_deprecated_group), ],
help='Specifies whether a certificate is required '
'from the other side of the connection, and '
'whether it will be validated if provided. It '
'must be one of the three values ``CERT_NONE``'
'(certificates ignored), ``CERT_OPTIONAL``'
'(not required, but validated if provided), or'
' ``CERT_REQUIRED``(required and validated). '
'If the value of this parameter is not '
'``CERT_NONE``, then the ``ssl_ca_cert`` '
'parameter must point to a file of CA '
'certificates.')
ssl_ca_certs = cfg.StrOpt(
'ssl_ca_certs',
deprecated_opts=[cfg.DeprecatedOpt(
'ssl_ca_certs',
group=_deprecated_group), ],
help='The ca_certs file contains a set of concatenated '
'"certification authority" certificates, which are '
'used to validate certificates passed from the other '
'end of the connection.')
uri = cfg.StrOpt(
'uri',
deprecated_opts=[cfg.DeprecatedOpt(
'uri',
group=_deprecated_group), ],
help='Mongodb Connection URI. If ssl connection enabled, '
'then ``ssl_keyfile``, ``ssl_certfile``, '
'``ssl_cert_reqs``, ``ssl_ca_certs`` need to be set '
'accordingly.')
database = cfg.StrOpt(
'database', default='zaqar',
deprecated_opts=[cfg.DeprecatedOpt(
'database',
group=_deprecated_group), ],
help='Database name.')
max_attempts = cfg.IntOpt(
'max_attempts', min=0, default=1000,
deprecated_opts=[cfg.DeprecatedOpt(
'max_attempts',
group=_deprecated_group), ],
help=('Maximum number of times to retry a failed operation. '
'Currently only used for retrying a message post.'))
max_retry_sleep = cfg.FloatOpt(
'max_retry_sleep', default=0.1,
deprecated_opts=[cfg.DeprecatedOpt(
'max_retry_sleep',
group=_deprecated_group), ],
help=('Maximum sleep interval between retries '
'(actual sleep time increases linearly '
'according to number of attempts performed).'))
max_retry_jitter = cfg.FloatOpt(
'max_retry_jitter', default=0.005,
deprecated_opts=[cfg.DeprecatedOpt(
'max_retry_jitter',
group=_deprecated_group), ],
help=('Maximum jitter interval, to be added to the '
'sleep interval, in order to decrease probability '
'that parallel requests will retry at the '
'same instant.'))
max_reconnect_attempts = cfg.IntOpt(
'max_reconnect_attempts', default=10,
deprecated_opts=[cfg.DeprecatedOpt(
'max_reconnect_attempts',
group=_deprecated_group), ],
help=('Maximum number of times to retry an operation that '
'failed due to a primary node failover.'))
reconnect_sleep = cfg.FloatOpt(
'reconnect_sleep', default=0.020,
deprecated_opts=[cfg.DeprecatedOpt(
'reconnect_sleep',
group=_deprecated_group), ],
help=('Base sleep interval between attempts to reconnect '
'after a primary node failover. '
'The actual sleep time increases exponentially (power '
'of 2) each time the operation is retried.'))
GROUP_NAME = 'drivers:management_store:mongodb'
ALL_OPTS = [
ssl_keyfile,
ssl_certfile,
ssl_cert_reqs,
ssl_ca_certs,
uri,
database,
max_attempts,
max_retry_sleep,
max_retry_jitter,
max_reconnect_attempts,
reconnect_sleep
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

86
zaqar/conf/drivers_management_store_redis.py Normal file
View File

@ -0,0 +1,86 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
_deprecated_group = 'drivers:storage:redis'
uri = cfg.StrOpt(
'uri', default="redis://127.0.0.1:6379",
deprecated_opts=[cfg.DeprecatedOpt(
'uri',
group=_deprecated_group), ],
help=('Redis connection URI, taking one of three forms. '
'For a direct connection to a Redis server, use '
'the form "redis://[:password]@host[:port][?options]", '
'where password is redis-server\'s password, when'
'redis-server is set password, the password option'
'needs to be set. port defaults to 6379 if not'
'specified. For an HA master-slave Redis cluster using'
' Redis Sentinel, use the form '
'"redis://[:password]@host1[:port1]'
'[,host2[:port2],...,hostN[:portN]][?options]", '
'where each host specified corresponds to an '
'instance of redis-sentinel. In this form, the '
'name of the Redis master used in the Sentinel '
'configuration must be included in the query '
'string as "master=<name>". Finally, to connect '
'to a local instance of Redis over a unix socket, '
'you may use the form '
'"redis:[:password]@/path/to/redis.sock[?options]".'
' In all forms, the "socket_timeout" option may be'
'specified in the query string. Its value is '
'given in seconds. If not provided, '
'"socket_timeout" defaults to 0.1 seconds.'
'There are multiple database instances in redis '
'database, for example in the /etc/redis/redis.conf, '
'if the parameter is "database 16", there are 16 '
'database instances. By default, the data is stored '
'in db = 0 database, if you want to use db = 1 '
'database, you can use the following form: '
'"redis://host[:port][?dbid=1]".'))
max_reconnect_attempts = cfg.IntOpt(
'max_reconnect_attempts', default=10,
deprecated_opts=[cfg.DeprecatedOpt(
'max_reconnect_attempts',
group=_deprecated_group), ],
help=('Maximum number of times to retry an operation that '
'failed due to a redis node failover.'))
reconnect_sleep = cfg.FloatOpt(
'reconnect_sleep', default=1.0,
deprecated_opts=[cfg.DeprecatedOpt(
'reconnect_sleep',
group=_deprecated_group), ],
help=('Base sleep interval between attempts to reconnect '
'after a redis node failover. '))
GROUP_NAME = 'drivers:management_store:redis'
ALL_OPTS = [
uri,
max_reconnect_attempts,
reconnect_sleep
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

38
zaqar/conf/drivers_management_store_sqlalchemy.py Normal file
View File

@ -0,0 +1,38 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
_deprecated_group = 'drivers:storage:sqlalchemy'
uri = cfg.StrOpt(
'uri', default='sqlite:///:memory:',
deprecated_opts=[cfg.DeprecatedOpt(
'uri',
group=_deprecated_group), ],
help='An sqlalchemy URL')
GROUP_NAME = 'drivers:management_store:sqlalchemy'
ALL_OPTS = [
uri
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

173
zaqar/conf/drivers_message_store_mongodb.py Normal file
View File

@ -0,0 +1,173 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
_deprecated_group = 'drivers:storage:mongodb'
ssl_keyfile = cfg.StrOpt(
'ssl_keyfile',
deprecated_opts=[cfg.DeprecatedOpt(
'ssl_keyfile',
group=_deprecated_group), ],
help='The private keyfile used to identify the local '
'connection against mongod. If included with '
'the ``certifle`` then only the ``ssl_certfile``'
' is needed.')
ssl_certfile = cfg.StrOpt(
'ssl_certfile',
deprecated_opts=[cfg.DeprecatedOpt(
'ssl_certfile',
group=_deprecated_group), ],
help='The certificate file used to identify the '
'local connection against mongod.')
ssl_cert_reqs = cfg.StrOpt(
'ssl_cert_reqs', default='CERT_REQUIRED',
deprecated_opts=[cfg.DeprecatedOpt(
'ssl_cert_reqs',
group=_deprecated_group), ],
help='Specifies whether a certificate is required '
'from the other side of the connection, and '
'whether it will be validated if provided. It '
'must be one of the three values ``CERT_NONE``'
'(certificates ignored), ``CERT_OPTIONAL``'
'(not required, but validated if provided), or'
' ``CERT_REQUIRED``(required and validated). '
'If the value of this parameter is not '
'``CERT_NONE``, then the ``ssl_ca_cert`` '
'parameter must point to a file of CA '
'certificates.')
ssl_ca_certs = cfg.StrOpt(
'ssl_ca_certs',
deprecated_opts=[cfg.DeprecatedOpt(
'ssl_ca_certs',
group=_deprecated_group), ],
help='The ca_certs file contains a set of concatenated '
'"certification authority" certificates, which are '
'used to validate certificates passed from the other '
'end of the connection.')
uri = cfg.StrOpt(
'uri',
deprecated_opts=[cfg.DeprecatedOpt(
'uri',
group=_deprecated_group), ],
help='Mongodb Connection URI. If ssl connection enabled, '
'then ``ssl_keyfile``, ``ssl_certfile``, '
'``ssl_cert_reqs``, ``ssl_ca_certs`` need to be set '
'accordingly.')
database = cfg.StrOpt(
'database', default='zaqar',
deprecated_opts=[cfg.DeprecatedOpt(
'database',
group=_deprecated_group), ],
help='Database name.')
max_attempts = cfg.IntOpt(
'max_attempts', min=0, default=1000,
deprecated_opts=[cfg.DeprecatedOpt(
'max_attempts',
group=_deprecated_group), ],
help=('Maximum number of times to retry a failed operation. '
'Currently only used for retrying a message post.'))
max_retry_sleep = cfg.FloatOpt(
'max_retry_sleep', default=0.1,
deprecated_opts=[cfg.DeprecatedOpt(
'max_retry_sleep',
group=_deprecated_group), ],
help=('Maximum sleep interval between retries '
'(actual sleep time increases linearly '
'according to number of attempts performed).'))
max_retry_jitter = cfg.FloatOpt(
'max_retry_jitter', default=0.005,
deprecated_opts=[cfg.DeprecatedOpt(
'max_retry_jitter',
group=_deprecated_group), ],
help=('Maximum jitter interval, to be added to the '
'sleep interval, in order to decrease probability '
'that parallel requests will retry at the '
'same instant.'))
max_reconnect_attempts = cfg.IntOpt(
'max_reconnect_attempts', default=10,
deprecated_opts=[cfg.DeprecatedOpt(
'max_reconnect_attempts',
group=_deprecated_group), ],
help=('Maximum number of times to retry an operation that '
'failed due to a primary node failover.'))
reconnect_sleep = cfg.FloatOpt(
'reconnect_sleep', default=0.020,
deprecated_opts=[cfg.DeprecatedOpt(
'reconnect_sleep',
group=_deprecated_group), ],
help=('Base sleep interval between attempts to reconnect '
'after a primary node failover. '
'The actual sleep time increases exponentially (power '
'of 2) each time the operation is retried.'))
partitions = cfg.IntOpt(
'partitions', default=2,
deprecated_opts=[cfg.DeprecatedOpt(
'partitions',
group=_deprecated_group), ],
help=('Number of databases across which to '
'partition message data, in order to '
'reduce writer lock %. DO NOT change '
'this setting after initial deployment. '
'It MUST remain static. Also, you '
'should not need a large number of partitions '
'to improve performance, esp. if deploying '
'MongoDB on SSD storage.'))
GROUP_NAME = 'drivers:message_store:mongodb'
ALL_OPTS = [
ssl_keyfile,
ssl_certfile,
ssl_cert_reqs,
ssl_ca_certs,
uri,
database,
max_attempts,
max_retry_sleep,
max_retry_jitter,
max_reconnect_attempts,
reconnect_sleep,
partitions
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

86
zaqar/conf/drivers_message_store_redis.py Normal file
View File

@ -0,0 +1,86 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
_deprecated_group = 'drivers:storage:redis'
uri = cfg.StrOpt(
'uri', default="redis://127.0.0.1:6379",
deprecated_opts=[cfg.DeprecatedOpt(
'uri',
group=_deprecated_group), ],
help=('Redis connection URI, taking one of three forms. '
'For a direct connection to a Redis server, use '
'the form "redis://[:password]@host[:port][?options]", '
'where password is redis-server\'s password, when'
'redis-server is set password, the password option'
'needs to be set. port defaults to 6379 if not'
'specified. For an HA master-slave Redis cluster using'
' Redis Sentinel, use the form '
'"redis://[:password]@host1[:port1]'
'[,host2[:port2],...,hostN[:portN]][?options]", '
'where each host specified corresponds to an '
'instance of redis-sentinel. In this form, the '
'name of the Redis master used in the Sentinel '
'configuration must be included in the query '
'string as "master=<name>". Finally, to connect '
'to a local instance of Redis over a unix socket, '
'you may use the form '
'"redis:[:password]@/path/to/redis.sock[?options]".'
' In all forms, the "socket_timeout" option may be'
'specified in the query string. Its value is '
'given in seconds. If not provided, '
'"socket_timeout" defaults to 0.1 seconds.'
'There are multiple database instances in redis '
'database, for example in the /etc/redis/redis.conf, '
'if the parameter is "database 16", there are 16 '
'database instances. By default, the data is stored '
'in db = 0 database, if you want to use db = 1 '
'database, you can use the following form: '
'"redis://host[:port][?dbid=1]".'))
max_reconnect_attempts = cfg.IntOpt(
'max_reconnect_attempts', default=10,
deprecated_opts=[cfg.DeprecatedOpt(
'max_reconnect_attempts',
group=_deprecated_group), ],
help=('Maximum number of times to retry an operation that '
'failed due to a redis node failover.'))
reconnect_sleep = cfg.FloatOpt(
'reconnect_sleep', default=1.0,
deprecated_opts=[cfg.DeprecatedOpt(
'reconnect_sleep',
group=_deprecated_group), ],
help=('Base sleep interval between attempts to reconnect '
'after a redis node failover. '))
GROUP_NAME = 'drivers:message_store:redis'
ALL_OPTS = [
uri,
max_reconnect_attempts,
reconnect_sleep
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

80
zaqar/conf/drivers_message_store_swift.py Normal file
View File

@ -0,0 +1,80 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
auth_url = cfg.StrOpt(
"auth_url", default="http://127.0.0.1:5000/v3/",
help="URI of Keystone endpoint to discover Swift")
uri = cfg.StrOpt(
"uri",
default="swift://demo:nomoresecrete@/demo",
help="Custom URI describing the swift connection.")
insecure = cfg.StrOpt(
"insecure", default=False,
help="Don't check SSL certificate")
project_domain_id = cfg.StrOpt(
"project_domain_id", default="default",
help="Domain ID containing project")
project_domain_name = cfg.StrOpt(
"project_domain_name",
help="Domain name containing project")
user_domain_id = cfg.StrOpt(
"user_domain_id", default="default",
help="User's domain id")
user_domain_name = cfg.StrOpt(
"user_domain_name", help="User's domain name")
region_name = cfg.StrOpt(
"region_name", help="Region name")
interface = cfg.StrOpt(
"interface", default="publicURL",
help="The default interface for endpoint URL "
"discovery.")
GROUP_NAME = 'drivers:message_store:swift'
ALL_OPTS = [
auth_url,
uri,
insecure,
project_domain_id,
project_domain_name,
user_domain_id,
user_domain_name,
region_name,
interface
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

59
zaqar/conf/drivers_transport_websocket.py Normal file
View File

@ -0,0 +1,59 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
bind = cfg.HostAddressOpt(
'bind', default='127.0.0.1',
help='Address on which the self-hosting server will '
'listen.')
port = cfg.PortOpt(
'port', default=9000,
help='Port on which the self-hosting server will listen.')
external_port = cfg.PortOpt(
'external-port',
help='Port on which the service is provided to the user.')
notification_bind = cfg.HostAddressOpt(
'notification-bind',
help='Address on which the notification server will '
'listen.')
notification_port = cfg.PortOpt(
'notification-port', default=0,
help='Port on which the notification server will listen.')
GROUP_NAME = 'drivers:transport:websocket'
ALL_OPTS = [
bind,
port,
external_port,
notification_bind,
notification_port
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

39
zaqar/conf/drivers_transport_wsgi.py Normal file
View File

@ -0,0 +1,39 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
bind = cfg.HostAddressOpt(
'bind', default='127.0.0.1',
help='Address on which the self-hosting server will '
'listen.')
port = cfg.PortOpt(
'port', default=8888,
help='Port on which the self-hosting server will listen.')
GROUP_NAME = 'drivers:transport:wsgi'
ALL_OPTS = [
bind,
port
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

93
zaqar/conf/notification.py Normal file
View File

@ -0,0 +1,93 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
smtp_command = cfg.StrOpt(
'smtp_command', default='/usr/sbin/sendmail -t -oi',
help=(
'The command of smtp to send email. The format is '
'"command_name arg1 arg2".'))
max_notifier_workers = cfg.IntOpt(
'max_notifier_workers', default=10,
help='The max amount of the notification workers.')
require_confirmation = cfg.BoolOpt(
'require_confirmation', default=False,
help='Whether the http/https/email subscription need to be confirmed '
'before notification.')
external_confirmation_url = cfg.StrOpt(
'external_confirmation_url',
help='The confirmation page url that will be used in email subscription '
'confirmation before notification.')
subscription_confirmation_email_template = cfg.DictOpt(
"subscription_confirmation_email_template",
default={'topic': 'Zaqar Notification - Subscription '
'Confirmation',
'body': 'You have chosen to subscribe to the '
'queue: {0}. This queue belongs to '
'project: {1}. '
'To confirm this subscription, '
'click or visit this link below: {2}',
'sender': 'Zaqar Notifications '
'<no-reply@openstack.org>'},
help="Defines the set of subscription confirmation email content, "
"including topic, body and sender. There is a mapping is "
"{0} -> queue name, {1} ->project id, {2}-> confirm url in body "
"string. User can use any of the three values. But they can't use "
"more than three.")
unsubscribe_confirmation_email_template = cfg.DictOpt(
"unsubscribe_confirmation_email_template",
default={'topic': 'Zaqar Notification - '
'Unsubscribe Confirmation',
'body': 'You have unsubscribed successfully to the '
'queue: {0}. This queue belongs to '
'project: {1}. '
'To resubscribe this subscription, '
'click or visit this link below: {2}',
'sender': 'Zaqar Notifications '
'<no-reply@openstack.org>'},
help="Defines the set of unsubscribe confirmation email content, "
"including topic, body and sender. There is a mapping is "
"{0} -> queue name, {1} ->project id, {2}-> confirm url in body "
"string. User can use any of the three values. But they can't use "
"more than three.")
GROUP_NAME = 'notification'
ALL_OPTS = [
smtp_command,
max_notifier_workers,
require_confirmation,
external_confirmation_url,
subscription_confirmation_email_template,
unsubscribe_confirmation_email_template
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

109
zaqar/conf/opts.py Normal file
View File

@ -0,0 +1,109 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Single point of entry to generate the sample configuration file.
This module collects all the necessary info from the other modules in this
package. It is assumed that:
* Every other module in this package has a 'list_opts' function which
returns a dict where:
* The keys are strings which are the group names.
* The value of each key is a list of config options for that group.
* The conf package doesn't have further packages with config options.
* This module is only used in the context of sample file generation.
"""
import collections
import importlib
import os
import pkgutil
LIST_OPTS_FUNC_NAME = 'list_opts'
IGNORED_MODULES = ('opts', 'constants', 'utils')
def list_opts():
opts = collections.defaultdict(list)
module_names = _list_module_names()
imported_modules = _import_modules(module_names)
_append_config_options(imported_modules, opts)
return _tupleize(opts)
def list_opts_by_group():
opts = []
module_names = _list_module_names()
imported_modules = _import_modules(module_names)
for module in imported_modules:
configs = module.list_opts()
group_name = (module.GROUP_NAME if module.GROUP_NAME != 'DEFAULT'
else None)
opts.append((group_name, configs[module.GROUP_NAME]))
return opts
def _tupleize(d):
"""Convert a dict of options to the 2-tuple format."""
return [(key, value) for key, value in d.items()]
def _list_module_names():
module_names = []
package_path = os.path.dirname(os.path.abspath(__file__))
for _, module_name, ispkg in pkgutil.iter_modules(path=[package_path]):
if module_name in IGNORED_MODULES or ispkg:
# Skip this module.
continue
else:
module_names.append(module_name)
return module_names
def _import_modules(module_names):
imported_modules = []
for module_name in module_names:
full_module_path = '.'.join(__name__.split('.')[:-1] + [module_name])
module = importlib.import_module(full_module_path)
if not hasattr(module, LIST_OPTS_FUNC_NAME):
raise Exception(
"The module '%s' should have a '%s' function which "
"returns the config options." % (
full_module_path,
LIST_OPTS_FUNC_NAME))
else:
imported_modules.append(module)
return imported_modules
def _process_old_opts(configs):
"""Convert old-style 2-tuple configs to dicts."""
if isinstance(configs, tuple):
configs = [configs]
return {label: options for label, options in configs}
def _append_config_options(imported_modules, config_options):
for module in imported_modules:
configs = module.list_opts()
# TODO(markus_z): Remove this compatibility shim once all list_opts()
# functions have been updated to return dicts.
if not isinstance(configs, dict):
configs = _process_old_opts(configs)
for key, val in configs.items():
config_options[key].extend(val)

33
zaqar/conf/pooling_catalog.py Normal file
View File

@ -0,0 +1,33 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
enable_virtual_pool = cfg.BoolOpt(
'enable_virtual_pool', default=False,
help='If enabled, the message_store will be used as the storage for the '
'virtual pool.')
GROUP_NAME = 'pooling:catalog'
ALL_OPTS = [
enable_virtual_pool
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

45
zaqar/conf/profiler.py Normal file
View File

@ -0,0 +1,45 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
trace_wsgi_transport = cfg.BoolOpt(
"trace_wsgi_transport", default=False,
help="If False doesn't trace any transport requests."
"Please note that it doesn't work for websocket now.")
trace_message_store = cfg.BoolOpt(
"trace_message_store", default=False,
help="If False doesn't trace any message store requests.")
trace_management_store = cfg.BoolOpt(
"trace_management_store", default=False,
help="If False doesn't trace any management store requests.")
GROUP_NAME = 'profiler'
ALL_OPTS = [
trace_wsgi_transport,
trace_message_store,
trace_management_store
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

32
zaqar/conf/signed_url.py Normal file
View File

@ -0,0 +1,32 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
secret_key = cfg.StrOpt('secret_key',
help='Secret key used to encrypt pre-signed URLs.')
GROUP_NAME = 'signed_url'
ALL_OPTS = [
secret_key
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

61
zaqar/conf/storage.py Normal file
View File

@ -0,0 +1,61 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from zaqar.i18n import _
queue_pipeline = cfg.ListOpt(
'queue_pipeline', default=[],
help=_('Pipeline to use for processing queue operations. This pipeline '
'will be consumed before calling the storage driver\'s controller '
'methods.'))
message_pipeline = cfg.ListOpt(
'message_pipeline', default=[],
help=_('Pipeline to use for processing message operations. This pipeline '
'will be consumed before calling the storage driver\'s controller '
'methods.'))
claim_pipeline = cfg.ListOpt(
'claim_pipeline', default=[],
help=_('Pipeline to use for processing claim operations. This pipeline '
'will be consumed before calling the storage driver\'s controller '
'methods.'))
subscription_pipeline = cfg.ListOpt(
'subscription_pipeline', default=[],
help=_('Pipeline to use for processing subscription operations. This '
'pipeline will be consumed before calling the storage driver\'s '
'controller methods.'))
GROUP_NAME = 'storage'
ALL_OPTS = [
queue_pipeline,
message_pipeline,
claim_pipeline,
subscription_pipeline
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

155
zaqar/conf/transport.py Normal file
View File

@ -0,0 +1,155 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
default_message_ttl = cfg.IntOpt(
'default_message_ttl', default=3600,
help='Defines how long a message will be accessible.')
default_message_delay = cfg.IntOpt(
'default_message_delay', default=0,
help=('Defines the defautl value for queue delay seconds.'
'The 0 means the delayed queues feature is close.'))
default_claim_ttl = cfg.IntOpt(
'default_claim_ttl', default=300,
help='Defines how long a message will be in claimed state.')
default_claim_grace = cfg.IntOpt(
'default_claim_grace', default=60,
help='Defines the message grace period in seconds.')
default_subscription_ttl = cfg.IntOpt(
'default_subscription_ttl', default=3600,
help='Defines how long a subscription will be available.')
max_queues_per_page = cfg.IntOpt(
'max_queues_per_page', default=20,
deprecated_name='queue_paging_uplimit',
deprecated_group='limits:transport',
help='Defines the maximum number of queues per page.')
max_messages_per_page = cfg.IntOpt(
'max_messages_per_page', default=20,
deprecated_name='message_paging_uplimit',
deprecated_group='limits:transport',
help='Defines the maximum number of messages per page.')
max_subscriptions_per_page = cfg.IntOpt(
'max_subscriptions_per_page', default=20,
deprecated_name='subscription_paging_uplimit',
deprecated_group='limits:transport',
help='Defines the maximum number of subscriptions per page.')
max_messages_per_claim_or_pop = cfg.IntOpt(
'max_messages_per_claim_or_pop', default=20,
deprecated_name='max_messages_per_claim',
help='The maximum number of messages that can be claimed (OR) '
'popped in a single request')
max_queue_metadata = cfg.IntOpt(
'max_queue_metadata', default=64 * 1024,
deprecated_name='metadata_size_uplimit',
deprecated_group='limits:transport',
help='Defines the maximum amount of metadata in a queue.')
max_messages_post_size = cfg.IntOpt(
'max_messages_post_size', default=256 * 1024,
deprecated_name='message_size_uplimit',
deprecated_group='limits:transport',
deprecated_opts=[cfg.DeprecatedOpt('max_message_size')],
help='Defines the maximum size of message posts.')
max_message_ttl = cfg.IntOpt(
'max_message_ttl', default=1209600,
deprecated_name='message_ttl_max',
deprecated_group='limits:transport',
help='Maximum amount of time a message will be available.')
max_message_delay = cfg.IntOpt(
'max_message_delay', default=900,
help='Maximum delay seconds for messages can be claimed.')
max_claim_ttl = cfg.IntOpt(
'max_claim_ttl', default=43200,
deprecated_name='claim_ttl_max',
deprecated_group='limits:transport',
help='Maximum length of a message in claimed state.')
max_claim_grace = cfg.IntOpt(
'max_claim_grace', default=43200,
deprecated_name='claim_grace_max',
deprecated_group='limits:transport',
help='Defines the maximum message grace period in seconds.')
subscriber_types = cfg.ListOpt(
'subscriber_types', default=['http', 'https', 'mailto',
'trust+http', 'trust+https'],
help='Defines supported subscriber types.')
max_flavors_per_page = cfg.IntOpt(
'max_flavors_per_page', default=20,
help='Defines the maximum number of flavors per page.')
max_pools_per_page = cfg.IntOpt(
'max_pools_per_page', default=20,
help='Defines the maximum number of pools per page.')
GROUP_NAME = 'transport'
ALL_OPTS = [
default_message_ttl,
default_message_delay,
default_claim_ttl,
default_claim_grace,
default_subscription_ttl,
max_queues_per_page,
max_messages_per_page,
max_subscriptions_per_page,
max_messages_per_claim_or_pop,
max_queue_metadata,
max_messages_post_size,
max_message_ttl,
max_message_delay,
max_claim_ttl,
max_claim_grace,
subscriber_types,
max_flavors_per_page,
max_pools_per_page
]
def register_opts(conf):
conf.register_opts(ALL_OPTS, group=GROUP_NAME)
def list_opts():
return {GROUP_NAME: ALL_OPTS}

18
zaqar/storage/mongodb/driver.py
View File

@ -22,10 +22,11 @@ import pymongo
import pymongo.errors
from zaqar.common import decorators
from zaqar.conf import drivers_management_store_mongodb
from zaqar.conf import drivers_message_store_mongodb
from zaqar.i18n import _
from zaqar import storage
from zaqar.storage.mongodb import controllers
from zaqar.storage.mongodb import options
def _connection(conf):
@ -71,14 +72,17 @@ class DataDriver(storage.DataDriverBase):
BASE_CAPABILITIES = tuple(storage.Capabilities)
_DRIVER_OPTIONS = options._config_options()
_DRIVER_OPTIONS = [(drivers_management_store_mongodb.GROUP_NAME,
drivers_management_store_mongodb.ALL_OPTS),
(drivers_message_store_mongodb.GROUP_NAME,
drivers_message_store_mongodb.ALL_OPTS)]
_COL_SUFIX = "_messages_p"
def __init__(self, conf, cache, control_driver):
super(DataDriver, self).__init__(conf, cache, control_driver)
self.mongodb_conf = self.conf[options.MESSAGE_MONGODB_GROUP]
self.mongodb_conf = self.conf[drivers_message_store_mongodb.GROUP_NAME]
conn = self.connection
server_info = conn.server_info()['version']
@ -236,10 +240,12 @@ class ControlDriver(storage.ControlDriverBase):
def __init__(self, conf, cache):
super(ControlDriver, self).__init__(conf, cache)
self.conf.register_opts(options.MANAGEMENT_MONGODB_OPTIONS,
group=options.MANAGEMENT_MONGODB_GROUP)
self.conf.register_opts(
drivers_management_store_mongodb.ALL_OPTS,
group=drivers_management_store_mongodb.GROUP_NAME)
self.mongodb_conf = self.conf[options.MANAGEMENT_MONGODB_GROUP]
self.mongodb_conf = self.conf[
drivers_management_store_mongodb.GROUP_NAME]
def close(self):
self.connection.close()

147
zaqar/storage/mongodb/options.py
View File

@ -1,147 +0,0 @@
# Copyright (c) 2013 Rackspace Hosting, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""MongoDB storage driver configuration options."""
from oslo_config import cfg
_deprecated_group = 'drivers:storage:mongodb'
# options that are common to both management and message storage
_COMMON_OPTIONS = (
cfg.StrOpt('ssl_keyfile',
deprecated_opts=[cfg.DeprecatedOpt(
'ssl_keyfile',
group=_deprecated_group), ],
help=('The private keyfile used to identify the local '
'connection against mongod. If included with the '
'``certifle`` then only the ``ssl_certfile`` '
'is needed.')),
cfg.StrOpt('ssl_certfile',
deprecated_opts=[cfg.DeprecatedOpt(
'ssl_certfile',
group=_deprecated_group), ],
help=('The certificate file used to identify the local '
'connection against mongod.')),
cfg.StrOpt('ssl_cert_reqs', default='CERT_REQUIRED',
deprecated_opts=[cfg.DeprecatedOpt(
'ssl_cert_reqs',
group=_deprecated_group), ],
help=('Specifies whether a certificate is required from '
'the other side of the connection, and whether it '
'will be validated if provided. It must be one of '
'the three values ``CERT_NONE``(certificates ignored), '
'``CERT_OPTIONAL``(not required, but validated if '
'provided), or ``CERT_REQUIRED``(required and '
'validated). If the value of this parameter is not '
'``CERT_NONE``, then the ``ssl_ca_cert`` parameter '
'must point to a file of CA certificates.')),
cfg.StrOpt('ssl_ca_certs',
deprecated_opts=[cfg.DeprecatedOpt(
'ssl_ca_certs',
group=_deprecated_group), ],
help=('The ca_certs file contains a set of concatenated '
'"certification authority" certificates, which are '
'used to validate certificates passed from the other '
'end of the connection.')),
cfg.StrOpt('uri',
deprecated_opts=[cfg.DeprecatedOpt(
'uri',
group=_deprecated_group), ],
help=('Mongodb Connection URI. If ssl connection enabled, '
'then ``ssl_keyfile``, ``ssl_certfile``, '
'``ssl_cert_reqs``, ``ssl_ca_certs`` need to be set '
'accordingly.')),
cfg.StrOpt('database', default='zaqar',
deprecated_opts=[cfg.DeprecatedOpt(
'database',
group=_deprecated_group), ],
help='Database name.'),
cfg.IntOpt('max_attempts', min=0, default=1000,
deprecated_opts=[cfg.DeprecatedOpt(
'max_attempts',
group=_deprecated_group), ],
help=('Maximum number of times to retry a failed operation. '
'Currently only used for retrying a message post.')),
cfg.FloatOpt('max_retry_sleep', default=0.1,
deprecated_opts=[cfg.DeprecatedOpt(
'max_retry_sleep',
group=_deprecated_group), ],
help=('Maximum sleep interval between retries '
'(actual sleep time increases linearly '
'according to number of attempts performed).')),
cfg.FloatOpt('max_retry_jitter', default=0.005,
deprecated_opts=[cfg.DeprecatedOpt(
'max_retry_jitter',
group=_deprecated_group), ],
help=('Maximum jitter interval, to be added to the '
'sleep interval, in order to decrease probability '
'that parallel requests will retry at the '
'same instant.')),
cfg.IntOpt('max_reconnect_attempts', default=10,
deprecated_opts=[cfg.DeprecatedOpt(
'max_reconnect_attempts',
group=_deprecated_group), ],
help=('Maximum number of times to retry an operation that '
'failed due to a primary node failover.')),
cfg.FloatOpt('reconnect_sleep', default=0.020,
deprecated_opts=[cfg.DeprecatedOpt(
'reconnect_sleep',
group=_deprecated_group), ],
help=('Base sleep interval between attempts to reconnect '
'after a primary node failover. '
'The actual sleep time increases exponentially (power '
'of 2) each time the operation is retried.')),
)
MANAGEMENT_MONGODB_OPTIONS = _COMMON_OPTIONS
MESSAGE_MONGODB_OPTIONS = _COMMON_OPTIONS + (
# options used only by message_store
cfg.IntOpt('partitions', default=2,
deprecated_opts=[cfg.DeprecatedOpt(
'partitions',
group=_deprecated_group), ],
help=('Number of databases across which to '
'partition message data, in order to '
'reduce writer lock %. DO NOT change '
'this setting after initial deployment. '
'It MUST remain static. Also, you '
'should not need a large number of partitions '
'to improve performance, esp. if deploying '
'MongoDB on SSD storage.')),
)
MANAGEMENT_MONGODB_GROUP = 'drivers:management_store:mongodb'
MESSAGE_MONGODB_GROUP = 'drivers:message_store:mongodb'
def _config_options():
"""Used by config generators.
Returns a list of (group-name, oslo_config-options) tuples
for management and message storage.
"""
return [(MANAGEMENT_MONGODB_GROUP, MANAGEMENT_MONGODB_OPTIONS),
(MESSAGE_MONGODB_GROUP, MESSAGE_MONGODB_OPTIONS)]

26
zaqar/storage/pipeline.py
View File

@ -12,7 +12,6 @@
# License for the specific language governing permissions and limitations under
# the License.
from oslo_config import cfg
from oslo_log import log as logging
from osprofiler import profiler
from stevedore import driver
@ -20,28 +19,11 @@ from stevedore import extension
from zaqar import common
from zaqar.common import decorators
from zaqar.i18n import _
from zaqar.conf import storage
from zaqar.storage import base
LOG = logging.getLogger(__name__)
_PIPELINE_RESOURCES = ('queue', 'message', 'claim', 'subscription')
_PIPELINE_CONFIGS = tuple((
cfg.ListOpt(resource + '_pipeline', default=[],
help=_('Pipeline to use for processing {0} operations. '
'This pipeline will be consumed before calling '
'the storage driver\'s controller methods.')
.format(resource))
for resource in _PIPELINE_RESOURCES
))
_PIPELINE_GROUP = 'storage'
def _config_options():
return [(_PIPELINE_GROUP, _PIPELINE_CONFIGS)]
def _get_storage_pipeline(resource_name, conf, *args, **kwargs):
"""Constructs and returns a storage resource pipeline.
@ -64,10 +46,10 @@ def _get_storage_pipeline(resource_name, conf, *args, **kwargs):
:returns: A pipeline to use.
:rtype: `Pipeline`
"""
conf.register_opts(_PIPELINE_CONFIGS,
group=_PIPELINE_GROUP)
conf.register_opts(storage.ALL_OPTS,
group=storage.GROUP_NAME)
storage_conf = conf[_PIPELINE_GROUP]
storage_conf = conf[storage.GROUP_NAME]
pipeline = []
for ns in storage_conf[resource_name + '_pipeline']:

18
zaqar/storage/pooling.py
View File

@ -16,13 +16,13 @@
import heapq
import itertools
from oslo_config import cfg
from oslo_log import log
from osprofiler import profiler
from zaqar.common import decorators
from zaqar.common import errors as cerrors
from zaqar.common.storage import select
from zaqar.conf import pooling_catalog
from zaqar.i18n import _
from zaqar import storage
from zaqar.storage import errors
@ -31,13 +31,6 @@ from zaqar.storage import utils
LOG = log.getLogger(__name__)
_CATALOG_OPTIONS = (
cfg.BoolOpt('enable_virtual_pool', default=False,
help=('If enabled, the message_store will be used '
'as the storage for the virtual pool.')),
)
_CATALOG_GROUP = 'pooling:catalog'
# NOTE(kgriffs): E.g.: 'zaqar-pooling:5083853/my-queue'
_POOL_CACHE_PREFIX = 'pooling:'
@ -51,10 +44,6 @@ _POOL_CACHE_PREFIX = 'pooling:'
_POOL_CACHE_TTL = 10
def _config_options():
return [(_CATALOG_GROUP, _CATALOG_OPTIONS)]
def _pool_cache_key(queue, project=None):
# NOTE(kgriffs): Use string concatenation for performance,
# also put project first since it is guaranteed to be
@ -452,8 +441,9 @@ class Catalog(object):
self._cache = cache
self.control = control
self._conf.register_opts(_CATALOG_OPTIONS, group=_CATALOG_GROUP)
self._catalog_conf = self._conf[_CATALOG_GROUP]
self._conf.register_opts(pooling_catalog.ALL_OPTS,
group=pooling_catalog.GROUP_NAME)
self._catalog_conf = self._conf[pooling_catalog.GROUP_NAME]
self._pools_ctrl = control.pools_controller
self._flavor_ctrl = control.flavors_controller

17
zaqar/storage/redis/driver.py
View File

@ -19,10 +19,11 @@ from six.moves import urllib
from zaqar.common import decorators
from zaqar.common import errors
from zaqar.conf import drivers_management_store_redis
from zaqar.conf import drivers_message_store_redis
from zaqar.i18n import _
from zaqar import storage
from zaqar.storage.redis import controllers
from zaqar.storage.redis import options
REDIS_DEFAULT_PORT = 6379
SENTINEL_DEFAULT_PORT = 26379
@ -156,11 +157,14 @@ class DataDriver(storage.DataDriverBase):
storage.Capabilities.AOD,
storage.Capabilities.HIGH_THROUGHPUT)
_DRIVER_OPTIONS = options._config_options()
_DRIVER_OPTIONS = [(drivers_management_store_redis.GROUP_NAME,
drivers_management_store_redis.ALL_OPTS),
(drivers_message_store_redis.GROUP_NAME,
drivers_message_store_redis.ALL_OPTS)]
def __init__(self, conf, cache, control_driver):
super(DataDriver, self).__init__(conf, cache, control_driver)
self.redis_conf = self.conf[options.MESSAGE_REDIS_GROUP]
self.redis_conf = self.conf[drivers_message_store_redis.GROUP_NAME]
server_version = self.connection.info()['redis_version']
if tuple(map(int, server_version.split('.'))) < (2, 6):
@ -240,10 +244,11 @@ class ControlDriver(storage.ControlDriverBase):
def __init__(self, conf, cache):
super(ControlDriver, self).__init__(conf, cache)
self.conf.register_opts(options.MANAGEMENT_REDIS_OPTIONS,
group=options.MANAGEMENT_REDIS_GROUP)
self.conf.register_opts(
drivers_management_store_redis.ALL_OPTS,
group=drivers_management_store_redis.GROUP_NAME)
self.redis_conf = self.conf[options.MANAGEMENT_REDIS_GROUP]
self.redis_conf = self.conf[drivers_management_store_redis.GROUP_NAME]
def close(self):
self.connection.close()

81
zaqar/storage/redis/options.py
View File

@ -1,81 +0,0 @@
# Copyright (c) 2014 Prashanth Raghu.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Redis storage driver configuration options."""
from oslo_config import cfg
_deprecated_group = 'drivers:storage:redis'
# options common to management and message storage
_COMMON_REDIS_OPTIONS = (
cfg.StrOpt('uri', default="redis://127.0.0.1:6379",
deprecated_opts=[cfg.DeprecatedOpt(
'uri',
group=_deprecated_group), ],
help=('Redis connection URI, taking one of three forms. '
'For a direct connection to a Redis server, use '
'the form "redis://[:password]@host[:port][?options]", '
'where password is redis-server\'s password, when'
'redis-server is set password, the password option'
'needs to be set. port defaults to 6379 if not'
'specified. For an HA master-slave Redis cluster using'
' Redis Sentinel, use the form '
'"redis://[:password]@host1[:port1]'
'[,host2[:port2],...,hostN[:portN]][?options]", '
'where each host specified corresponds to an '
'instance of redis-sentinel. In this form, the '
'name of the Redis master used in the Sentinel '
'configuration must be included in the query '
'string as "master=<name>". Finally, to connect '
'to a local instance of Redis over a unix socket, '
'you may use the form '
'"redis:[:password]@/path/to/redis.sock[?options]".'
' In all forms, the "socket_timeout" option may be'
'specified in the query string. Its value is '
'given in seconds. If not provided, '
'"socket_timeout" defaults to 0.1 seconds.'
'There are multiple database instances in redis '
'database, for example in the /etc/redis/redis.conf, '
'if the parameter is "database 16", there are 16 '
'database instances. By default, the data is stored '
'in db = 0 database, if you want to use db = 1 '
'database, you can use the following form: '
'"redis://host[:port][?dbid=1]".')),
cfg.IntOpt('max_reconnect_attempts', default=10,
deprecated_opts=[cfg.DeprecatedOpt(
'max_reconnect_attempts',
group=_deprecated_group), ],
help=('Maximum number of times to retry an operation that '
'failed due to a redis node failover.')),
cfg.FloatOpt('reconnect_sleep', default=1.0,
deprecated_opts=[cfg.DeprecatedOpt(
'reconnect_sleep',
group=_deprecated_group), ],
help=('Base sleep interval between attempts to reconnect '
'after a redis node failover. '))
)
MANAGEMENT_REDIS_OPTIONS = _COMMON_REDIS_OPTIONS
MESSAGE_REDIS_OPTIONS = _COMMON_REDIS_OPTIONS
MANAGEMENT_REDIS_GROUP = 'drivers:management_store:redis'
MESSAGE_REDIS_GROUP = 'drivers:message_store:redis'
def _config_options():
return [(MANAGEMENT_REDIS_GROUP, MANAGEMENT_REDIS_OPTIONS),
(MESSAGE_REDIS_GROUP, MESSAGE_REDIS_OPTIONS)]

10
zaqar/storage/sqlalchemy/driver.py
View File

@ -19,18 +19,20 @@ from osprofiler import sqlalchemy as sa_tracer
import sqlalchemy as sa
from zaqar.common import decorators
from zaqar.conf import drivers_management_store_sqlalchemy
from zaqar import storage
from zaqar.storage.sqlalchemy import controllers
from zaqar.storage.sqlalchemy import options
class ControlDriver(storage.ControlDriverBase):
def __init__(self, conf, cache):
super(ControlDriver, self).__init__(conf, cache)
self.conf.register_opts(options.MANAGEMENT_SQLALCHEMY_OPTIONS,
group=options.MANAGEMENT_SQLALCHEMY_GROUP)
self.sqlalchemy_conf = self.conf[options.MANAGEMENT_SQLALCHEMY_GROUP]
self.conf.register_opts(
drivers_management_store_sqlalchemy.ALL_OPTS,
group=drivers_management_store_sqlalchemy.GROUP_NAME)
self.sqlalchemy_conf = self.conf[
drivers_management_store_sqlalchemy.GROUP_NAME]
def _mysql_on_connect(self, conn, record):
# NOTE(flaper87): This is necessary in order

34
zaqar/storage/sqlalchemy/options.py
View File

@ -1,34 +0,0 @@
# Copyright (c) 2014 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""SQLAlchemy storage driver implementation."""
from oslo_config import cfg
_deprecated_group = 'drivers:storage:sqlalchemy'
_COMMON_SQLALCHEMY_OPTIONS = (
cfg.StrOpt('uri', default='sqlite:///:memory:',
deprecated_opts=[cfg.DeprecatedOpt(
'uri',
group=_deprecated_group), ],
help='An sqlalchemy URL'),
)
MANAGEMENT_SQLALCHEMY_OPTIONS = _COMMON_SQLALCHEMY_OPTIONS
MANAGEMENT_SQLALCHEMY_GROUP = 'drivers:management_store:sqlalchemy'
def _config_options():
return [(MANAGEMENT_SQLALCHEMY_GROUP, MANAGEMENT_SQLALCHEMY_OPTIONS)]

7
zaqar/storage/swift/driver.py
View File

@ -21,20 +21,21 @@ from oslo_log import log as oslo_logging
import swiftclient
from zaqar.common import decorators
from zaqar.conf import drivers_message_store_swift
from zaqar import storage
from zaqar.storage.swift import controllers
from zaqar.storage.swift import options
LOG = oslo_logging.getLogger(__name__)
class DataDriver(storage.DataDriverBase):
_DRIVER_OPTIONS = options._config_options()
_DRIVER_OPTIONS = [(drivers_message_store_swift.GROUP_NAME,
drivers_message_store_swift.ALL_OPTS)]
def __init__(self, conf, cache, control_driver):
super(DataDriver, self).__init__(conf, cache, control_driver)
self.swift_conf = self.conf[options.MESSAGE_SWIFT_GROUP]
self.swift_conf = self.conf[drivers_message_store_swift.GROUP_NAME]
if not self.conf.debug:
# Reduce swiftclient logging, in particular to remove 404s
logging.getLogger("swiftclient").setLevel(logging.WARNING)

39
zaqar/storage/swift/options.py
View File

@ -1,39 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Swift storage driver configuration options."""
from oslo_config import cfg
MESSAGE_SWIFT_OPTIONS = (
cfg.StrOpt("auth_url", default="http://127.0.0.1:5000/v3/",
help="URI of Keystone endpoint to discover Swift"),
cfg.StrOpt("uri",
default="swift://demo:nomoresecrete@/demo",
help="Custom URI describing the swift connection."),
cfg.StrOpt("insecure", default=False, help="Don't check SSL certificate"),
cfg.StrOpt("project_domain_id", default="default",
help="Domain ID containing project"),
cfg.StrOpt("project_domain_name", help="Domain name containing project"),
cfg.StrOpt("user_domain_id", default="default", help="User's domain id"),
cfg.StrOpt("user_domain_name", help="User's domain name"),
cfg.StrOpt("region_name", help="Region name"),
cfg.StrOpt("interface", default="publicURL",
help="The default interface for endpoint URL discovery."),
)
MESSAGE_SWIFT_GROUP = 'drivers:message_store:swift'
def _config_options():
return [(MESSAGE_SWIFT_GROUP, MESSAGE_SWIFT_OPTIONS), ]

4
zaqar/storage/utils.py
View File

@ -128,9 +128,9 @@ def load_storage_driver(conf, cache, storage_type=None,
driver_type = 'zaqar.{0}.storage'.format(mode)
_invoke_args = [conf, cache]
_invoke_args = (conf, cache)
if control_driver is not None:
_invoke_args.append(control_driver)
_invoke_args = (conf, cache, control_driver)
try:
mgr = driver.DriverManager(driver_type,

26
zaqar/tests/base.py
View File

@ -21,7 +21,11 @@ from oslo_log import log
from osprofiler import opts
import testtools
from zaqar.common import configs
from zaqar.conf import default
from zaqar.conf import drivers
from zaqar.conf import notification
from zaqar.conf import profiler
from zaqar.conf import signed_url
from zaqar.tests import helpers
@ -54,18 +58,16 @@ class TestBase(testtools.TestCase):
else:
self.conf = cfg.ConfigOpts()
self.conf.register_opts(configs._GENERAL_OPTIONS)
self.conf.register_opts(configs._DRIVER_OPTIONS,
group=configs._DRIVER_GROUP)
self.conf.register_opts(configs._NOTIFICATION_OPTIONS,
group=configs._NOTIFICATION_GROUP)
self.conf.register_opts(configs._NOTIFICATION_OPTIONS,
group=configs._NOTIFICATION_GROUP)
self.conf.register_opts(configs._SIGNED_URL_OPTIONS,
group=configs._SIGNED_URL_GROUP)
self.conf.register_opts(default.ALL_OPTS)
self.conf.register_opts(drivers.ALL_OPTS,
group=drivers.GROUP_NAME)
self.conf.register_opts(notification.ALL_OPTS,
group=notification.GROUP_NAME)
self.conf.register_opts(signed_url.ALL_OPTS,
group=signed_url.GROUP_NAME)
opts.set_defaults(self.conf)
self.conf.register_opts(configs._PROFILER_OPTIONS,
group=configs._PROFILER_GROUP)
self.conf.register_opts(profiler.ALL_OPTS,
group=profiler.GROUP_NAME)
self.redis_url = os.environ.get('ZAQAR_TEST_REDIS_URL',
'redis://127.0.0.1:6379')
self.mongodb_url = os.environ.get('ZAQAR_TEST_MONGODB_URL',

2
zaqar/tests/functional/base.py
View File

@ -79,8 +79,6 @@ class FunctionalTestBase(testing.TestBase):
validator = validation.Validator(self.mconf)
self.limits = validator._limits_conf
transport_base._config_options()
self.resource_defaults = transport_base.ResourceDefaults(self.mconf)
# Always register options

4
zaqar/tests/unit/common/storage/test_utils.py
View File

@ -15,7 +15,7 @@
import ddt
from zaqar.common import configs
from zaqar.conf import default
from zaqar.storage import utils
from zaqar import tests as testing
@ -25,7 +25,7 @@ class TestUtils(testing.TestBase):
def setUp(self):
super(TestUtils, self).setUp()
self.conf.register_opts(configs._GENERAL_OPTIONS)
self.conf.register_opts(default.ALL_OPTS)
@testing.requires_mongodb
def test_can_connect_succeeds_if_good_uri_mongo(self):

4
zaqar/tests/unit/common/test_decorators.py
View File

@ -18,8 +18,8 @@ from oslo_cache import core
from oslo_config import cfg
from zaqar.common import cache as oslo_cache
from zaqar.common import configs
from zaqar.common import decorators
from zaqar.conf import default
from zaqar.tests import base
@ -27,7 +27,7 @@ class TestDecorators(base.TestBase):
def setUp(self):
super(TestDecorators, self).setUp()
self.conf.register_opts(configs._GENERAL_OPTIONS)
self.conf.register_opts(default.ALL_OPTS)
def test_memoized_getattr(self):

21
zaqar/tests/unit/storage/test_impl_mongodb.py
View File

@ -26,12 +26,13 @@ import six
from testtools import matchers
from zaqar.common import cache as oslo_cache
from zaqar.common import configs
from zaqar.conf import default
from zaqar.conf import drivers_management_store_mongodb
from zaqar.conf import drivers_message_store_mongodb
from zaqar import storage
from zaqar.storage import errors
from zaqar.storage import mongodb
from zaqar.storage.mongodb import controllers
from zaqar.storage.mongodb import options
from zaqar.storage.mongodb import utils
from zaqar.storage import pooling
from zaqar import tests as testing
@ -51,12 +52,12 @@ class MongodbSetupMixin(object):
self.driver.connection.drop_database(db)
def _prepare_conf(self):
if options.MESSAGE_MONGODB_GROUP in self.conf:
self.config(options.MESSAGE_MONGODB_GROUP,
if drivers_message_store_mongodb.GROUP_NAME in self.conf:
self.config(drivers_message_store_mongodb.GROUP_NAME,
database=uuid.uuid4().hex)
if options.MANAGEMENT_MONGODB_GROUP in self.conf:
self.config(options.MANAGEMENT_MONGODB_GROUP,
if drivers_management_store_mongodb.GROUP_NAME in self.conf:
self.config(drivers_management_store_mongodb.GROUP_NAME,
database=uuid.uuid4().hex)
@ -67,10 +68,10 @@ class MongodbUtilsTest(MongodbSetupMixin, testing.TestBase):
def setUp(self):
super(MongodbUtilsTest, self).setUp()
self.conf.register_opts(options.MESSAGE_MONGODB_OPTIONS,
group=options.MESSAGE_MONGODB_GROUP)
self.conf.register_opts(drivers_message_store_mongodb.ALL_OPTS,
group=drivers_message_store_mongodb.GROUP_NAME)
self.mongodb_conf = self.conf[options.MESSAGE_MONGODB_GROUP]
self.mongodb_conf = self.conf[drivers_message_store_mongodb.GROUP_NAME]
MockDriver = collections.namedtuple('MockDriver', 'mongodb_conf')
@ -152,7 +153,7 @@ class MongodbDriverTest(MongodbSetupMixin, testing.TestBase):
def setUp(self):
super(MongodbDriverTest, self).setUp()
self.conf.register_opts(configs._GENERAL_OPTIONS)
self.conf.register_opts(default.ALL_OPTS)
self.config(unreliable=False)
oslo_cache.register_config(self.conf)

8
zaqar/tests/unit/storage/test_impl_redis.py
View File

@ -23,12 +23,12 @@ import redis
from zaqar.common import cache as oslo_cache
from zaqar.common import errors
from zaqar.conf import drivers_message_store_redis
from zaqar import storage
from zaqar.storage import pooling
from zaqar.storage.redis import controllers
from zaqar.storage.redis import driver
from zaqar.storage.redis import messages
from zaqar.storage.redis import options
from zaqar.storage.redis import utils
from zaqar import tests as testing
from zaqar.tests.unit.storage import base
@ -65,10 +65,10 @@ class RedisUtilsTest(testing.TestBase):
def setUp(self):
super(RedisUtilsTest, self).setUp()
self.conf.register_opts(options.MESSAGE_REDIS_OPTIONS,
group=options.MESSAGE_REDIS_GROUP)
self.conf.register_opts(drivers_message_store_redis.ALL_OPTS,
group=drivers_message_store_redis.GROUP_NAME)
self.redis_conf = self.conf[options.MESSAGE_REDIS_GROUP]
self.redis_conf = self.conf[drivers_message_store_redis.GROUP_NAME]
MockDriver = collections.namedtuple('MockDriver', 'redis_conf')

20
zaqar/tests/unit/transport/websocket/base.py
View File

@ -15,10 +15,10 @@
from oslo_serialization import jsonutils
from zaqar import bootstrap
from zaqar.common import configs
from zaqar.conf import default
from zaqar.conf import drivers_transport_websocket
from zaqar.conf import transport
from zaqar import tests as testing
from zaqar.transport import validation
from zaqar.transport.websocket import driver
class TestBase(testing.TestBase):
@ -31,14 +31,14 @@ class TestBase(testing.TestBase):
if not self.config_file:
self.skipTest("No config specified")
self.conf.register_opts(configs._GENERAL_OPTIONS)
self.conf.register_opts(validation._TRANSPORT_LIMITS_OPTIONS,
group=validation._TRANSPORT_LIMITS_GROUP)
self.transport_cfg = self.conf[validation._TRANSPORT_LIMITS_GROUP]
self.conf.register_opts(default.ALL_OPTS)
self.conf.register_opts(transport.ALL_OPTS,
group=transport.GROUP_NAME)
self.transport_cfg = self.conf[transport.GROUP_NAME]
self.conf.register_opts(driver._WS_OPTIONS,
group=driver._WS_GROUP)
self.ws_cfg = self.conf[driver._WS_GROUP]
self.conf.register_opts(drivers_transport_websocket.ALL_OPTS,
group=drivers_transport_websocket.GROUP_NAME)
self.ws_cfg = self.conf[drivers_transport_websocket.GROUP_NAME]
self.conf.unreliable = True
self.conf.admin_mode = True

20
zaqar/tests/unit/transport/wsgi/base.py
View File

@ -20,10 +20,10 @@ from oslo_serialization import jsonutils
from oslo_utils import uuidutils
from zaqar import bootstrap
from zaqar.common import configs
from zaqar.conf import default
from zaqar.conf import drivers_transport_wsgi
from zaqar.conf import transport
from zaqar import tests as testing
from zaqar.transport import validation
from zaqar.transport.wsgi import driver
class TestBase(testing.TestBase):
@ -36,14 +36,14 @@ class TestBase(testing.TestBase):
if not self.config_file:
self.skipTest("No config specified")
self.conf.register_opts(configs._GENERAL_OPTIONS)
self.conf.register_opts(validation._TRANSPORT_LIMITS_OPTIONS,
group=validation._TRANSPORT_LIMITS_GROUP)
self.transport_cfg = self.conf[validation._TRANSPORT_LIMITS_GROUP]
self.conf.register_opts(default.ALL_OPTS)
self.conf.register_opts(transport.ALL_OPTS,
group=transport.GROUP_NAME)
self.transport_cfg = self.conf[transport.GROUP_NAME]
self.conf.register_opts(driver._WSGI_OPTIONS,
group=driver._WSGI_GROUP)
self.wsgi_cfg = self.conf[driver._WSGI_GROUP]
self.conf.register_opts(drivers_transport_wsgi.ALL_OPTS,
group=drivers_transport_wsgi.GROUP_NAME)
self.wsgi_cfg = self.conf[drivers_transport_wsgi.GROUP_NAME]
self.conf.unreliable = True
self.conf.admin_mode = True

41
zaqar/transport/base.py
View File

@ -15,40 +15,10 @@
import abc
from oslo_config import cfg
import six
_GENERAL_TRANSPORT_OPTIONS = (
cfg.StrOpt('auth_strategy', default='',
help=('Backend to use for authentication. '
'For no auth, keep it empty. '
'Existing strategies: keystone. '
'See also the keystone_authtoken section below')),
)
_RESOURCE_DEFAULTS = (
cfg.IntOpt('default_message_ttl', default=3600,
help=('Defines how long a message will be accessible.')),
cfg.IntOpt('default_message_delay', default=0,
help=('Defines the defautl value for queue delay seconds.'
'The 0 means the delayed queues feature is close.')),
cfg.IntOpt('default_claim_ttl', default=300,
help=('Defines how long a message will be in claimed state.')),
cfg.IntOpt('default_claim_grace', default=60,
help=('Defines the message grace period in seconds.')),
cfg.IntOpt('default_subscription_ttl', default=3600,
help=('Defines how long a subscription will be available.')),
)
_TRANSPORT_GROUP = 'transport'
def _config_options():
return [
(None, _GENERAL_TRANSPORT_OPTIONS),
(_TRANSPORT_GROUP, _RESOURCE_DEFAULTS),
]
from zaqar.conf import default
from zaqar.conf import transport
class ResourceDefaults(object):
@ -56,8 +26,9 @@ class ResourceDefaults(object):
def __init__(self, conf):
self._conf = conf
self._conf.register_opts(_RESOURCE_DEFAULTS, group=_TRANSPORT_GROUP)
self._defaults = self._conf[_TRANSPORT_GROUP]
self._conf.register_opts(transport.ALL_OPTS,
group=transport.GROUP_NAME)
self._defaults = self._conf[transport.GROUP_NAME]
@property
def message_ttl(self):
@ -96,7 +67,7 @@ class DriverBase(object):
self._cache = cache
self._control = control
self._conf.register_opts(_GENERAL_TRANSPORT_OPTIONS)
self._conf.register_opts([default.auth_strategy])
self._defaults = ResourceDefaults(self._conf)
@abc.abstractmethod

76
zaqar/transport/validation.py
View File

@ -17,11 +17,11 @@
import datetime
import re
from oslo_config import cfg
from oslo_utils import timeutils
import six
from zaqar.common import consts
from zaqar.conf import transport
from zaqar.i18n import _
@ -31,70 +31,6 @@ MIN_CLAIM_GRACE = 60
MIN_DELAY_TTL = 0
MIN_SUBSCRIPTION_TTL = 60
_PURGBLE_RESOURCE_TYPES = {'messages', 'subscriptions'}
_TRANSPORT_LIMITS_OPTIONS = (
cfg.IntOpt('max_queues_per_page', default=20,
deprecated_name='queue_paging_uplimit',
deprecated_group='limits:transport',
help='Defines the maximum number of queues per page.'),
cfg.IntOpt('max_messages_per_page', default=20,
deprecated_name='message_paging_uplimit',
deprecated_group='limits:transport',
help='Defines the maximum number of messages per page.'),
cfg.IntOpt('max_subscriptions_per_page', default=20,
deprecated_name='subscription_paging_uplimit',
deprecated_group='limits:transport',
help='Defines the maximum number of subscriptions per page.'),
cfg.IntOpt('max_messages_per_claim_or_pop', default=20,
deprecated_name='max_messages_per_claim',
help='The maximum number of messages that can be claimed (OR) '
'popped in a single request'),
cfg.IntOpt('max_queue_metadata', default=64 * 1024,
deprecated_name='metadata_size_uplimit',
deprecated_group='limits:transport',
help='Defines the maximum amount of metadata in a queue.'),
cfg.IntOpt('max_messages_post_size', default=256 * 1024,
deprecated_name='message_size_uplimit',
deprecated_group='limits:transport',
deprecated_opts=[cfg.DeprecatedOpt('max_message_size')],
help='Defines the maximum size of message posts.'),
cfg.IntOpt('max_message_ttl', default=1209600,
deprecated_name='message_ttl_max',
deprecated_group='limits:transport',
help='Maximum amount of time a message will be available.'),
cfg.IntOpt('max_message_delay', default=900,
help='Maximum delay seconds for messages can be claimed.'),
cfg.IntOpt('max_claim_ttl', default=43200,
deprecated_name='claim_ttl_max',
deprecated_group='limits:transport',
help='Maximum length of a message in claimed state.'),
cfg.IntOpt('max_claim_grace', default=43200,
deprecated_name='claim_grace_max',
deprecated_group='limits:transport',
help='Defines the maximum message grace period in seconds.'),
cfg.ListOpt('subscriber_types', default=['http', 'https', 'mailto',
'trust+http', 'trust+https'],
help='Defines supported subscriber types.'),
cfg.IntOpt('max_flavors_per_page', default=20,
help='Defines the maximum number of flavors per page.'),
cfg.IntOpt('max_pools_per_page', default=20,
help='Defines the maximum number of pools per page.'),
)
_TRANSPORT_LIMITS_GROUP = 'transport'
# NOTE(kgriffs): Don't use \w because it isn't guaranteed to match
# only ASCII characters.
QUEUE_NAME_REGEX = re.compile('^[a-zA-Z0-9_\-.]+$')
@ -102,10 +38,6 @@ QUEUE_NAME_MAX_LEN = 64
PROJECT_ID_MAX_LEN = 256
def _config_options():
return [(_TRANSPORT_LIMITS_GROUP, _TRANSPORT_LIMITS_OPTIONS)]
class ValidationFailed(ValueError):
"""User input did not follow API restrictions."""
@ -117,9 +49,9 @@ class ValidationFailed(ValueError):
class Validator(object):
def __init__(self, conf):
self._conf = conf
self._conf.register_opts(_TRANSPORT_LIMITS_OPTIONS,
group=_TRANSPORT_LIMITS_GROUP)
self._limits_conf = self._conf[_TRANSPORT_LIMITS_GROUP]
self._conf.register_opts(transport.ALL_OPTS,
group=transport.GROUP_NAME)
self._limits_conf = self._conf[transport.GROUP_NAME]
self._supported_operations = ('add', 'remove', 'replace')
def queue_identification(self, queue, project):

33
zaqar/transport/websocket/driver.py
View File

@ -15,7 +15,6 @@
import socket
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import netutils
@ -25,41 +24,16 @@ except ImportError:
import trollius as asyncio
from zaqar.common import decorators
from zaqar.conf import drivers_transport_websocket
from zaqar.i18n import _
from zaqar.transport import base
from zaqar.transport.middleware import auth
from zaqar.transport.websocket import factory
_WS_OPTIONS = (
cfg.HostAddressOpt('bind', default='127.0.0.1',
help='Address on which the self-hosting server will '
'listen.'),
cfg.PortOpt('port', default=9000,
help='Port on which the self-hosting server will listen.'),
cfg.PortOpt('external-port',
help='Port on which the service is provided to the user.'),
cfg.HostAddressOpt('notification-bind',
help='Address on which the notification server will '
'listen.'),
cfg.PortOpt('notification-port', default=0,
help='Port on which the notification server will listen.'),
)
_WS_GROUP = 'drivers:transport:websocket'
LOG = logging.getLogger(__name__)
def _config_options():
return [(_WS_GROUP, _WS_OPTIONS)]
# TODO(derekh): use escape_ipv6 from oslo.utils once available
def _escape_ipv6(address):
"""Escape an IP address in square brackets if IPv6"""
@ -75,8 +49,9 @@ class Driver(base.DriverBase):
self._api = api
self._cache = cache
self._conf.register_opts(_WS_OPTIONS, group=_WS_GROUP)
self._ws_conf = self._conf[_WS_GROUP]
self._conf.register_opts(drivers_transport_websocket.ALL_OPTS,
group=drivers_transport_websocket.GROUP_NAME)
self._ws_conf = self._conf[drivers_transport_websocket.GROUP_NAME]
if self._conf.auth_strategy:
auth_strategy = auth.strategy(self._conf.auth_strategy)

21
zaqar/transport/wsgi/driver.py
View File

@ -19,12 +19,12 @@ import six
import socket
from wsgiref import simple_server
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import netutils
from zaqar.common import decorators
from zaqar.common.transport.wsgi import helpers
from zaqar.conf import drivers_transport_wsgi
from zaqar.i18n import _
from zaqar import transport
from zaqar.transport import acl
@ -37,24 +37,10 @@ from zaqar.transport.wsgi import v1_1
from zaqar.transport.wsgi import v2_0
from zaqar.transport.wsgi import version
_WSGI_OPTIONS = (
cfg.HostAddressOpt('bind', default='127.0.0.1',
help='Address on which the self-hosting server will '
'listen.'),
cfg.PortOpt('port', default=8888,
help='Port on which the self-hosting server will listen.'),
)
_WSGI_GROUP = 'drivers:transport:wsgi'
LOG = logging.getLogger(__name__)
def _config_options():
return [(_WSGI_GROUP, _WSGI_OPTIONS)]
class FuncMiddleware(object):
def __init__(self, func):
@ -69,8 +55,9 @@ class Driver(transport.DriverBase):
def __init__(self, conf, storage, cache, control):
super(Driver, self).__init__(conf, storage, cache, control)
self._conf.register_opts(_WSGI_OPTIONS, group=_WSGI_GROUP)
self._wsgi_conf = self._conf[_WSGI_GROUP]
self._conf.register_opts(drivers_transport_wsgi.ALL_OPTS,
group=drivers_transport_wsgi.GROUP_NAME)
self._wsgi_conf = self._conf[drivers_transport_wsgi.GROUP_NAME]
self._validate = validation.Validator(self._conf)
self.app = None