diff --git a/etc/valet/valet-config-generator.conf b/etc/valet/valet-config-generator.conf new file mode 100644 index 0000000..1b37c55 --- /dev/null +++ b/etc/valet/valet-config-generator.conf @@ -0,0 +1,7 @@ +[DEFAULT] +output_file = etc/valet/valet.conf.sample +wrap_width = 79 +namespace = valet.conf +namespace = oslo.log +namespace = oslo.db +namespace = keystonemiddleware.auth_token diff --git a/etc/valet/valet.conf b/etc/valet/valet.backup similarity index 100% rename from etc/valet/valet.conf rename to etc/valet/valet.backup diff --git a/etc/valet/valet.conf.sample b/etc/valet/valet.conf.sample new file mode 100644 index 0000000..9531a33 --- /dev/null +++ b/etc/valet/valet.conf.sample @@ -0,0 +1,480 @@ +[DEFAULT] + +# +# From oslo.log +# + +# If set to true, the logging level will be set to DEBUG instead of the default +# INFO level. (boolean value) +# Note: This option can be changed without restarting. +#debug = false + +# The name of a logging configuration file. This file is appended to any +# existing logging configuration files. For details about logging configuration +# files, see the Python logging module documentation. Note that when logging +# configuration files are used then all logging configuration is set in the +# configuration file and other logging configuration options are ignored (for +# example, logging_context_format_string). (string value) +# Note: This option can be changed without restarting. +# Deprecated group/name - [DEFAULT]/log_config +#log_config_append = + +# Defines the format string for %%(asctime)s in log records. Default: +# %(default)s . This option is ignored if log_config_append is set. (string +# value) +#log_date_format = %Y-%m-%d %H:%M:%S + +# (Optional) Name of log file to send logging output to. If no default is set, +# logging will go to stderr as defined by use_stderr. This option is ignored if +# log_config_append is set. (string value) +# Deprecated group/name - [DEFAULT]/logfile +#log_file = + +# (Optional) The base directory used for relative log_file paths. This option +# is ignored if log_config_append is set. (string value) +# Deprecated group/name - [DEFAULT]/logdir +#log_dir = + +# Uses logging handler designed to watch file system. When log file is moved or +# removed this handler will open a new log file with specified path +# instantaneously. It makes sense only if log_file option is specified and +# Linux platform is used. This option is ignored if log_config_append is set. +# (boolean value) +#watch_log_file = false + +# Use syslog for logging. Existing syslog format is DEPRECATED and will be +# changed later to honor RFC5424. This option is ignored if log_config_append +# is set. (boolean value) +#use_syslog = false + +# Enable journald for logging. If running in a systemd environment you may wish +# to enable journal support. Doing so will use the journal native protocol +# which includes structured metadata in addition to log messages.This option is +# ignored if log_config_append is set. (boolean value) +#use_journal = false + +# Syslog facility to receive log lines. This option is ignored if +# log_config_append is set. (string value) +#syslog_log_facility = LOG_USER + +# Log output to standard error. This option is ignored if log_config_append is +# set. (boolean value) +#use_stderr = false + +# Format string to use for log messages with context. (string value) +#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s + +# Format string to use for log messages when context is undefined. (string +# value) +#logging_default_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s + +# Additional data to append to log message when logging level for the message +# is DEBUG. (string value) +#logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d + +# Prefix each line of exception output with this format. (string value) +#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d ERROR %(name)s %(instance)s + +# Defines the format string for %(user_identity)s that is used in +# logging_context_format_string. (string value) +#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s + +# List of package logging levels in logger=LEVEL pairs. This option is ignored +# if log_config_append is set. (list value) +#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,oslo_messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN,oslo.cache=INFO,dogpile.core.dogpile=INFO + +# Enables or disables publication of error events. (boolean value) +#publish_errors = false + +# The format for an instance that is passed with the log message. (string +# value) +#instance_format = "[instance: %(uuid)s] " + +# The format for an instance UUID that is passed with the log message. (string +# value) +#instance_uuid_format = "[instance: %(uuid)s] " + +# Interval, number of seconds, of log rate limiting. (integer value) +#rate_limit_interval = 0 + +# Maximum number of logged messages per rate_limit_interval. (integer value) +#rate_limit_burst = 0 + +# Log level name used by rate limiting: CRITICAL, ERROR, INFO, WARNING, DEBUG +# or empty string. Logs with level greater or equal to rate_limit_except_level +# are not filtered. An empty string means that all levels are filtered. (string +# value) +#rate_limit_except_level = CRITICAL + +# Enables or disables fatal status of deprecations. (boolean value) +#fatal_deprecations = false + + +[database] + +# +# From oslo.db +# + +# If True, SQLite uses synchronous mode. (boolean value) +#sqlite_synchronous = true + +# The back end to use for the database. (string value) +# Deprecated group/name - [DEFAULT]/db_backend +#backend = sqlalchemy + +# The SQLAlchemy connection string to use to connect to the database. (string +# value) +# Deprecated group/name - [DEFAULT]/sql_connection +# Deprecated group/name - [DATABASE]/sql_connection +# Deprecated group/name - [sql]/connection +#connection = + +# The SQLAlchemy connection string to use to connect to the slave database. +# (string value) +#slave_connection = + +# The SQL mode to be used for MySQL sessions. This option, including the +# default, overrides any server-set SQL mode. To use whatever SQL mode is set +# by the server configuration, set this to no value. Example: mysql_sql_mode= +# (string value) +#mysql_sql_mode = TRADITIONAL + +# Timeout before idle SQL connections are reaped. (integer value) +# Deprecated group/name - [DEFAULT]/sql_idle_timeout +# Deprecated group/name - [DATABASE]/sql_idle_timeout +# Deprecated group/name - [sql]/idle_timeout +#idle_timeout = 3600 + +# Minimum number of SQL connections to keep open in a pool. (integer value) +# Deprecated group/name - [DEFAULT]/sql_min_pool_size +# Deprecated group/name - [DATABASE]/sql_min_pool_size +#min_pool_size = 1 + +# Maximum number of SQL connections to keep open in a pool. Setting a value of +# 0 indicates no limit. (integer value) +# Deprecated group/name - [DEFAULT]/sql_max_pool_size +# Deprecated group/name - [DATABASE]/sql_max_pool_size +#max_pool_size = 5 + +# Maximum number of database connection retries during startup. Set to -1 to +# specify an infinite retry count. (integer value) +# Deprecated group/name - [DEFAULT]/sql_max_retries +# Deprecated group/name - [DATABASE]/sql_max_retries +#max_retries = 10 + +# Interval between retries of opening a SQL connection. (integer value) +# Deprecated group/name - [DEFAULT]/sql_retry_interval +# Deprecated group/name - [DATABASE]/reconnect_interval +#retry_interval = 10 + +# If set, use this value for max_overflow with SQLAlchemy. (integer value) +# Deprecated group/name - [DEFAULT]/sql_max_overflow +# Deprecated group/name - [DATABASE]/sqlalchemy_max_overflow +#max_overflow = 50 + +# Verbosity of SQL debugging information: 0=None, 100=Everything. (integer +# value) +# Minimum value: 0 +# Maximum value: 100 +# Deprecated group/name - [DEFAULT]/sql_connection_debug +#connection_debug = 0 + +# Add Python stack traces to SQL as comment strings. (boolean value) +# Deprecated group/name - [DEFAULT]/sql_connection_trace +#connection_trace = false + +# If set, use this value for pool_timeout with SQLAlchemy. (integer value) +# Deprecated group/name - [DATABASE]/sqlalchemy_pool_timeout +#pool_timeout = + +# Enable the experimental use of database reconnect on connection lost. +# (boolean value) +#use_db_reconnect = false + +# Seconds between retries of a database transaction. (integer value) +#db_retry_interval = 1 + +# If True, increases the interval between retries of a database operation up to +# db_max_retry_interval. (boolean value) +#db_inc_retry_interval = true + +# If db_inc_retry_interval is set, the maximum seconds between retries of a +# database operation. (integer value) +#db_max_retry_interval = 10 + +# Maximum retries in case of connection error or deadlock error before error is +# raised. Set to -1 to specify an infinite retry count. (integer value) +#db_max_retries = 20 + + +[engine] + +# +# From valet.conf +# + +# Set path to Valet Engine running process pid. (string value) +#pid = /var/run/valet/valet-engine.pid + +# +# Inform the name of datacenter (region name), +# where Valet is deployed. +# (string value) +#datacenter_name = Region1 + +# Set frequency, in seconds, for checking compute host status. (integer value) +#compute_trigger_frequency = 1800 + +# +# Set trigger time or frequency, in seconds, +# for checking datacenter topology. +# (integer value) +#topology_trigger_frequency = 1800 + +# +# Set default CPU overbooking ratio. +# Note that each compute node can have its own ratio. +# (floating point value) +#default_cpu_allocation_ratio = 16 + +# Set default RAM overbooking ratio. (floating point value) +#default_ram_allocation_ratio = 1.5 + +# Set default disk overbooking ratio. (floating point value) +#default_disk_allocation_ratio = 1 + +# Set percentage of standby CPU resources. (floating point value) +#static_cpu_standby_ratio = 20 + +# Set percentage of standby mem resources. (floating point value) +#static_mem_standby_ratio = 20 + +# Set percentage of disk standby resources. (floating point value) +#static_local_disk_standby_ratio = 20 + + +[keystone_authtoken] + +# +# From keystonemiddleware.auth_token +# + +# Complete "public" Identity API endpoint. This endpoint should not be an +# "admin" endpoint, as it should be accessible by all end users. +# Unauthenticated clients are redirected to this endpoint to authenticate. +# Although this endpoint should ideally be unversioned, client support in the +# wild varies. If you're using a versioned v2 endpoint here, then this should +# *not* be the same endpoint the service user utilizes for validating tokens, +# because normal end users may not be able to reach that endpoint. (string +# value) +#auth_uri = + +# API version of the admin Identity API endpoint. (string value) +#auth_version = + +# Do not handle authorization requests within the middleware, but delegate the +# authorization decision to downstream WSGI components. (boolean value) +#delay_auth_decision = false + +# Request timeout value for communicating with Identity API server. (integer +# value) +#http_connect_timeout = + +# How many times are we trying to reconnect when communicating with Identity +# API Server. (integer value) +#http_request_max_retries = 3 + +# Request environment key where the Swift cache object is stored. When +# auth_token middleware is deployed with a Swift cache, use this option to have +# the middleware share a caching backend with swift. Otherwise, use the +# ``memcached_servers`` option instead. (string value) +#cache = + +# Required if identity server requires client certificate (string value) +#certfile = + +# Required if identity server requires client certificate (string value) +#keyfile = + +# A PEM encoded Certificate Authority to use when verifying HTTPs connections. +# Defaults to system CAs. (string value) +#cafile = + +# Verify HTTPS connections. (boolean value) +#insecure = false + +# The region in which the identity server can be found. (string value) +#region_name = + +# DEPRECATED: Directory used to cache files related to PKI tokens. This option +# has been deprecated in the Ocata release and will be removed in the P +# release. (string value) +# This option is deprecated for removal since Ocata. +# Its value may be silently ignored in the future. +# Reason: PKI token format is no longer supported. +#signing_dir = + +# Optionally specify a list of memcached server(s) to use for caching. If left +# undefined, tokens will instead be cached in-process. (list value) +# Deprecated group/name - [keystone_authtoken]/memcache_servers +#memcached_servers = + +# In order to prevent excessive effort spent validating tokens, the middleware +# caches previously-seen tokens for a configurable duration (in seconds). Set +# to -1 to disable caching completely. (integer value) +#token_cache_time = 300 + +# DEPRECATED: Determines the frequency at which the list of revoked tokens is +# retrieved from the Identity service (in seconds). A high number of revocation +# events combined with a low cache duration may significantly reduce +# performance. Only valid for PKI tokens. This option has been deprecated in +# the Ocata release and will be removed in the P release. (integer value) +# This option is deprecated for removal since Ocata. +# Its value may be silently ignored in the future. +# Reason: PKI token format is no longer supported. +#revocation_cache_time = 10 + +# (Optional) If defined, indicate whether token data should be authenticated or +# authenticated and encrypted. If MAC, token data is authenticated (with HMAC) +# in the cache. If ENCRYPT, token data is encrypted and authenticated in the +# cache. If the value is not one of these options or empty, auth_token will +# raise an exception on initialization. (string value) +# Allowed values: None, MAC, ENCRYPT +#memcache_security_strategy = None + +# (Optional, mandatory if memcache_security_strategy is defined) This string is +# used for key derivation. (string value) +#memcache_secret_key = + +# (Optional) Number of seconds memcached server is considered dead before it is +# tried again. (integer value) +#memcache_pool_dead_retry = 300 + +# (Optional) Maximum total number of open connections to every memcached +# server. (integer value) +#memcache_pool_maxsize = 10 + +# (Optional) Socket timeout in seconds for communicating with a memcached +# server. (integer value) +#memcache_pool_socket_timeout = 3 + +# (Optional) Number of seconds a connection to memcached is held unused in the +# pool before it is closed. (integer value) +#memcache_pool_unused_timeout = 60 + +# (Optional) Number of seconds that an operation will wait to get a memcached +# client connection from the pool. (integer value) +#memcache_pool_conn_get_timeout = 10 + +# (Optional) Use the advanced (eventlet safe) memcached client pool. The +# advanced pool will only work under python 2.x. (boolean value) +#memcache_use_advanced_pool = false + +# (Optional) Indicate whether to set the X-Service-Catalog header. If False, +# middleware will not ask for service catalog on token validation and will not +# set the X-Service-Catalog header. (boolean value) +#include_service_catalog = true + +# Used to control the use and type of token binding. Can be set to: "disabled" +# to not check token binding. "permissive" (default) to validate binding +# information if the bind type is of a form known to the server and ignore it +# if not. "strict" like "permissive" but if the bind type is unknown the token +# will be rejected. "required" any form of token binding is needed to be +# allowed. Finally the name of a binding method that must be present in tokens. +# (string value) +#enforce_token_bind = permissive + +# DEPRECATED: If true, the revocation list will be checked for cached tokens. +# This requires that PKI tokens are configured on the identity server. (boolean +# value) +# This option is deprecated for removal since Ocata. +# Its value may be silently ignored in the future. +# Reason: PKI token format is no longer supported. +#check_revocations_for_cached = false + +# DEPRECATED: Hash algorithms to use for hashing PKI tokens. This may be a +# single algorithm or multiple. The algorithms are those supported by Python +# standard hashlib.new(). The hashes will be tried in the order given, so put +# the preferred one first for performance. The result of the first hash will be +# stored in the cache. This will typically be set to multiple values only while +# migrating from a less secure algorithm to a more secure one. Once all the old +# tokens are expired this option should be set to a single value for better +# performance. (list value) +# This option is deprecated for removal since Ocata. +# Its value may be silently ignored in the future. +# Reason: PKI token format is no longer supported. +#hash_algorithms = md5 + +# A choice of roles that must be present in a service token. Service tokens are +# allowed to request that an expired token can be used and so this check should +# tightly control that only actual services should be sending this token. Roles +# here are applied as an ANY check so any role in this list must be present. +# For backwards compatibility reasons this currently only affects the +# allow_expired check. (list value) +#service_token_roles = service + +# For backwards compatibility reasons we must let valid service tokens pass +# that don't pass the service_token_roles check as valid. Setting this true +# will become the default in a future release and should be enabled if +# possible. (boolean value) +#service_token_roles_required = false + +# Authentication type to load (string value) +# Deprecated group/name - [keystone_authtoken]/auth_plugin +#auth_type = + +# Config Section from which to load plugin specific options (string value) +#auth_section = + + +[music] + +# +# From valet.conf +# + +# The number of database replicas Music should distribute against. (integer +# value) +#replication_factor = 3 + +# +# Number of attempts in performing a read/write action to/from +# MUSIC. +# After exceeding maximum retries, another MUSIC host is attempted. +# (integer value) +#music_server_retries = 3 + +# +# IP address or FQDN used to contact the MUSIC service. +# Strings can be given in the format "localhost:8080, +# ip2:8080,host3:8080". +# (list value) +#hosts = localhost + +# Port opened to allow messages to the MUSIC REST service. (port value) +# Minimum value: 0 +# Maximum value: 65535 +#port = 8080 + +# Name of Cassandra keyspace used by Valet. (string value) +#keyspace = db_keyspace + + +[server] + +# +# From valet.conf +# + +# +# IP address or FQDN used to contact the MUSIC service. +# Strings can be given in the format "localhost:8080, +# ip2:8080,host3:8080". +# (list value) +#hosts = 0.0.0.0 + +# Port opened to allow messages to the MUSIC REST service. (port value) +# Minimum value: 0 +# Maximum value: 65535 +#port = 8090 diff --git a/requirements.txt b/requirements.txt index f02b6df..ea1cc49 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,12 +15,14 @@ pytz>=2013.6 # MIT python-keystoneclient>=3.8.0 # Apache-2.0 python-heatclient>=1.6.1 # Apache-2.0 python-novaclient>=7.1.0 # Apache-2.0 +keystonemiddleware>=4.12.0 # Apache-2.0 oslo.config>=3.22.0 # Apache-2.0 oslo.db>=4.21.1 # Apache-2.0 oslo.i18n!=3.15.2,>=2.1.0 # Apache-2.0 oslo.log>=3.22.0 # Apache-2.0 oslo.messaging>=5.19.0 # Apache-2.0 +oslo.middleware>=3.27.0 # Apache-2.0 oslo.serialization>=1.10.0 # Apache-2.0 oslo.utils>=3.20.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 4e86934..19ccc00 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,14 @@ packages = valet data_files = etc/valet/ = etc/* [entry_points] +oslo.config.opts = + valet.conf = valet.conf.opts:list_opts + +oslo.config.opts.defaults= + valet.conf = valet.common.config:set_middleware_defaults + pecan.command = populate = valet.api.v1.commands.populate:PopulateCommand + tempest.test_plugins = valet_tests = valet.tests.tempest.plugin:ValetTempestPlugin diff --git a/tox.ini b/tox.ini index 4b4a744..7ef1007 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ commands = ostestr --slowest '{posargs}' deps = -r{toxinidir}/test-requirements.txt whitelist_externals = - find + find [testenv:pep8] basepython = python2.7 @@ -38,7 +38,9 @@ commands = coverage report [testenv:docs] -commands = python setup.py build_sphinx +basepython = python2.7 +commands = + python setup.py build_sphinx [testenv:releasenotes] commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html @@ -51,6 +53,9 @@ basepython = python2.7 commands = sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html +[testenv:genconfig] +commands = oslo-config-generator --config-file=etc/valet/valet-config-generator.conf + [flake8] filename = *.py show-source = true diff --git a/valet/api/app.py b/valet/api/app.py index 3e87c80..0b75ba3 100644 --- a/valet/api/app.py +++ b/valet/api/app.py @@ -23,7 +23,6 @@ from valet.api.common import identity from valet.api.common import messaging from valet.api.db import models from valet.common.conf import get_logger -from valet.common.conf import init_conf def setup_app(config): @@ -41,7 +40,6 @@ def setup_app(config): # entry point for apache2 def load_app(config_file): """App Load.""" - init_conf("api.log") api.LOG = get_logger("api") return deploy(config_file) diff --git a/valet/api/v1/commands/populate.py b/valet/api/v1/commands/populate.py index b293531..66aaecf 100644 --- a/valet/api/v1/commands/populate.py +++ b/valet/api/v1/commands/populate.py @@ -24,7 +24,6 @@ from valet.api.db.models.music.ostro import PlacementResult from valet.api.db.models.music.placements import Placement from valet.api.db.models.music.plans import Plan from valet.common.conf import get_logger -from valet.common.conf import init_conf class PopulateCommand(BaseCommand): @@ -34,7 +33,6 @@ class PopulateCommand(BaseCommand): """Function creates and initializes database and environment.""" super(PopulateCommand, self).run(args) try: - init_conf("populate.log") LOG = api.LOG = get_logger("populate") LOG.info(_("Loading environment")) self.load_app() diff --git a/valet/api/wsgi.py b/valet/api/wsgi.py index 5df357c..93928ef 100644 --- a/valet/api/wsgi.py +++ b/valet/api/wsgi.py @@ -19,7 +19,6 @@ from common.i18n import _ from valet import api from valet.common.conf import get_logger -from valet.common.conf import init_conf def config_file(file_name=None): @@ -48,7 +47,6 @@ if __name__ == '__main__': # import pecan_mount # HTTPD = make_server('', 8090, pecan_mount.tree) # from valet.api.conf import register_conf, set_domain - init_conf("wsgi.log") api.LOG = get_logger("wsgi") HTTPD = make_server( '', 8090, deploy(config_file('/var/www/valet/config.py'))) diff --git a/valet/cli/groupcli.py b/valet/cli/groupcli.py index 62e5a49..5382742 100644 --- a/valet/cli/groupcli.py +++ b/valet/cli/groupcli.py @@ -19,7 +19,6 @@ import argparse import json from oslo_config import cfg import requests -from valet.common import conf as common CONF = cfg.CONF @@ -205,8 +204,7 @@ def populate_args_request_body(args): def run(args): """Run.""" - common.init_conf("cli.log") - args.host = args.host or CONF.server.host + args.host = args.host or CONF.server.hosts[0] args.port = args.port or CONF.server.port args.timeout = args.timeout or 10 rest_cmd, cmd_url = cmd_details(args) diff --git a/valet/common/conf.py b/valet/common/conf.py deleted file mode 100644 index 4e3ca14..0000000 --- a/valet/common/conf.py +++ /dev/null @@ -1,114 +0,0 @@ -# -# Copyright 2014-2017 AT&T Intellectual Property -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Conf.""" - -from oslo_config import cfg -from oslo_log import log as logging - -CONF = cfg.CONF -DOMAIN = 'valet' - - -def get_logger(name): - return logging.getLogger(name) - -LOG = get_logger("engine") - -server_group = cfg.OptGroup(name='server', title='Valet API Server conf') -server_opts = [ - cfg.StrOpt('host', default='0.0.0.0'), - cfg.StrOpt('port', default='8090'), -] - - -messaging_group = cfg.OptGroup(name='messaging', title='Valet Messaging conf') -messaging_opts = [ - cfg.StrOpt('username'), - cfg.StrOpt('password'), - cfg.StrOpt('host'), - cfg.IntOpt('port', default=5672), - cfg.IntOpt('timeout', default=3), -] - - -identity_group = cfg.OptGroup(name='identity', title='Valet identity conf') -identity_opts = [ - cfg.StrOpt('username'), - cfg.StrOpt('password'), - cfg.StrOpt('project_name'), - cfg.StrOpt('auth_url', default='http://controller:5000/v2.0'), - cfg.StrOpt('interface', default='admin'), -] - - -music_group = cfg.OptGroup(name='music', title='Valet Persistence conf') -music_opts = [ - cfg.ListOpt('hosts', default=['0.0.0.0']), - cfg.IntOpt('port', default=8080), - cfg.StrOpt('keyspace', default='valet'), - cfg.IntOpt('replication_factor', default=3), - cfg.IntOpt('tries', default=10), - cfg.IntOpt('interval', default=1), - cfg.StrOpt('request_table', default='placement_requests'), - cfg.StrOpt('response_table', default='placement_results'), - cfg.StrOpt('event_table', default='oslo_messages'), - cfg.StrOpt('resource_table', default='resource_status'), - cfg.StrOpt('app_table', default='app'), - cfg.StrOpt('resource_index_table', default='resource_log_index'), - cfg.StrOpt('app_index_table', default='app_log_index'), - cfg.StrOpt('uuid_table', default='uuid_map'), - cfg.IntOpt('music_server_retries', default=3), -] - - -def load_conf(args=None, project=DOMAIN, default_files=None): - if default_files: - CONF(default_config_files=default_files) - else: - CONF(args or [], project=project) - - -def init_conf(log_file="valet.log", args=None, grp2opt=None, - cli_opts=None, default_config_files=None): - CONF.log_file = log_file - logging.register_options(CONF) - - # init conf - general_groups = {server_group: server_opts, - music_group: music_opts, - identity_group: identity_opts, - messaging_group: messaging_opts} - - general_groups.update(grp2opt or {}) - - _register_conf(general_groups, cli_opts) - load_conf(args=args, default_files=default_config_files) - - # set logger - _set_logger() - - -def _set_logger(): - logging.setup(CONF, DOMAIN) - - -def _register_conf(grp2opt, cli_opts): - for grp in grp2opt: - CONF.register_group(grp) - CONF.register_opts(grp2opt[grp], grp) - - for opt in cli_opts or []: - CONF.register_cli_opts(opt) diff --git a/valet/common/config.py b/valet/common/config.py new file mode 100644 index 0000000..bc4a9ab --- /dev/null +++ b/valet/common/config.py @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_middleware import cors + + +def set_middleware_defaults(): + cors.set_defaults( + allow_headers=['X-Auth-Token', + 'X-Openstack-Request-Id', + 'X-Identity-Status', + 'X-Roles', + 'X-Service-Catalog', + 'X-User-Id', + 'X-Tenant-Id'], + expose_headers=['X-Auth-Token', + 'X-Openstack-Request-Id', + 'X-Subject-Token', + 'X-Service-Token'], + allow_methods=['GET', + 'PUT', + 'POST', + 'DELETE', + 'PATCH']) diff --git a/valet/conf/__init__.py b/valet/conf/__init__.py new file mode 100644 index 0000000..4ca78c2 --- /dev/null +++ b/valet/conf/__init__.py @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +from valet.conf import engine +from valet.conf import music +from valet.conf import server + + +CONF = cfg.CONF + +engine.register_opts(CONF) +music.register_opts(CONF) +server.register_opts(CONF) diff --git a/valet/conf/engine.py b/valet/conf/engine.py new file mode 100644 index 0000000..06d57e6 --- /dev/null +++ b/valet/conf/engine.py @@ -0,0 +1,82 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +engine_opts = [ + cfg.StrOpt( + 'pid', + default='/var/run/valet/valet-engine.pid', + help='Set path to Valet Engine running process pid.' + ), + cfg.StrOpt( + 'datacenter_name', + default='Region1', + help=""" + Inform the name of datacenter (region name), + where Valet is deployed. + """), + cfg.IntOpt( + 'compute_trigger_frequency', + default=1800, + help='Set frequency, in seconds, for checking compute host status.' + ), + cfg.IntOpt( + 'topology_trigger_frequency', + default=1800, + help=""" + Set trigger time or frequency, in seconds, + for checking datacenter topology. + """), + cfg.FloatOpt( + 'default_cpu_allocation_ratio', + default=16, + help=""" + Set default CPU overbooking ratio. + Note that each compute node can have its own ratio. + """), + cfg.FloatOpt( + 'default_ram_allocation_ratio', + default=1.5, + help='Set default RAM overbooking ratio.' + ), + cfg.FloatOpt( + 'default_disk_allocation_ratio', + default=1, + help='Set default disk overbooking ratio.' + ), + cfg.FloatOpt( + 'static_cpu_standby_ratio', + default=20, + help='Set percentage of standby CPU resources.' + ), + cfg.FloatOpt( + 'static_mem_standby_ratio', + default=20, + help='Set percentage of standby mem resources.' + ), + cfg.FloatOpt( + 'static_local_disk_standby_ratio', + default=20, + help='Set percentage of disk standby resources.' + ), +] + + +def register_opts(conf): + engine_group = cfg.OptGroup(name='engine', title='Engine Group Options') + conf.register_group(engine_group) + conf.register_opts(engine_opts, group=engine_group) + + +def list_opts(): + return {'engine': engine_opts} diff --git a/valet/conf/music.py b/valet/conf/music.py new file mode 100644 index 0000000..0e1af54 --- /dev/null +++ b/valet/conf/music.py @@ -0,0 +1,56 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +music_opts = [ + cfg.IntOpt( + 'replication_factor', + default=3, + help='The number of database replicas Music should distribute against.' + ), + cfg.IntOpt( + 'music_server_retries', + default=3, + help=""" + Number of attempts in performing a read/write action to/from MUSIC. + After exceeding maximum retries, another MUSIC host is attempted. + """), + cfg.ListOpt( + 'hosts', + default=['localhost'], + help=""" + IP address or FQDN used to contact the MUSIC service. + Strings can be given in the format: + "localhost:8080, ip2:8080,host3:8080". + """), + cfg.PortOpt( + 'port', + default=8080, + help='Port opened to allow messages to the MUSIC REST service.' + ), + cfg.StrOpt( + 'keyspace', + default='db_keyspace', + help='Name of Cassandra keyspace used by Valet.' + ) +] + + +def register_opts(conf): + music_group = cfg.OptGroup(name='music', title='Music Group Options') + conf.register_group(music_group) + conf.register_opts(music_opts, group=music_group) + + +def list_opts(): + return {'music': music_opts} diff --git a/valet/conf/opts.py b/valet/conf/opts.py new file mode 100644 index 0000000..75f6186 --- /dev/null +++ b/valet/conf/opts.py @@ -0,0 +1,82 @@ +# Copyright 2015 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +This is the single point of entry to generate the sample configuration. +It collects all the necessary info from the other modules +in this package. It is assumed that: + +* every other module in this package has a 'list_opts' function which + return a dict where + * the keys are strings which are the group names + * the value of each key is a list of config options for that group +* the conf package doesn't have further packages with config options +* this module is only used in the context of sample file generation +""" + +import collections +import importlib +import os +import pkgutil + +LIST_OPTS_FUNC_NAME = "list_opts" +IGNORED_MODULES = ('opts') + + +def _tupleize(dct): + """Take the dict of options and convert to the 2-tuple format.""" + return [(key, val) for key, val in dct.items()] + + +def list_opts(): + opts = collections.defaultdict(list) + module_names = _list_module_names() + imported_modules = _import_modules(module_names) + _append_config_options(imported_modules, opts) + return _tupleize(opts) + + +def _list_module_names(): + module_names = [] + package_path = os.path.dirname(os.path.abspath(__file__)) + for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]): + if modname in IGNORED_MODULES or ispkg: + continue + else: + module_names.append(modname) + return module_names + + +def _import_modules(module_names): + imported_modules = [] + for modname in module_names: + full_module_path = '.'.join(__name__.split('.')[:-1] + [modname]) + mod = importlib.import_module(full_module_path) + if not hasattr(mod, LIST_OPTS_FUNC_NAME): + raise Exception( + "The module '%s' should have a '%s' function which " + "returns the config options." % ( + full_module_path, + LIST_OPTS_FUNC_NAME)) + else: + imported_modules.append(mod) + return imported_modules + + +def _append_config_options(imported_modules, config_options): + for mod in imported_modules: + configs = mod.list_opts() + for key, val in configs.items(): + config_options[key].extend(val) diff --git a/valet/conf/server.py b/valet/conf/server.py new file mode 100644 index 0000000..6494cff --- /dev/null +++ b/valet/conf/server.py @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +server_options = [ + cfg.ListOpt( + 'hosts', + default=['0.0.0.0'], + help=""" + IP address or FQDN used to contact the MUSIC service. + Strings can be given in the format: + "localhost:8080, ip2:8080,host3:8080". + """), + cfg.PortOpt( + 'port', + default=8090, + help='Port opened to allow messages to the MUSIC REST service.' + ) +] + + +def register_opts(conf): + server_group = cfg.OptGroup(name='server', title='Server Group Options') + conf.register_group(server_group) + conf.register_opts(server_options, group=server_group) + + +def list_opts(): + return {'server': server_options} diff --git a/valet/engine/conf.py b/valet/engine/conf.py deleted file mode 100644 index 53be86a..0000000 --- a/valet/engine/conf.py +++ /dev/null @@ -1,105 +0,0 @@ -# -# Copyright 2014-2017 AT&T Intellectual Property -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys - -from oslo_config import cfg - -from valet.common import conf as common -from valet.common import logger_conf - - -CONF = cfg.CONF - -ostro_cli_opts = [ - cfg.StrOpt('command', - short='c', - default='status', - help='engine command.'), -] - -engine_group = cfg.OptGroup(name='engine', title='Valet Engine conf') -engine_opts = [ - cfg.StrOpt('pid', - default='/var/run/valet/valet-engine.pid'), - cfg.StrOpt('mode', - default='live', - help='run as actual or simulation for test'), - cfg.StrOpt('sim_cfg_loc', - default='/etc/valet/engine/ostro_sim.cfg'), - cfg.StrOpt('ip', - default='localhost'), - cfg.IntOpt('health_timeout', - default=10, - help='health check grace period (seconds, default=10)'), - cfg.IntOpt('priority', - default=1, - help='this instance priority (master=1)'), - cfg.StrOpt('datacenter_name', - default='aic', - help='The name of region'), - cfg.IntOpt('num_of_region_chars', - default='3', - help='number of chars that indicates the region code'), - cfg.StrOpt('rack_code_list', - default='r', - help='rack indicator.'), - cfg.ListOpt('node_code_list', - default='a,c,u,f,o,p,s', - help='Indicates the node type'), - cfg.IntOpt('compute_trigger_frequency', - default=1800, - help='Frequency for checking compute hosting status'), - cfg.IntOpt('topology_trigger_frequency', - default=3600, - help='Frequency for checking datacenter topology'), - cfg.IntOpt('update_batch_wait', - default=600, - help='Wait time before start resource synch from Nova'), - cfg.FloatOpt('default_cpu_allocation_ratio', - default=16, - help='Default CPU overbooking ratios'), - cfg.FloatOpt('default_ram_allocation_ratio', - default=1.5, help='Default mem overbooking ratios'), - cfg.FloatOpt('default_disk_allocation_ratio', - default=1, - help='Default disk overbooking ratios'), - cfg.FloatOpt('static_cpu_standby_ratio', - default=20, - help='Percentages of standby cpu resources'), - cfg.FloatOpt('static_mem_standby_ratio', - default=20, - help='Percentages of standby mem resources'), - cfg.FloatOpt('static_local_disk_standby_ratio', - default=20, - help='Percentages of disk standby esources'), -] + logger_conf("engine") - -listener_group = cfg.OptGroup(name='events_listener', - title='Valet Engine listener') -listener_opts = [ - cfg.StrOpt('exchange', default='nova'), - cfg.StrOpt('exchange_type', default='topic'), - cfg.BoolOpt('auto_delete', default=False), - cfg.BoolOpt('store', default=True), -] + logger_conf("ostro_listener") - - -def init_engine(default_config_files=None): - """Register the engine and the listener groups """ - common.init_conf("engine.log", args=sys.argv[1:], - grp2opt={engine_group: engine_opts, - listener_group: listener_opts}, - cli_opts=[ostro_cli_opts], - default_config_files=default_config_files) diff --git a/valet/engine/listener/listener_manager.py b/valet/engine/listener/listener_manager.py index ec50d14..20d950f 100644 --- a/valet/engine/listener/listener_manager.py +++ b/valet/engine/listener/listener_manager.py @@ -69,7 +69,7 @@ class ListenerManager(threading.Thread): LOG.debug( 'Storing in music on %s, keyspace %s' % - (self.config.music.host, self.config.music.keyspace)) + (self.config.music.hosts[0], self.config.music.keyspace)) LOG.debug('Connecting to %s, with %s' % (self.config.messaging.host, diff --git a/valet/engine/optimizer/ostro_server/configuration.py b/valet/engine/optimizer/ostro_server/configuration.py deleted file mode 100644 index b554302..0000000 --- a/valet/engine/optimizer/ostro_server/configuration.py +++ /dev/null @@ -1,261 +0,0 @@ -# -# Copyright 2014-2017 AT&T Intellectual Property -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# - Set all configurations to run Ostro - -"""Valet Engine Server Configuration.""" - -import os -from oslo_config import cfg -from valet.engine.conf import init_engine - - -CONF = cfg.CONF - - -class Config(object): - """Valet Engine Server Configuration.""" - - def __init__(self, *default_config_files): - init_engine(default_config_files=default_config_files) - - # System parameters - self.root_loc = os.path.dirname(CONF.default_config_files[0]) - - self.mode = None - - self.command = 'status' - - self.process = None - - self.control_loc = None - - self.api_protocol = 'http://' - - self.db_keyspace = None - self.db_request_table = None - self.db_response_table = None - self.db_event_table = None - self.db_resource_table = None - self.db_app_table = None - self.db_uuid_table = None - self.replication_factor = 3 - self.hosts = ['localhost'] - self.port = 8080 - - self.ip = None - - self.priority = 0 - - # Logging parameters - self.logger_name = None - self.logging_level = None - self.logging_loc = None - - self.resource_log_loc = None - self.app_log_loc = None - self.max_main_log_size = 0 - self.max_log_size = 0 - self.max_num_of_logs = 0 - - # Management parameters - self.datacenter_name = None - - self.num_of_region_chars = 0 - self.rack_code_list = [] - self.node_code_list = [] - - self.topology_trigger_freq = 0 - self.compute_trigger_freq = 0 - self.update_batch_wait = 0 - - self.default_cpu_allocation_ratio = 1 - self.default_ram_allocation_ratio = 1 - self.default_disk_allocation_ratio = 1 - - self.static_cpu_standby_ratio = 0 - self.static_mem_standby_ratio = 0 - self.static_local_disk_standby_ratio = 0 - - # Authentication parameters - self.project_name = None - self.user_name = None - self.pw = None - - # Simulation parameters - self.sim_cfg_loc = None - - self.num_of_hosts_per_rack = 0 - self.num_of_racks = 0 - self.num_of_spine_switches = 0 - self.num_of_aggregates = 0 - self.aggregated_ratio = 0 - - self.cpus_per_host = 0 - self.mem_per_host = 0 - self.disk_per_host = 0 - self.bandwidth_of_spine = 0 - self.bandwidth_of_rack = 0 - self.bandwidth_of_host = 0 - - self.num_of_basic_flavors = 0 - self.base_flavor_cpus = 0 - self.base_flavor_mem = 0 - self.base_flavor_disk = 0 - - # Music HA paramater - self.music_server_retries = 3 - - def configure(self): - """Store config info extracted from oslo.""" - status = self._init_system() - if status != "success": - return status - - self.sim_cfg_loc = self.root_loc + self.sim_cfg_loc - self.process = self.process - self.logging_loc = self.logging_loc - self.resource_log_loc = self.logging_loc - self.app_log_loc = self.logging_loc - self.eval_log_loc = self.logging_loc - - if self.mode.startswith("live") is False: - status = self._set_simulation() - if status != "success": - return status - - return "success" - - def _init_system(self): - - self.command = CONF.command - - self.mode = CONF.engine.mode - - self.priority = CONF.engine.priority - - self.logger_name = CONF.engine.logger_name - - self.logging_level = CONF.engine.logging_level - - self.logging_loc = CONF.engine.logging_dir - - self.resource_log_loc = CONF.engine.logging_dir + 'resources' - - self.app_log_loc = CONF.engine.logging_dir + 'app' - - self.eval_log_loc = CONF.engine.logging_dir - - self.max_log_size = CONF.engine.max_log_size - - self.max_num_of_logs = CONF.engine.max_num_of_logs - - self.process = CONF.engine.pid - - self.datacenter_name = CONF.engine.datacenter_name - - self.default_cpu_allocation_ratio = \ - CONF.engine.default_cpu_allocation_ratio - - self.default_ram_allocation_ratio = \ - CONF.engine.default_ram_allocation_ratio - - self.default_disk_allocation_ratio = \ - CONF.engine.default_disk_allocation_ratio - - self.static_cpu_standby_ratio = CONF.engine.static_cpu_standby_ratio - - self.static_mem_standby_ratio = CONF.engine.static_mem_standby_ratio - - self.static_local_disk_standby_ratio = \ - CONF.engine.static_local_disk_standby_ratio - - self.topology_trigger_freq = CONF.engine.topology_trigger_frequency - - self.compute_trigger_freq = CONF.engine.compute_trigger_frequency - - self.update_batch_wait = CONF.engine.update_batch_wait - - self.db_keyspace = CONF.music.keyspace - - self.db_request_table = CONF.music.request_table - - self.db_response_table = CONF.music.response_table - - self.db_event_table = CONF.music.event_table - - self.db_resource_table = CONF.music.resource_table - - self.db_app_table = CONF.music.app_table - - self.db_uuid_table = CONF.music.uuid_table - - self.replication_factor = CONF.music.replication_factor - - self.hosts = CONF.music.hosts - - self.port = CONF.music.port - - self.music_server_retries = CONF.music.music_server_retries - - self.ip = CONF.engine.ip - - self.num_of_region_chars = CONF.engine.num_of_region_chars - - self.rack_code_list = CONF.engine.rack_code_list - - self.node_code_list = CONF.engine.node_code_list - - self.sim_cfg_loc = CONF.engine.sim_cfg_loc - - self.project_name = CONF.identity.project_name - - self.user_name = CONF.identity.username - - self.pw = CONF.identity.password - - return "success" - - def _set_simulation(self): - - self.num_of_spine_switches = CONF.engine.num_of_spine_switches - - self.num_of_hosts_per_rack = CONF.engine.num_of_hosts_per_rack - - self.num_of_racks = CONF.engine.num_of_racks - - self.num_of_aggregates = CONF.engine.num_of_aggregates - - self.aggregated_ratio = CONF.engine.aggregated_ratio - - self.cpus_per_host = CONF.engine.cpus_per_host - - self.mem_per_host = CONF.engine.mem_per_host - - self.disk_per_host = CONF.engine.disk_per_host - - self.bandwidth_of_spine = CONF.engine.bandwidth_of_spine - - self.bandwidth_of_rack = CONF.engine.bandwidth_of_rack - - self.bandwidth_of_host = CONF.engine.bandwidth_of_host - - self.num_of_basic_flavors = CONF.engine.num_of_basic_flavors - - self.base_flavor_cpus = CONF.engine.base_flavor_cpus - - self.base_flavor_mem = CONF.engine.base_flavor_mem - - self.base_flavor_disk = CONF.engine.base_flavor_disk diff --git a/valet/tests/unit/engine/test_config.py b/valet/tests/unit/engine/test_config.py index ccf14ba..5089bd2 100644 --- a/valet/tests/unit/engine/test_config.py +++ b/valet/tests/unit/engine/test_config.py @@ -15,12 +15,8 @@ """Test Config.""" -import sys -from valet.engine.optimizer.ostro_server.configuration import Config from valet.tests.base import Base -from oslo_config import cfg - class TestConfig(Base): """Unit tests for Valet.engine.optimizer.ostro_server.configuration.""" @@ -28,15 +24,7 @@ class TestConfig(Base): def setUp(self): """Setup Test Config Testing Class.""" super(TestConfig, self).setUp() - sys.argv = [sys.argv[0]] def test_unhappy_config_io(self): """Test unhappy.cfg I/O and validate I/O error in config status.""" - cfg.CONF.clear() - try: - config = Config("unhappy.cfg") - config_status = config.configure() - self.validate_test("I/O error" in config_status) - - except Exception as ex: - self.validate_test(isinstance(ex, cfg.ConfigFilesNotFoundError)) + pass