x/valet
Code Issues Proposed changes

Centralized configuration and auto generation

Browse Source 
This patch set centralizes all oslo configuration setting(s) into a single
module and provides necessary entrypoints for the configuration
to be generated via ``tox -e genconfig``

Story: #2001045
Task: #4619

Change-Id: I0473c7176f412d176be24deb588f7ef4c113848a
Co-Authored-By: Tin Lam <tin@irrational.io>
This commit is contained in:
Jenna Kleiman 2017-06-04 07:32:49 -05:00
parent 6fe9c32e4a
commit 224f1be6fd
21 changed files with 822 additions and 505 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
etc/valet/valet-config-generator.conf Normal file
View File

@ -0,0 +1,7 @@
[DEFAULT]
output_file = etc/valet/valet.conf.sample
wrap_width = 79
namespace = valet.conf
namespace = oslo.log
namespace = oslo.db
namespace = keystonemiddleware.auth_token

0
etc/valet/valet.conf → etc/valet/valet.backup
View File

480
etc/valet/valet.conf.sample Normal file
View File

@ -0,0 +1,480 @@
[DEFAULT]
#
# From oslo.log
#
# If set to true, the logging level will be set to DEBUG instead of the default
# INFO level. (boolean value)
# Note: This option can be changed without restarting.
#debug = false
# The name of a logging configuration file. This file is appended to any
# existing logging configuration files. For details about logging configuration
# files, see the Python logging module documentation. Note that when logging
# configuration files are used then all logging configuration is set in the
# configuration file and other logging configuration options are ignored (for
# example, logging_context_format_string). (string value)
# Note: This option can be changed without restarting.
# Deprecated group/name - [DEFAULT]/log_config
#log_config_append = <None>
# Defines the format string for %%(asctime)s in log records. Default:
# %(default)s . This option is ignored if log_config_append is set. (string
# value)
#log_date_format = %Y-%m-%d %H:%M:%S
# (Optional) Name of log file to send logging output to. If no default is set,
# logging will go to stderr as defined by use_stderr. This option is ignored if
# log_config_append is set. (string value)
# Deprecated group/name - [DEFAULT]/logfile
#log_file = <None>
# (Optional) The base directory used for relative log_file paths. This option
# is ignored if log_config_append is set. (string value)
# Deprecated group/name - [DEFAULT]/logdir
#log_dir = <None>
# Uses logging handler designed to watch file system. When log file is moved or
# removed this handler will open a new log file with specified path
# instantaneously. It makes sense only if log_file option is specified and
# Linux platform is used. This option is ignored if log_config_append is set.
# (boolean value)
#watch_log_file = false
# Use syslog for logging. Existing syslog format is DEPRECATED and will be
# changed later to honor RFC5424. This option is ignored if log_config_append
# is set. (boolean value)
#use_syslog = false
# Enable journald for logging. If running in a systemd environment you may wish
# to enable journal support. Doing so will use the journal native protocol
# which includes structured metadata in addition to log messages.This option is
# ignored if log_config_append is set. (boolean value)
#use_journal = false
# Syslog facility to receive log lines. This option is ignored if
# log_config_append is set. (string value)
#syslog_log_facility = LOG_USER
# Log output to standard error. This option is ignored if log_config_append is
# set. (boolean value)
#use_stderr = false
# Format string to use for log messages with context. (string value)
#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s
# Format string to use for log messages when context is undefined. (string
# value)
#logging_default_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s
# Additional data to append to log message when logging level for the message
# is DEBUG. (string value)
#logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d
# Prefix each line of exception output with this format. (string value)
#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d ERROR %(name)s %(instance)s
# Defines the format string for %(user_identity)s that is used in
# logging_context_format_string. (string value)
#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s
# List of package logging levels in logger=LEVEL pairs. This option is ignored
# if log_config_append is set. (list value)
#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,oslo_messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN,oslo.cache=INFO,dogpile.core.dogpile=INFO
# Enables or disables publication of error events. (boolean value)
#publish_errors = false
# The format for an instance that is passed with the log message. (string
# value)
#instance_format = "[instance: %(uuid)s] "
# The format for an instance UUID that is passed with the log message. (string
# value)
#instance_uuid_format = "[instance: %(uuid)s] "
# Interval, number of seconds, of log rate limiting. (integer value)
#rate_limit_interval = 0
# Maximum number of logged messages per rate_limit_interval. (integer value)
#rate_limit_burst = 0
# Log level name used by rate limiting: CRITICAL, ERROR, INFO, WARNING, DEBUG
# or empty string. Logs with level greater or equal to rate_limit_except_level
# are not filtered. An empty string means that all levels are filtered. (string
# value)
#rate_limit_except_level = CRITICAL
# Enables or disables fatal status of deprecations. (boolean value)
#fatal_deprecations = false
[database]
#
# From oslo.db
#
# If True, SQLite uses synchronous mode. (boolean value)
#sqlite_synchronous = true
# The back end to use for the database. (string value)
# Deprecated group/name - [DEFAULT]/db_backend
#backend = sqlalchemy
# The SQLAlchemy connection string to use to connect to the database. (string
# value)
# Deprecated group/name - [DEFAULT]/sql_connection
# Deprecated group/name - [DATABASE]/sql_connection
# Deprecated group/name - [sql]/connection
#connection = <None>
# The SQLAlchemy connection string to use to connect to the slave database.
# (string value)
#slave_connection = <None>
# The SQL mode to be used for MySQL sessions. This option, including the
# default, overrides any server-set SQL mode. To use whatever SQL mode is set
# by the server configuration, set this to no value. Example: mysql_sql_mode=
# (string value)
#mysql_sql_mode = TRADITIONAL
# Timeout before idle SQL connections are reaped. (integer value)
# Deprecated group/name - [DEFAULT]/sql_idle_timeout
# Deprecated group/name - [DATABASE]/sql_idle_timeout
# Deprecated group/name - [sql]/idle_timeout
#idle_timeout = 3600
# Minimum number of SQL connections to keep open in a pool. (integer value)
# Deprecated group/name - [DEFAULT]/sql_min_pool_size
# Deprecated group/name - [DATABASE]/sql_min_pool_size
#min_pool_size = 1
# Maximum number of SQL connections to keep open in a pool. Setting a value of
# 0 indicates no limit. (integer value)
# Deprecated group/name - [DEFAULT]/sql_max_pool_size
# Deprecated group/name - [DATABASE]/sql_max_pool_size
#max_pool_size = 5
# Maximum number of database connection retries during startup. Set to -1 to
# specify an infinite retry count. (integer value)
# Deprecated group/name - [DEFAULT]/sql_max_retries
# Deprecated group/name - [DATABASE]/sql_max_retries
#max_retries = 10
# Interval between retries of opening a SQL connection. (integer value)
# Deprecated group/name - [DEFAULT]/sql_retry_interval
# Deprecated group/name - [DATABASE]/reconnect_interval
#retry_interval = 10
# If set, use this value for max_overflow with SQLAlchemy. (integer value)
# Deprecated group/name - [DEFAULT]/sql_max_overflow
# Deprecated group/name - [DATABASE]/sqlalchemy_max_overflow
#max_overflow = 50
# Verbosity of SQL debugging information: 0=None, 100=Everything. (integer
# value)
# Minimum value: 0
# Maximum value: 100
# Deprecated group/name - [DEFAULT]/sql_connection_debug
#connection_debug = 0
# Add Python stack traces to SQL as comment strings. (boolean value)
# Deprecated group/name - [DEFAULT]/sql_connection_trace
#connection_trace = false
# If set, use this value for pool_timeout with SQLAlchemy. (integer value)
# Deprecated group/name - [DATABASE]/sqlalchemy_pool_timeout
#pool_timeout = <None>
# Enable the experimental use of database reconnect on connection lost.
# (boolean value)
#use_db_reconnect = false
# Seconds between retries of a database transaction. (integer value)
#db_retry_interval = 1
# If True, increases the interval between retries of a database operation up to
# db_max_retry_interval. (boolean value)
#db_inc_retry_interval = true
# If db_inc_retry_interval is set, the maximum seconds between retries of a
# database operation. (integer value)
#db_max_retry_interval = 10
# Maximum retries in case of connection error or deadlock error before error is
# raised. Set to -1 to specify an infinite retry count. (integer value)
#db_max_retries = 20
[engine]
#
# From valet.conf
#
# Set path to Valet Engine running process pid. (string value)
#pid = /var/run/valet/valet-engine.pid
#
# Inform the name of datacenter (region name),
# where Valet is deployed.
# (string value)
#datacenter_name = Region1
# Set frequency, in seconds, for checking compute host status. (integer value)
#compute_trigger_frequency = 1800
#
# Set trigger time or frequency, in seconds,
# for checking datacenter topology.
# (integer value)
#topology_trigger_frequency = 1800
#
# Set default CPU overbooking ratio.
# Note that each compute node can have its own ratio.
# (floating point value)
#default_cpu_allocation_ratio = 16
# Set default RAM overbooking ratio. (floating point value)
#default_ram_allocation_ratio = 1.5
# Set default disk overbooking ratio. (floating point value)
#default_disk_allocation_ratio = 1
# Set percentage of standby CPU resources. (floating point value)
#static_cpu_standby_ratio = 20
# Set percentage of standby mem resources. (floating point value)
#static_mem_standby_ratio = 20
# Set percentage of disk standby resources. (floating point value)
#static_local_disk_standby_ratio = 20
[keystone_authtoken]
#
# From keystonemiddleware.auth_token
#
# Complete "public" Identity API endpoint. This endpoint should not be an
# "admin" endpoint, as it should be accessible by all end users.
# Unauthenticated clients are redirected to this endpoint to authenticate.
# Although this endpoint should ideally be unversioned, client support in the
# wild varies. If you're using a versioned v2 endpoint here, then this should
# *not* be the same endpoint the service user utilizes for validating tokens,
# because normal end users may not be able to reach that endpoint. (string
# value)
#auth_uri = <None>
# API version of the admin Identity API endpoint. (string value)
#auth_version = <None>
# Do not handle authorization requests within the middleware, but delegate the
# authorization decision to downstream WSGI components. (boolean value)
#delay_auth_decision = false
# Request timeout value for communicating with Identity API server. (integer
# value)
#http_connect_timeout = <None>
# How many times are we trying to reconnect when communicating with Identity
# API Server. (integer value)
#http_request_max_retries = 3
# Request environment key where the Swift cache object is stored. When
# auth_token middleware is deployed with a Swift cache, use this option to have
# the middleware share a caching backend with swift. Otherwise, use the
# ``memcached_servers`` option instead. (string value)
#cache = <None>
# Required if identity server requires client certificate (string value)
#certfile = <None>
# Required if identity server requires client certificate (string value)
#keyfile = <None>
# A PEM encoded Certificate Authority to use when verifying HTTPs connections.
# Defaults to system CAs. (string value)
#cafile = <None>
# Verify HTTPS connections. (boolean value)
#insecure = false
# The region in which the identity server can be found. (string value)
#region_name = <None>
# DEPRECATED: Directory used to cache files related to PKI tokens. This option
# has been deprecated in the Ocata release and will be removed in the P
# release. (string value)
# This option is deprecated for removal since Ocata.
# Its value may be silently ignored in the future.
# Reason: PKI token format is no longer supported.
#signing_dir = <None>
# Optionally specify a list of memcached server(s) to use for caching. If left
# undefined, tokens will instead be cached in-process. (list value)
# Deprecated group/name - [keystone_authtoken]/memcache_servers
#memcached_servers = <None>
# In order to prevent excessive effort spent validating tokens, the middleware
# caches previously-seen tokens for a configurable duration (in seconds). Set
# to -1 to disable caching completely. (integer value)
#token_cache_time = 300
# DEPRECATED: Determines the frequency at which the list of revoked tokens is
# retrieved from the Identity service (in seconds). A high number of revocation
# events combined with a low cache duration may significantly reduce
# performance. Only valid for PKI tokens. This option has been deprecated in
# the Ocata release and will be removed in the P release. (integer value)
# This option is deprecated for removal since Ocata.
# Its value may be silently ignored in the future.
# Reason: PKI token format is no longer supported.
#revocation_cache_time = 10
# (Optional) If defined, indicate whether token data should be authenticated or
# authenticated and encrypted. If MAC, token data is authenticated (with HMAC)
# in the cache. If ENCRYPT, token data is encrypted and authenticated in the
# cache. If the value is not one of these options or empty, auth_token will
# raise an exception on initialization. (string value)
# Allowed values: None, MAC, ENCRYPT
#memcache_security_strategy = None
# (Optional, mandatory if memcache_security_strategy is defined) This string is
# used for key derivation. (string value)
#memcache_secret_key = <None>
# (Optional) Number of seconds memcached server is considered dead before it is
# tried again. (integer value)
#memcache_pool_dead_retry = 300
# (Optional) Maximum total number of open connections to every memcached
# server. (integer value)
#memcache_pool_maxsize = 10
# (Optional) Socket timeout in seconds for communicating with a memcached
# server. (integer value)
#memcache_pool_socket_timeout = 3
# (Optional) Number of seconds a connection to memcached is held unused in the
# pool before it is closed. (integer value)
#memcache_pool_unused_timeout = 60
# (Optional) Number of seconds that an operation will wait to get a memcached
# client connection from the pool. (integer value)
#memcache_pool_conn_get_timeout = 10
# (Optional) Use the advanced (eventlet safe) memcached client pool. The
# advanced pool will only work under python 2.x. (boolean value)
#memcache_use_advanced_pool = false
# (Optional) Indicate whether to set the X-Service-Catalog header. If False,
# middleware will not ask for service catalog on token validation and will not
# set the X-Service-Catalog header. (boolean value)
#include_service_catalog = true
# Used to control the use and type of token binding. Can be set to: "disabled"
# to not check token binding. "permissive" (default) to validate binding
# information if the bind type is of a form known to the server and ignore it
# if not. "strict" like "permissive" but if the bind type is unknown the token
# will be rejected. "required" any form of token binding is needed to be
# allowed. Finally the name of a binding method that must be present in tokens.
# (string value)
#enforce_token_bind = permissive
# DEPRECATED: If true, the revocation list will be checked for cached tokens.
# This requires that PKI tokens are configured on the identity server. (boolean
# value)
# This option is deprecated for removal since Ocata.
# Its value may be silently ignored in the future.
# Reason: PKI token format is no longer supported.
#check_revocations_for_cached = false
# DEPRECATED: Hash algorithms to use for hashing PKI tokens. This may be a
# single algorithm or multiple. The algorithms are those supported by Python
# standard hashlib.new(). The hashes will be tried in the order given, so put
# the preferred one first for performance. The result of the first hash will be
# stored in the cache. This will typically be set to multiple values only while
# migrating from a less secure algorithm to a more secure one. Once all the old
# tokens are expired this option should be set to a single value for better
# performance. (list value)
# This option is deprecated for removal since Ocata.
# Its value may be silently ignored in the future.
# Reason: PKI token format is no longer supported.
#hash_algorithms = md5
# A choice of roles that must be present in a service token. Service tokens are
# allowed to request that an expired token can be used and so this check should
# tightly control that only actual services should be sending this token. Roles
# here are applied as an ANY check so any role in this list must be present.
# For backwards compatibility reasons this currently only affects the
# allow_expired check. (list value)
#service_token_roles = service
# For backwards compatibility reasons we must let valid service tokens pass
# that don't pass the service_token_roles check as valid. Setting this true
# will become the default in a future release and should be enabled if
# possible. (boolean value)
#service_token_roles_required = false
# Authentication type to load (string value)
# Deprecated group/name - [keystone_authtoken]/auth_plugin
#auth_type = <None>
# Config Section from which to load plugin specific options (string value)
#auth_section = <None>
[music]
#
# From valet.conf
#
# The number of database replicas Music should distribute against. (integer
# value)
#replication_factor = 3
#
# Number of attempts in performing a read/write action to/from
# MUSIC.
# After exceeding maximum retries, another MUSIC host is attempted.
# (integer value)
#music_server_retries = 3
#
# IP address or FQDN used to contact the MUSIC service.
# Strings can be given in the format "localhost:8080,
# ip2:8080,host3:8080".
# (list value)
#hosts = localhost
# Port opened to allow messages to the MUSIC REST service. (port value)
# Minimum value: 0
# Maximum value: 65535
#port = 8080
# Name of Cassandra keyspace used by Valet. (string value)
#keyspace = db_keyspace
[server]
#
# From valet.conf
#
#
# IP address or FQDN used to contact the MUSIC service.
# Strings can be given in the format "localhost:8080,
# ip2:8080,host3:8080".
# (list value)
#hosts = 0.0.0.0
# Port opened to allow messages to the MUSIC REST service. (port value)
# Minimum value: 0
# Maximum value: 65535
#port = 8090

2
requirements.txt
View File

@ -15,12 +15,14 @@ pytz>=2013.6 # MIT
python-keystoneclient>=3.8.0 # Apache-2.0
python-heatclient>=1.6.1 # Apache-2.0
python-novaclient>=7.1.0 # Apache-2.0
keystonemiddleware>=4.12.0 # Apache-2.0
oslo.config>=3.22.0 # Apache-2.0
oslo.db>=4.21.1 # Apache-2.0
oslo.i18n!=3.15.2,>=2.1.0 # Apache-2.0
oslo.log>=3.22.0 # Apache-2.0
oslo.messaging>=5.19.0 # Apache-2.0
oslo.middleware>=3.27.0 # Apache-2.0
oslo.serialization>=1.10.0 # Apache-2.0
oslo.utils>=3.20.0 # Apache-2.0

7
setup.cfg
View File

@ -33,7 +33,14 @@ packages = valet
data_files = etc/valet/ = etc/*
[entry_points]
oslo.config.opts =
valet.conf = valet.conf.opts:list_opts
oslo.config.opts.defaults=
valet.conf = valet.common.config:set_middleware_defaults
pecan.command =
populate = valet.api.v1.commands.populate:PopulateCommand
tempest.test_plugins =
valet_tests = valet.tests.tempest.plugin:ValetTempestPlugin

9
tox.ini
View File

@ -13,7 +13,7 @@ commands =
ostestr --slowest '{posargs}'
deps = -r{toxinidir}/test-requirements.txt
whitelist_externals =
find
find
[testenv:pep8]
basepython = python2.7
@ -38,7 +38,9 @@ commands =
coverage report
[testenv:docs]
commands = python setup.py build_sphinx
basepython = python2.7
commands =
python setup.py build_sphinx
[testenv:releasenotes]
commands = sphinx-build -a -W -E -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html
@ -51,6 +53,9 @@ basepython = python2.7
commands =
sphinx-build -W -b html -d api-ref/build/doctrees api-ref/source api-ref/build/html
[testenv:genconfig]
commands = oslo-config-generator --config-file=etc/valet/valet-config-generator.conf
[flake8]
filename = *.py
show-source = true

2
valet/api/app.py
View File

@ -23,7 +23,6 @@ from valet.api.common import identity
from valet.api.common import messaging
from valet.api.db import models
from valet.common.conf import get_logger
from valet.common.conf import init_conf
def setup_app(config):
@ -41,7 +40,6 @@ def setup_app(config):
# entry point for apache2
def load_app(config_file):
"""App Load."""
init_conf("api.log")
api.LOG = get_logger("api")
return deploy(config_file)

2
valet/api/v1/commands/populate.py
View File

@ -24,7 +24,6 @@ from valet.api.db.models.music.ostro import PlacementResult
from valet.api.db.models.music.placements import Placement
from valet.api.db.models.music.plans import Plan
from valet.common.conf import get_logger
from valet.common.conf import init_conf
class PopulateCommand(BaseCommand):
@ -34,7 +33,6 @@ class PopulateCommand(BaseCommand):
"""Function creates and initializes database and environment."""
super(PopulateCommand, self).run(args)
try:
init_conf("populate.log")
LOG = api.LOG = get_logger("populate")
LOG.info(_("Loading environment"))
self.load_app()

2
valet/api/wsgi.py
View File

@ -19,7 +19,6 @@ from common.i18n import _
from valet import api
from valet.common.conf import get_logger
from valet.common.conf import init_conf
def config_file(file_name=None):
@ -48,7 +47,6 @@ if __name__ == '__main__':
# import pecan_mount
# HTTPD = make_server('', 8090, pecan_mount.tree)
# from valet.api.conf import register_conf, set_domain
init_conf("wsgi.log")
api.LOG = get_logger("wsgi")
HTTPD = make_server(
'', 8090, deploy(config_file('/var/www/valet/config.py')))

4
valet/cli/groupcli.py
View File

@ -19,7 +19,6 @@ import argparse
import json
from oslo_config import cfg
import requests
from valet.common import conf as common
CONF = cfg.CONF
@ -205,8 +204,7 @@ def populate_args_request_body(args):
def run(args):
"""Run."""
common.init_conf("cli.log")
args.host = args.host or CONF.server.host
args.host = args.host or CONF.server.hosts[0]
args.port = args.port or CONF.server.port
args.timeout = args.timeout or 10
rest_cmd, cmd_url = cmd_details(args)

114
valet/common/conf.py
View File

@ -1,114 +0,0 @@
#
# Copyright 2014-2017 AT&T Intellectual Property
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Conf."""
from oslo_config import cfg
from oslo_log import log as logging
CONF = cfg.CONF
DOMAIN = 'valet'
def get_logger(name):
return logging.getLogger(name)
LOG = get_logger("engine")
server_group = cfg.OptGroup(name='server', title='Valet API Server conf')
server_opts = [
cfg.StrOpt('host', default='0.0.0.0'),
cfg.StrOpt('port', default='8090'),
]
messaging_group = cfg.OptGroup(name='messaging', title='Valet Messaging conf')
messaging_opts = [
cfg.StrOpt('username'),
cfg.StrOpt('password'),
cfg.StrOpt('host'),
cfg.IntOpt('port', default=5672),
cfg.IntOpt('timeout', default=3),
]
identity_group = cfg.OptGroup(name='identity', title='Valet identity conf')
identity_opts = [
cfg.StrOpt('username'),
cfg.StrOpt('password'),
cfg.StrOpt('project_name'),
cfg.StrOpt('auth_url', default='http://controller:5000/v2.0'),
cfg.StrOpt('interface', default='admin'),
]
music_group = cfg.OptGroup(name='music', title='Valet Persistence conf')
music_opts = [
cfg.ListOpt('hosts', default=['0.0.0.0']),
cfg.IntOpt('port', default=8080),
cfg.StrOpt('keyspace', default='valet'),
cfg.IntOpt('replication_factor', default=3),
cfg.IntOpt('tries', default=10),
cfg.IntOpt('interval', default=1),
cfg.StrOpt('request_table', default='placement_requests'),
cfg.StrOpt('response_table', default='placement_results'),
cfg.StrOpt('event_table', default='oslo_messages'),
cfg.StrOpt('resource_table', default='resource_status'),
cfg.StrOpt('app_table', default='app'),
cfg.StrOpt('resource_index_table', default='resource_log_index'),
cfg.StrOpt('app_index_table', default='app_log_index'),
cfg.StrOpt('uuid_table', default='uuid_map'),
cfg.IntOpt('music_server_retries', default=3),
]
def load_conf(args=None, project=DOMAIN, default_files=None):
if default_files:
CONF(default_config_files=default_files)
else:
CONF(args or [], project=project)
def init_conf(log_file="valet.log", args=None, grp2opt=None,
cli_opts=None, default_config_files=None):
CONF.log_file = log_file
logging.register_options(CONF)
# init conf
general_groups = {server_group: server_opts,
music_group: music_opts,
identity_group: identity_opts,
messaging_group: messaging_opts}
general_groups.update(grp2opt or {})
_register_conf(general_groups, cli_opts)
load_conf(args=args, default_files=default_config_files)
# set logger
_set_logger()
def _set_logger():
logging.setup(CONF, DOMAIN)
def _register_conf(grp2opt, cli_opts):
for grp in grp2opt:
CONF.register_group(grp)
CONF.register_opts(grp2opt[grp], grp)
for opt in cli_opts or []:
CONF.register_cli_opts(opt)

33
valet/common/config.py Normal file
View File

@ -0,0 +1,33 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_middleware import cors
def set_middleware_defaults():
cors.set_defaults(
allow_headers=['X-Auth-Token',
'X-Openstack-Request-Id',
'X-Identity-Status',
'X-Roles',
'X-Service-Catalog',
'X-User-Id',
'X-Tenant-Id'],
expose_headers=['X-Auth-Token',
'X-Openstack-Request-Id',
'X-Subject-Token',
'X-Service-Token'],
allow_methods=['GET',
'PUT',
'POST',
'DELETE',
'PATCH'])

24
valet/conf/__init__.py Normal file
View File

@ -0,0 +1,24 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from valet.conf import engine
from valet.conf import music
from valet.conf import server
CONF = cfg.CONF
engine.register_opts(CONF)
music.register_opts(CONF)
server.register_opts(CONF)

82
valet/conf/engine.py Normal file
View File

@ -0,0 +1,82 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
engine_opts = [
cfg.StrOpt(
'pid',
default='/var/run/valet/valet-engine.pid',
help='Set path to Valet Engine running process pid.'
),
cfg.StrOpt(
'datacenter_name',
default='Region1',
help="""
Inform the name of datacenter (region name),
where Valet is deployed.
"""),
cfg.IntOpt(
'compute_trigger_frequency',
default=1800,
help='Set frequency, in seconds, for checking compute host status.'
),
cfg.IntOpt(
'topology_trigger_frequency',
default=1800,
help="""
Set trigger time or frequency, in seconds,
for checking datacenter topology.
"""),
cfg.FloatOpt(
'default_cpu_allocation_ratio',
default=16,
help="""
Set default CPU overbooking ratio.
Note that each compute node can have its own ratio.
"""),
cfg.FloatOpt(
'default_ram_allocation_ratio',
default=1.5,
help='Set default RAM overbooking ratio.'
),
cfg.FloatOpt(
'default_disk_allocation_ratio',
default=1,
help='Set default disk overbooking ratio.'
),
cfg.FloatOpt(
'static_cpu_standby_ratio',
default=20,
help='Set percentage of standby CPU resources.'
),
cfg.FloatOpt(
'static_mem_standby_ratio',
default=20,
help='Set percentage of standby mem resources.'
),
cfg.FloatOpt(
'static_local_disk_standby_ratio',
default=20,
help='Set percentage of disk standby resources.'
),
]
def register_opts(conf):
engine_group = cfg.OptGroup(name='engine', title='Engine Group Options')
conf.register_group(engine_group)
conf.register_opts(engine_opts, group=engine_group)
def list_opts():
return {'engine': engine_opts}

56
valet/conf/music.py Normal file
View File

@ -0,0 +1,56 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
music_opts = [
cfg.IntOpt(
'replication_factor',
default=3,
help='The number of database replicas Music should distribute against.'
),
cfg.IntOpt(
'music_server_retries',
default=3,
help="""
Number of attempts in performing a read/write action to/from MUSIC.
After exceeding maximum retries, another MUSIC host is attempted.
"""),
cfg.ListOpt(
'hosts',
default=['localhost'],
help="""
IP address or FQDN used to contact the MUSIC service.
Strings can be given in the format:
"localhost:8080, ip2:8080,host3:8080".
"""),
cfg.PortOpt(
'port',
default=8080,
help='Port opened to allow messages to the MUSIC REST service.'
),
cfg.StrOpt(
'keyspace',
default='db_keyspace',
help='Name of Cassandra keyspace used by Valet.'
)
]
def register_opts(conf):
music_group = cfg.OptGroup(name='music', title='Music Group Options')
conf.register_group(music_group)
conf.register_opts(music_opts, group=music_group)
def list_opts():
return {'music': music_opts}

82
valet/conf/opts.py Normal file
View File

@ -0,0 +1,82 @@
# Copyright 2015 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
This is the single point of entry to generate the sample configuration.
It collects all the necessary info from the other modules
in this package. It is assumed that:
* every other module in this package has a 'list_opts' function which
return a dict where
* the keys are strings which are the group names
* the value of each key is a list of config options for that group
* the conf package doesn't have further packages with config options
* this module is only used in the context of sample file generation
"""
import collections
import importlib
import os
import pkgutil
LIST_OPTS_FUNC_NAME = "list_opts"
IGNORED_MODULES = ('opts')
def _tupleize(dct):
"""Take the dict of options and convert to the 2-tuple format."""
return [(key, val) for key, val in dct.items()]
def list_opts():
opts = collections.defaultdict(list)
module_names = _list_module_names()
imported_modules = _import_modules(module_names)
_append_config_options(imported_modules, opts)
return _tupleize(opts)
def _list_module_names():
module_names = []
package_path = os.path.dirname(os.path.abspath(__file__))
for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]):
if modname in IGNORED_MODULES or ispkg:
continue
else:
module_names.append(modname)
return module_names
def _import_modules(module_names):
imported_modules = []
for modname in module_names:
full_module_path = '.'.join(__name__.split('.')[:-1] + [modname])
mod = importlib.import_module(full_module_path)
if not hasattr(mod, LIST_OPTS_FUNC_NAME):
raise Exception(
"The module '%s' should have a '%s' function which "
"returns the config options." % (
full_module_path,
LIST_OPTS_FUNC_NAME))
else:
imported_modules.append(mod)
return imported_modules
def _append_config_options(imported_modules, config_options):
for mod in imported_modules:
configs = mod.list_opts()
for key, val in configs.items():
config_options[key].extend(val)

39
valet/conf/server.py Normal file
View File

@ -0,0 +1,39 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
server_options = [
cfg.ListOpt(
'hosts',
default=['0.0.0.0'],
help="""
IP address or FQDN used to contact the MUSIC service.
Strings can be given in the format:
"localhost:8080, ip2:8080,host3:8080".
"""),
cfg.PortOpt(
'port',
default=8090,
help='Port opened to allow messages to the MUSIC REST service.'
)
]
def register_opts(conf):
server_group = cfg.OptGroup(name='server', title='Server Group Options')
conf.register_group(server_group)
conf.register_opts(server_options, group=server_group)
def list_opts():
return {'server': server_options}

105
valet/engine/conf.py
View File

@ -1,105 +0,0 @@
#
# Copyright 2014-2017 AT&T Intellectual Property
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
from oslo_config import cfg
from valet.common import conf as common
from valet.common import logger_conf
CONF = cfg.CONF
ostro_cli_opts = [
cfg.StrOpt('command',
short='c',
default='status',
help='engine command.'),
]
engine_group = cfg.OptGroup(name='engine', title='Valet Engine conf')
engine_opts = [
cfg.StrOpt('pid',
default='/var/run/valet/valet-engine.pid'),
cfg.StrOpt('mode',
default='live',
help='run as actual or simulation for test'),
cfg.StrOpt('sim_cfg_loc',
default='/etc/valet/engine/ostro_sim.cfg'),
cfg.StrOpt('ip',
default='localhost'),
cfg.IntOpt('health_timeout',
default=10,
help='health check grace period (seconds, default=10)'),
cfg.IntOpt('priority',
default=1,
help='this instance priority (master=1)'),
cfg.StrOpt('datacenter_name',
default='aic',
help='The name of region'),
cfg.IntOpt('num_of_region_chars',
default='3',
help='number of chars that indicates the region code'),
cfg.StrOpt('rack_code_list',
default='r',
help='rack indicator.'),
cfg.ListOpt('node_code_list',
default='a,c,u,f,o,p,s',
help='Indicates the node type'),
cfg.IntOpt('compute_trigger_frequency',
default=1800,
help='Frequency for checking compute hosting status'),
cfg.IntOpt('topology_trigger_frequency',
default=3600,
help='Frequency for checking datacenter topology'),
cfg.IntOpt('update_batch_wait',
default=600,
help='Wait time before start resource synch from Nova'),
cfg.FloatOpt('default_cpu_allocation_ratio',
default=16,
help='Default CPU overbooking ratios'),
cfg.FloatOpt('default_ram_allocation_ratio',
default=1.5, help='Default mem overbooking ratios'),
cfg.FloatOpt('default_disk_allocation_ratio',
default=1,
help='Default disk overbooking ratios'),
cfg.FloatOpt('static_cpu_standby_ratio',
default=20,
help='Percentages of standby cpu resources'),
cfg.FloatOpt('static_mem_standby_ratio',
default=20,
help='Percentages of standby mem resources'),
cfg.FloatOpt('static_local_disk_standby_ratio',
default=20,
help='Percentages of disk standby esources'),
] + logger_conf("engine")
listener_group = cfg.OptGroup(name='events_listener',
title='Valet Engine listener')
listener_opts = [
cfg.StrOpt('exchange', default='nova'),
cfg.StrOpt('exchange_type', default='topic'),
cfg.BoolOpt('auto_delete', default=False),
cfg.BoolOpt('store', default=True),
] + logger_conf("ostro_listener")
def init_engine(default_config_files=None):
"""Register the engine and the listener groups """
common.init_conf("engine.log", args=sys.argv[1:],
grp2opt={engine_group: engine_opts,
listener_group: listener_opts},
cli_opts=[ostro_cli_opts],
default_config_files=default_config_files)

2
valet/engine/listener/listener_manager.py
View File

@ -69,7 +69,7 @@ class ListenerManager(threading.Thread):
LOG.debug(
'Storing in music on %s, keyspace %s' %
(self.config.music.host, self.config.music.keyspace))
(self.config.music.hosts[0], self.config.music.keyspace))
LOG.debug('Connecting to %s, with %s' %
(self.config.messaging.host,

261
valet/engine/optimizer/ostro_server/configuration.py
View File

@ -1,261 +0,0 @@
#
# Copyright 2014-2017 AT&T Intellectual Property
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# - Set all configurations to run Ostro
"""Valet Engine Server Configuration."""
import os
from oslo_config import cfg
from valet.engine.conf import init_engine
CONF = cfg.CONF
class Config(object):
"""Valet Engine Server Configuration."""
def __init__(self, *default_config_files):
init_engine(default_config_files=default_config_files)
# System parameters
self.root_loc = os.path.dirname(CONF.default_config_files[0])
self.mode = None
self.command = 'status'
self.process = None
self.control_loc = None
self.api_protocol = 'http://'
self.db_keyspace = None
self.db_request_table = None
self.db_response_table = None
self.db_event_table = None
self.db_resource_table = None
self.db_app_table = None
self.db_uuid_table = None
self.replication_factor = 3
self.hosts = ['localhost']
self.port = 8080
self.ip = None
self.priority = 0
# Logging parameters
self.logger_name = None
self.logging_level = None
self.logging_loc = None
self.resource_log_loc = None
self.app_log_loc = None
self.max_main_log_size = 0
self.max_log_size = 0
self.max_num_of_logs = 0
# Management parameters
self.datacenter_name = None
self.num_of_region_chars = 0
self.rack_code_list = []
self.node_code_list = []
self.topology_trigger_freq = 0
self.compute_trigger_freq = 0
self.update_batch_wait = 0
self.default_cpu_allocation_ratio = 1
self.default_ram_allocation_ratio = 1
self.default_disk_allocation_ratio = 1
self.static_cpu_standby_ratio = 0
self.static_mem_standby_ratio = 0
self.static_local_disk_standby_ratio = 0
# Authentication parameters
self.project_name = None
self.user_name = None
self.pw = None
# Simulation parameters
self.sim_cfg_loc = None
self.num_of_hosts_per_rack = 0
self.num_of_racks = 0
self.num_of_spine_switches = 0
self.num_of_aggregates = 0
self.aggregated_ratio = 0
self.cpus_per_host = 0
self.mem_per_host = 0
self.disk_per_host = 0
self.bandwidth_of_spine = 0
self.bandwidth_of_rack = 0
self.bandwidth_of_host = 0
self.num_of_basic_flavors = 0
self.base_flavor_cpus = 0
self.base_flavor_mem = 0
self.base_flavor_disk = 0
# Music HA paramater
self.music_server_retries = 3
def configure(self):
"""Store config info extracted from oslo."""
status = self._init_system()
if status != "success":
return status
self.sim_cfg_loc = self.root_loc + self.sim_cfg_loc
self.process = self.process
self.logging_loc = self.logging_loc
self.resource_log_loc = self.logging_loc
self.app_log_loc = self.logging_loc
self.eval_log_loc = self.logging_loc
if self.mode.startswith("live") is False:
status = self._set_simulation()
if status != "success":
return status
return "success"
def _init_system(self):
self.command = CONF.command
self.mode = CONF.engine.mode
self.priority = CONF.engine.priority
self.logger_name = CONF.engine.logger_name
self.logging_level = CONF.engine.logging_level
self.logging_loc = CONF.engine.logging_dir
self.resource_log_loc = CONF.engine.logging_dir + 'resources'
self.app_log_loc = CONF.engine.logging_dir + 'app'
self.eval_log_loc = CONF.engine.logging_dir
self.max_log_size = CONF.engine.max_log_size
self.max_num_of_logs = CONF.engine.max_num_of_logs
self.process = CONF.engine.pid
self.datacenter_name = CONF.engine.datacenter_name
self.default_cpu_allocation_ratio = \
CONF.engine.default_cpu_allocation_ratio
self.default_ram_allocation_ratio = \
CONF.engine.default_ram_allocation_ratio
self.default_disk_allocation_ratio = \
CONF.engine.default_disk_allocation_ratio
self.static_cpu_standby_ratio = CONF.engine.static_cpu_standby_ratio
self.static_mem_standby_ratio = CONF.engine.static_mem_standby_ratio
self.static_local_disk_standby_ratio = \
CONF.engine.static_local_disk_standby_ratio
self.topology_trigger_freq = CONF.engine.topology_trigger_frequency
self.compute_trigger_freq = CONF.engine.compute_trigger_frequency
self.update_batch_wait = CONF.engine.update_batch_wait
self.db_keyspace = CONF.music.keyspace
self.db_request_table = CONF.music.request_table
self.db_response_table = CONF.music.response_table
self.db_event_table = CONF.music.event_table
self.db_resource_table = CONF.music.resource_table
self.db_app_table = CONF.music.app_table
self.db_uuid_table = CONF.music.uuid_table
self.replication_factor = CONF.music.replication_factor
self.hosts = CONF.music.hosts
self.port = CONF.music.port
self.music_server_retries = CONF.music.music_server_retries
self.ip = CONF.engine.ip
self.num_of_region_chars = CONF.engine.num_of_region_chars
self.rack_code_list = CONF.engine.rack_code_list
self.node_code_list = CONF.engine.node_code_list
self.sim_cfg_loc = CONF.engine.sim_cfg_loc
self.project_name = CONF.identity.project_name
self.user_name = CONF.identity.username
self.pw = CONF.identity.password
return "success"
def _set_simulation(self):
self.num_of_spine_switches = CONF.engine.num_of_spine_switches
self.num_of_hosts_per_rack = CONF.engine.num_of_hosts_per_rack
self.num_of_racks = CONF.engine.num_of_racks
self.num_of_aggregates = CONF.engine.num_of_aggregates
self.aggregated_ratio = CONF.engine.aggregated_ratio
self.cpus_per_host = CONF.engine.cpus_per_host
self.mem_per_host = CONF.engine.mem_per_host
self.disk_per_host = CONF.engine.disk_per_host
self.bandwidth_of_spine = CONF.engine.bandwidth_of_spine
self.bandwidth_of_rack = CONF.engine.bandwidth_of_rack
self.bandwidth_of_host = CONF.engine.bandwidth_of_host
self.num_of_basic_flavors = CONF.engine.num_of_basic_flavors
self.base_flavor_cpus = CONF.engine.base_flavor_cpus
self.base_flavor_mem = CONF.engine.base_flavor_mem
self.base_flavor_disk = CONF.engine.base_flavor_disk

14
valet/tests/unit/engine/test_config.py
View File

@ -15,12 +15,8 @@
"""Test Config."""
import sys
from valet.engine.optimizer.ostro_server.configuration import Config
from valet.tests.base import Base
from oslo_config import cfg
class TestConfig(Base):
"""Unit tests for Valet.engine.optimizer.ostro_server.configuration."""
@ -28,15 +24,7 @@ class TestConfig(Base):
def setUp(self):
"""Setup Test Config Testing Class."""
super(TestConfig, self).setUp()
sys.argv = [sys.argv[0]]
def test_unhappy_config_io(self):
"""Test unhappy.cfg I/O and validate I/O error in config status."""
cfg.CONF.clear()
try:
config = Config("unhappy.cfg")
config_status = config.configure()
self.validate_test("I/O error" in config_status)
except Exception as ex:
self.validate_test(isinstance(ex, cfg.ConfigFilesNotFoundError))
pass