openstack/zaqar
Code Issues Proposed changes

Make zaqar-bench use credentials from os_client_config

Browse Source 
Currently zaqar-bench can't benchmark Zaqar while Zaqar is using
keystone authentication backend.

This patch makes it possible by using os_client_config library. The
library gets authentication parameters from "clouds.yaml". If keystone
authentication environment variables present, they override values from
"clouds.yaml". The aquired parameters then passed to constructors of
each python-zaqarclient's Client object used in zaqar-bench.

The patch also makes benchmark queue names reusable across all
zaqar-bench parts and by this fixes the old DRY princible bug.

To use zaqar-bench with keystone authentication the user must
explicitly set OS_AUTH_STRATEGY=keystone as environment variable
before running the tool. Otherwise the default 'noauth' auth strategy
will be used. This allows the user to run zaqar-bench as usual.

This patch also adds option "--api-version" with it's short version
"api" to zaqar-bench which defaults to Zaqar API v2.

Change-Id: I0a7aaeaeac6da1b2c9f08fbfdddd467de5747a28
Closes-Bug: 1523752
This commit is contained in:
Eva Balycheva 2015-11-28 07:42:09 +03:00
parent 2145c9bf5a
commit 1b8334b35a
7 changed files with 134 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bench-requirements.txt
View File

@ -2,3 +2,4 @@ argparse>=1.2.1
gevent>=1.0.1
marktime>=0.2.0
python-zaqarclient>=0.0.2
os-client-config!=1.6.2,>=1.4.0

11
zaqar/bench/conductor.py
View File

@ -16,6 +16,10 @@ from __future__ import print_function
import json
import multiprocessing as mp
import os
# NOTE(Eva-i): See https://github.com/gevent/gevent/issues/349. Let's keep
# it until the new stable version of gevent(>=1.1) will be released.
os.environ["GEVENT_RESOLVER"] = "ares"
from zaqar.bench import config
from zaqar.bench import consumer
@ -39,11 +43,8 @@ def _print_verbose_stats(name, stats):
def _reset_queues():
cli = helpers.get_new_client()
for i in range(CONF.num_queues):
# TODO(kgriffs): DRY up name generation so it is done
# in a helper, vs. being copy-pasted everywhere.
queue = cli.queue(CONF.queue_prefix + '-' + str(i))
for queue_name in helpers.queue_names:
queue = cli.queue(queue_name)
queue.delete()

5
zaqar/bench/config.py
View File

@ -49,11 +49,14 @@ _CLI_OPTIONS = (
default=5,
help='Number of Observer Workers'),
cfg.FloatOpt('api_version', short='api', default='2',
help='Zaqar API version to use'),
cfg.IntOpt('messages_per_claim', short='cno', default=5,
help=('Number of messages the consumer will attempt to '
'claim at a time')),
cfg.IntOpt('messages_per_list', short='lno', default=5,
help=('Number of messages the obserer will attempt to '
help=('Number of messages the observer will attempt to '
'list at a time')),
cfg.IntOpt('time', short='t', default=5,

5
zaqar/bench/consumer.py
View File

@ -96,8 +96,9 @@ def load_generator(stats, num_workers, num_queues,
test_duration, url, ttl, grace, limit):
cli = helpers.get_new_client()
queues = [cli.queue(CONF.queue_prefix + '-' + str(i))
for i in range(num_queues)]
queues = []
for queue_name in helpers.queue_names:
queues.append(cli.queue(queue_name))
gevent.joinall([
gevent.spawn(claim_delete,

109
zaqar/bench/helpers.py
View File

@ -11,13 +11,86 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# NOTE(Eva-i): Some code was taken from python-zaqarclient.
import os
import sys
import os_client_config
from six.moves import urllib_parse
from zaqarclient.queues import client
from zaqar.bench import config
CONF = config.conf
client_conf = {
def _get_credential_args():
"""Retrieves credential arguments for keystone
Credentials are either read via os-client-config from the environment
or from a config file ('clouds.yaml'). Config file variables override those
from the environment variables.
devstack produces a clouds.yaml with two named clouds - one named
'devstack' which has user privs and one named 'devstack-admin' which
has admin privs. This function will default to getting the credentials from
environment variables. If not all required credentials present in
environment variables, it tries to get credentials for 'devstack-admin'
cloud in clouds.yaml. If no 'devstack-admin' cloud found, it tried to get
credentials for 'devstack' cloud. If no 'devstack' cloud found, throws
an error and stops the application.
"""
os_cfg = os_client_config.OpenStackConfig()
cloud = os_cfg.get_one_cloud()
cred_args = cloud.get_auth_args()
required_options = ['username', 'password', 'auth_url', 'project_name']
if not all(arg in cred_args for arg in required_options):
try:
cloud = os_cfg.get_one_cloud(cloud='devstack-admin')
except Exception:
try:
cloud = os_cfg.get_one_cloud(cloud='devstack')
except Exception:
print("Insufficient amount of credentials found for keystone "
"authentication. Credentials should reside either in "
"environment variables or in 'clouds.yaml' file. If "
"both present, the ones in environment variables will "
"be preferred. Exiting.")
sys.exit()
cred_args = cloud.get_auth_args()
print("Using '{}' credentials".format(cloud.name))
return cred_args
def _generate_client_conf():
auth_strategy = os.environ.get
if auth_strategy == 'keystone':
args = _get_credential_args()
# FIXME(flwang): Now we're hardcode the keystone auth version, since
# there is a 'bug' with the osc-config which is returning the auth_url
# without version. This should be fixed as long as the bug is fixed.
parsed_url = urllib_parse.urlparse(args['auth_url'])
auth_url = args['auth_url']
if not parsed_url.path or parsed_url.path == '/':
auth_url = urllib_parse.urljoin(args['auth_url'], 'v2.0')
conf = {
'auth_opts': {
'backend': 'keystone',
'options': {
'os_username': args['username'],
'os_password': args['password'],
'os_project_name': args['project_name'],
'os_auth_url': auth_url,
'insecure': '',
},
},
}
else:
conf = {
'auth_opts': {
'backend': 'noauth',
'options': {
@ -25,7 +98,39 @@ client_conf = {
},
},
}
print("Using '{0}' authentication method".format(conf['auth_opts']
['backend']))
return conf
class LazyAPIVersion(object):
def __init__(self):
self.api_version = None
@property
def get(self):
if self.api_version is None:
conversion_map = {
1.0: 1,
1.1: 1.1,
2.0: 2,
}
try:
self.api_version = conversion_map[CONF.api_version]
except KeyError:
print("Unknown Zaqar API version: '{}'. Exiting...".format(
CONF.api_version))
sys.exit()
print("Benchmarking Zaqar API v{0}...".format(self.api_version))
return self.api_version
client_conf = _generate_client_conf()
client_api = LazyAPIVersion()
queue_names = []
for i in range(CONF.num_queues):
queue_names.append((CONF.queue_prefix + '-' + str(i)))
def get_new_client():
return client.Client(CONF.server_url, 1.1, conf=client_conf)
return client.Client(CONF.server_url, client_api.get, conf=client_conf)

5
zaqar/bench/observer.py
View File

@ -100,8 +100,9 @@ def load_generator(stats, num_workers, num_queues,
test_duration, limit):
cli = helpers.get_new_client()
queues = [cli.queue(CONF.queue_prefix + '-' + str(i))
for i in range(num_queues)]
queues = []
for queue_name in helpers.queue_names:
queues.append(cli.queue(queue_name))
gevent.joinall([
gevent.spawn(observer,

5
zaqar/bench/producer.py
View File

@ -105,8 +105,9 @@ def producer(queues, message_pool, stats, test_duration):
def load_generator(stats, num_workers, num_queues, test_duration):
cli = helpers.get_new_client()
queues = [cli.queue(CONF.queue_prefix + '-' + str(i))
for i in range(num_queues)]
queues = []
for queue_name in helpers.queue_names:
queues.append(cli.queue(queue_name))
message_pool = load_messages()