openinfra/refstack
Code Issues Proposed changes

more cleaning

Browse Source
This commit is contained in:
termie 2013-11-19 22:23:49 -08:00
parent 8469db41eb
commit 7d795d23ef
8 changed files with 68 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
refstack/cli/refstack
View File

@ -14,8 +14,6 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
"""
"""
import sys import sys
import argparse import argparse
from textwrap import dedent from textwrap import dedent
@ -26,8 +24,8 @@ from refstack.common.tester import Tester
def add(args): def add(args):
"""adds a cloud """adds a cloud
refstack add --endpoint='http://127.0.0.1:5000/v3/' --test-user='demo' \ refstack add --endpoint='http://127.0.0.1:5000/v3/' --test-user='demo' \
--test-key='pass' --admin-endpoint='http://127.0.0.1:5000/v3/' \ --test-key='pass' --admin-endpoint='http://127.0.0.1:5000/v3/' \
--admin-user='admin' --admin-key='pass' --admin-user='admin' --admin-key='pass'
@ -51,7 +49,7 @@ def add(args):
clouds(args) clouds(args)
except IntegrityError: except IntegrityError:
print 'A Cloud with %s as its endpoint has already been added. ' % args.endpoint print 'A Cloud with %s as its endpoint has already been added. ' % args.endpoint
def config(args): def config(args):
@ -64,23 +62,23 @@ def config(args):
sys.exit(1) sys.exit(1)
t = TempestConfig(args.cloud_id) t = TempestConfig(args.cloud_id)
print t.build_config_from_keystone() print t.build_config_from_keystone()
def remove(args): def remove(args):
"""removes a cloud """removes a cloud
refstack remove {cloud_id} refstack remove {cloud_id}
confirms that cloud-id 123 has been removed from the database as well as confirms that cloud-id 123 has been removed from the database as well as
all tests assosiateed with it.""" all tests assosiateed with it."""
cloud = db.query(Cloud).get(args.cloud_id) cloud = db.query(Cloud).get(args.cloud_id)
if cloud is None: if cloud is None:
print 'Invalid cloud-id, Please use one from this list.' print 'Invalid cloud-id, Please use one from this list.'
clouds(args) clouds(args)
else: else:
db.delete(cloud) db.delete(cloud)
db.commit() db.commit()
print 'cloud %s has been deleted.' % args.cloud_id print 'cloud %s has been deleted.' % args.cloud_id
@ -92,16 +90,16 @@ def clouds(args):
print 'id | endpoint | test-user | admin-user ' print 'id | endpoint | test-user | admin-user '
print '---------------------------------------' print '---------------------------------------'
for row in db.query(Cloud).all(): for row in db.query(Cloud).all():
print "%s | %s | %s | %s " % (row.id, row.endpoint, row.test_user, row.admin_user) print "%s | %s | %s | %s " % (row.id, row.endpoint, row.test_user, row.admin_user)
print '' print ''
def start(args): def start(args):
"""start test command """start test command
refstack start {cloud_id} --sha {sha} refstack start {cloud_id} --sha {sha}
triggers local run of tempest with specified cloud_id returns a triggers local run of tempest with specified cloud_id returns a
test_id so that the user can check status or cancel the test""" test_id so that the user can check status or cancel the test"""
#load the cloud from the specified id #load the cloud from the specified id
cloud = db.query(Cloud).get(args.cloud_id) cloud = db.query(Cloud).get(args.cloud_id)
@ -115,7 +113,7 @@ def start(args):
t = Tester(args.cloud_id) t = Tester(args.cloud_id)
results = t.run_local() results = t.run_local()
# store the results # store the results
test = db.query(Test).filter_by(cloud_id=args.cloud_id).first() test = db.query(Test).filter_by(cloud_id=args.cloud_id).first()
# creat a new test # creat a new test
@ -126,18 +124,19 @@ def start(args):
db.commit() db.commit()
print 'test added with id: %s' % test.id print 'test added with id: %s' % test.id
''' '''
# do cleanup and then mark the last status to 'canceled' # do cleanup and then mark the last status to 'canceled'
test_result = TestResults() test_result = TestResults()
test_result.test_id = test.id test_result.test_id = test.id
test_result.subunit = result test_result.subunit = result
db.add(test_result) db.add(test_result)
db.commit() db.commit()
''' '''
def status(args): def status(args):
"""get the status of a running test """get the status of a running test
@ -150,7 +149,7 @@ def status(args):
print '%s is not a valid test-id.' % args.test_id print '%s is not a valid test-id.' % args.test_id
sys.exit(1) sys.exit(1)
else: else:
test_status = db.query(TestStatus).filter_by(test_id=test.id).order_by(TestStatus.id.desc()).all() test_status = db.query(TestStatus).filter_by(test_id=test.id).order_by(TestStatus.id.desc()).all()
print 'Status Log for test-id %s (top is most recent)' % args.test_id print 'Status Log for test-id %s (top is most recent)' % args.test_id
@ -181,7 +180,7 @@ def cancel(args):
test_status = TestStatus(test.id, 'canceled') test_status = TestStatus(test.id, 'canceled')
db.add(test_status) db.add(test_status)
db.commit() db.commit()
else: else:
print 'test %s does not apear to be running' % args.test_id print 'test %s does not apear to be running' % args.test_id
@ -215,17 +214,17 @@ def tests(args):
print '---------------------------------------' print '---------------------------------------'
for row in db.query(Test).filter_by(cloud_id=args.cloud_id).all(): for row in db.query(Test).filter_by(cloud_id=args.cloud_id).all():
_status = db.query(TestStatus).filter_by(test_id=row.id).order_by(TestStatus.id.desc()).first() _status = db.query(TestStatus).filter_by(test_id=row.id).order_by(TestStatus.id.desc()).first()
print "%s | %s " % (row.id, _status.message) print "%s | %s " % (row.id, _status.message)
print '' print ''
def subcommands(subparsers): def subcommands(subparsers):
"""argparse options for the clouds command """ """argparse options for the clouds command """
clouds_parser = subparsers.add_parser('clouds', help='list clouds') clouds_parser = subparsers.add_parser('clouds', help='list clouds')
"""argparse subparsers with """ """argparse subparsers with """
add_cloud_parser = subparsers.add_parser('add', help='Add a new Cloud') add_cloud_parser = subparsers.add_parser('add', help='Add a new Cloud')
add_cloud_parser.add_argument('--endpoint', add_cloud_parser.add_argument('--endpoint',
required=True, required=True,
action='store', action='store',
@ -262,14 +261,14 @@ def subcommands(subparsers):
dest='admin_key', dest='admin_key',
help='Admin keystone key or password') help='Admin keystone key or password')
"""argparse options for the remove command """ """argparse options for the remove command """
remove_parser = subparsers.add_parser('remove', help='remove a Cloud') remove_parser = subparsers.add_parser('remove', help='remove a Cloud')
remove_parser.add_argument(action='store', remove_parser.add_argument(action='store',
dest='cloud_id', dest='cloud_id',
help='The id of the cloud you want to remove') help='The id of the cloud you want to remove')
"""argparse options for the start command """ """argparse options for the start command """
start_parser = subparsers.add_parser('start', help='start tests on cloud') start_parser = subparsers.add_parser('start', help='start tests on cloud')
start_parser.add_argument(action='store', start_parser.add_argument(action='store',
@ -282,7 +281,7 @@ def subcommands(subparsers):
dest='sha', dest='sha',
help='optionally specify a sha for the tempest version to use') help='optionally specify a sha for the tempest version to use')
"""argparse options for the status command """ """argparse options for the status command """
status_parser = subparsers.add_parser('status', help='returns status of test') status_parser = subparsers.add_parser('status', help='returns status of test')
status_parser.add_argument(action='store', status_parser.add_argument(action='store',
dest='test_id', dest='test_id',
@ -294,35 +293,36 @@ def subcommands(subparsers):
help='list status history') help='list status history')
"""argparse options for the cancel command """ """argparse options for the cancel command """
cancel_parser = subparsers.add_parser('cancel', help='cancel a test') cancel_parser = subparsers.add_parser('cancel', help='cancel a test')
cancel_parser.add_argument(action='store', cancel_parser.add_argument(action='store',
dest='test_id', dest='test_id',
help='The id of the test you want to cancel') help='The id of the test you want to cancel')
"""argparse options for the result command """ """argparse options for the result command """
result_parser = subparsers.add_parser('result', help='provides results') result_parser = subparsers.add_parser('result', help='provides results')
result_parser.add_argument(action='store', result_parser.add_argument(action='store',
dest='test_id', dest='test_id',
help='The id of the test you want to cancel') help='The id of the test you want to cancel')
"""argparse options for the tests command """ """argparse options for the tests command """
tests_parser = subparsers.add_parser('tests', help='list tests') tests_parser = subparsers.add_parser('tests', help='list tests')
tests_parser.add_argument(action='store', tests_parser.add_argument(action='store',
dest='cloud_id', dest='cloud_id',
help='The id of the cloud you want to test') help='The id of the cloud you want to test')
"""argparse options for the tests command """ """argparse options for the tests command """
tests_parser = subparsers.add_parser('config', help='output tempest config for cloud') tests_parser = subparsers.add_parser('config', help='output tempest config for cloud')
tests_parser.add_argument(action='store', tests_parser.add_argument(action='store',
dest='cloud_id', dest='cloud_id',
help='The id of the cloud you want a config for') help='The id of the cloud you want a config for')
def main(): def main():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
@ -364,6 +364,5 @@ def main():
sys.exit(1) sys.exit(1)
if __name__ == '__main__': if __name__ == '__main__':
main() main()

15
refstack/common/__init__.py
View File

@ -1,15 +0,0 @@
#
# Copyright (c) 2013 Piston Cloud Computing, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

0
refstack/common/client.py
View File

69
refstack/common/cloud.py
View File

@ -1,69 +0,0 @@
#
# Copyright (c) 2013 Piston Cloud Computing, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from keystoneclient.v2_0 import client
from refstack.common import *
import refstack.models
class Clouds
class Cloud(object):
""" Cloud functions"""
cloud_id = None
def __init__(self, cloud_id = None):
""" init method loads specified id or fails"""
self.cloud_id = cloud_id
if not cloud_id:
# we have a new cloud.
# do nothing because now we'll call the add method
return None
else:
# load an existing cloud
self._cloud = models.Cloud.query.filter_by(id=self.cloud_id).first()
if not self._cloud:
# cloud not found.. invalid id
# maybe I should do someting about this ..
return None
self._keystone = client.Client(username=self._cloud.admin_user,
password=self._cloud.admin_key,
auth_url=self._cloud.admin_endpoint )
self._end_point = None
def add(self,endpoint,test_user,test_key,
admin_endpoint,admin_user,admin_key,vendor_id=None):
#adds a new cloud to the db
models.db.session.add(Cloud(endpoint,test_user,test_key,
admin_endpoint,admin_user,admin_key,vendor_id))
models.db.session.commit()
@property
def end_point(self):
"""end_point property"""
return self._end_point
@end_point.setter
def end_point(self, value):
self._end_point = value
def get_config(self):
"""uses end_point and creditials from the specified cloud_id to
get a list of services and enpoints from keystone then outputs a
usable tempest config"""

23
refstack/common/tempest_config.py
View File

@ -17,11 +17,10 @@ from keystoneclient.v2_0 import client
from refstack.models import * from refstack.models import *
class TempestConfig(object): class TempestConfig(object):
"""temptest config options. gets converted to a tempest config file""" """temptest config options. gets converted to a tempest config file"""
config = {} config = {}
def output(self): def output(self):
"""outputs config in propper format""" """outputs config in propper format"""
output = '' output = ''
@ -34,19 +33,18 @@ class TempestConfig(object):
def build_config_from_keystone(self): def build_config_from_keystone(self):
"""uses the keystoneclient libs to query a clouds endpoint and """uses the keystoneclient libs to query a clouds endpoint and
retrive a service catelog. that it then uses to populate the retrive a service catelog. that it then uses to populate the
values for our specific tempest config""" values for our specific tempest config"""
# load an existing cloud # load an existing cloud
self._cloud = db.query(Cloud).filter_by(id=self.cloud_id).first() self._cloud = db.query(Cloud).filter_by(id=self.cloud_id).first()
if not self._cloud: if not self._cloud:
# cloud not found.. invalid id # cloud not found.. invalid id
# maybe I should do someting about this .. # maybe I should do someting about this ..
return None return None
# This stuff we know before hitting up keystone
# This stuff we know before hitting up keystone
self.config['identity']['uri'] = self._cloud.admin_endpoint self.config['identity']['uri'] = self._cloud.admin_endpoint
self.config['identity']['admin_username'] = self._cloud.admin_user self.config['identity']['admin_username'] = self._cloud.admin_user
self.config['identity']['admin_password'] = self._cloud.admin_key self.config['identity']['admin_password'] = self._cloud.admin_key
@ -54,7 +52,7 @@ class TempestConfig(object):
self.config['identity']['password'] = self._cloud.test_key self.config['identity']['password'] = self._cloud.test_key
self.config['identity']['tenant_name'] = self._cloud.admin_user self.config['identity']['tenant_name'] = self._cloud.admin_user
# keystone client object # keystone client object
self._keystone = client.Client(username=self._cloud.admin_user, self._keystone = client.Client(username=self._cloud.admin_user,
password=self._cloud.admin_key, password=self._cloud.admin_key,
tenant_name=self._cloud.admin_user, tenant_name=self._cloud.admin_user,
@ -87,15 +85,14 @@ class TempestConfig(object):
if self.service_catalog.has_key(service): if self.service_catalog.has_key(service):
self.config['service_available'][service] = True self.config['service_available'][service] = True
# boto settings # boto settings
self.config['boto']['ec2_url'] = self.service_catalog['ec2'] self.config['boto']['ec2_url'] = self.service_catalog['ec2']
self.config['boto']['s3_url'] = self.service_catalog['s3'] self.config['boto']['s3_url'] = self.service_catalog['s3']
# return the actual config # return the actual config
return self.output() return self.output()
def __init__(self, cloud_id): def __init__(self, cloud_id):
""" sets up the default configs""" """ sets up the default configs"""
self.cloud_id = cloud_id self.cloud_id = cloud_id
@ -190,7 +187,7 @@ class TempestConfig(object):
'backend2_name': 'BACKEND_2', 'backend2_name': 'BACKEND_2',
'storage_protocol': 'iSCSI', 'storage_protocol': 'iSCSI',
'vendor_name': 'Open Source' } 'vendor_name': 'Open Source' }
self.config['object-storage'] = { self.config['object-storage'] = {
'catalog_type': 'object-store', 'catalog_type': 'object-store',
'container_sync_timeout': 120, 'container_sync_timeout': 120,

57
refstack/common/tester.py
View File

@ -14,11 +14,7 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import os import os
import sys
import errno
from subprocess import call from subprocess import call
from textwrap import dedent
from refstack.app import app
from refstack.common.tempest_config import TempestConfig from refstack.common.tempest_config import TempestConfig
import testrepository.repository.file import testrepository.repository.file
from testrepository import ui from testrepository import ui
@ -27,11 +23,12 @@ from testrepository.commands import init
import gear import gear
class TesterWorker(object): class TesterWorker(object):
"""gearman worker code""" """gearman worker code"""
def __init__(self,app): def __init__(self,app):
self.worker = gear.Worker('run_remote_test') self.worker = gear.Worker('run_remote_test')
self.worker.addServer(app.gearman_server) self.worker.addServer(app.gearman_server)
self.worker.registerFunction('run_remote_test') self.worker.registerFunction('run_remote_test')
@ -46,7 +43,6 @@ class TesterWorker(object):
job.sendWorkComplete(job.arguments.reverse()) job.sendWorkComplete(job.arguments.reverse())
class TestRepositoryUI(ui.AbstractUI): class TestRepositoryUI(ui.AbstractUI):
"""nothing""" """nothing"""
def __init__(self, here): def __init__(self, here):
@ -57,11 +53,10 @@ class TestRepositoryUI(ui.AbstractUI):
self.here = here self.here = here
class TestRepositorySource(object): class TestRepositorySource(object):
"""Get test results from a testrepository. """Get test results from a testrepository.
Reloading asks testr to re-run (and implicitly record) a new Reloading asks testr to re-run (and implicitly record) a new
test result. test result.
:ivar testr_directory: path to directory containing .testr repository :ivar testr_directory: path to directory containing .testr repository
@ -70,7 +65,7 @@ class TestRepositorySource(object):
def __init__(self, testr_directory): def __init__(self, testr_directory):
self.testr_directory = os.path.expanduser(testr_directory) self.testr_directory = os.path.expanduser(testr_directory)
self._ui = TestRepositoryUI(self.testr_directory) self._ui = TestRepositoryUI(self.testr_directory)
self.init_repo() self.init_repo()
@ -89,7 +84,7 @@ class TestRepositorySource(object):
cmd.run() cmd.run()
except OSError: except OSError:
# if this happens its fine .. just means the repo is already there # if this happens its fine .. just means the repo is already there
pass pass
def run(self): def run(self):
@ -98,20 +93,19 @@ class TestRepositorySource(object):
self._ui.c = self.testr_directory+'tempest.conf' self._ui.c = self.testr_directory+'tempest.conf'
cmd = run.run(self._ui) cmd = run.run(self._ui)
res = cmd.execute() res = cmd.execute()
def testrepository_last_stream(self): def testrepository_last_stream(self):
factory = testrepository.repository.file.RepositoryFactory() factory = testrepository.repository.file.RepositoryFactory()
repo = factory.open(self.testr_directory) repo = factory.open(self.testr_directory)
# this is poor because it just returns a stringio for the whole # this is poor because it just returns a stringio for the whole
# thing; we should instead try to read from it as a file so we can # thing; we should instead try to read from it as a file so we can
# get nonblocking io # get nonblocking io
return repo.get_test_run(repo.latest_id()).get_subunit_stream() return repo.get_test_run(repo.latest_id()).get_subunit_stream()
class Tester(object): class Tester(object):
""" Test functions""" """ Test functions"""
test_id = None test_id = None
@ -125,7 +119,7 @@ class Tester(object):
if not test_id: if not test_id:
#create a new test id #create a new test id
self.test_id = 10 self.test_id = 10
else: else:
# set test id # set test id
self.test_id = id self.test_id = id
@ -136,29 +130,28 @@ class Tester(object):
def run_remote(self): def run_remote(self):
"""triggers remote run""" """triggers remote run"""
# install tempest in virt env # install tempest in virt env
# start tests against cloud_id using sha of tempest # start tests against cloud_id using sha of tempest
# no sha indicates trunk # no sha indicates trunk
def run_local(self): def run_local(self):
"""triggers local run""" """triggers local run"""
# make sure we have a folder to put a repo in.. # make sure we have a folder to put a repo in..
if not os.path.exists(self.test_path): if not os.path.exists(self.test_path):
os.makedirs(self.test_path) os.makedirs(self.test_path)
# write the tempest config to that folder # write the tempest config to that folder
self.write_config(self.test_path) self.write_config(self.test_path)
# setup the repo wrapper.. this creates the repo if its not already there # setup the repo wrapper.. this creates the repo if its not already there
tr = TestRepositorySource(self.test_path) tr = TestRepositorySource(self.test_path)
"""TODO: So this is supposed to use the testr wrapper to trigger a run.. however.. """TODO: So this is supposed to use the testr wrapper to trigger a run.. however..
I am completly blocked on how to make it work the right way.. so I am moving on I am completly blocked on how to make it work the right way.. so I am moving on
for now once the congigs are setup and repo initiated it will call a subprocess for now once the congigs are setup and repo initiated it will call a subprocess
run the command .. THEN query the repo for the last set of results and store the run the command .. THEN query the repo for the last set of results and store the
subunit stream. subunit stream.
# run tests # run tests
#tr.run() #tr.run()
@ -168,10 +161,10 @@ class Tester(object):
call([self.test_path+'runtests.sh']) # replace this call([self.test_path+'runtests.sh']) # replace this
print "finished with tests" print "finished with tests"
# get back the results # get back the results
result = tr.testrepository_last_stream() result = tr.testrepository_last_stream()
# write results to database maybe .. or return them .. not sure which .. # write results to database maybe .. or return them .. not sure which ..
return result.read() return result.read()
#return None #return None
@ -196,7 +189,7 @@ test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
test_id_option=--load-list $IDFILE test_id_option=--load-list $IDFILE
test_list_option=--list test_list_option=--list
group_regex=([^\.]*\.)*""" group_regex=([^\.]*\.)*"""
with open(path+"runtests.sh", "w") as runtests_script_file: with open(path+"runtests.sh", "w") as runtests_script_file:
runtests_script_file.write(runtests_script) runtests_script_file.write(runtests_script)
@ -208,12 +201,9 @@ group_regex=([^\.]*\.)*"""
with open(path+".testr.conf", "w") as testr_config_file: with open(path+".testr.conf", "w") as testr_config_file:
testr_config_file.write(testr_output) testr_config_file.write(testr_output)
def cancel(self): def cancel(self):
""" cancels a running test""" """ cancels a running test"""
@property @property
def status(self): def status(self):
"""The status property.""" """The status property."""
@ -225,9 +215,8 @@ group_regex=([^\.]*\.)*"""
del self._status del self._status
return locals() return locals()
@property @property
def config(self): def config(self):
"""The config property. outputs a tempest config based on settings""" """The config property. outputs a tempest config based on settings"""
return self.tempest_config.output() return self.tempest_config.output()

2
refstack/common/vendor.py
View File

@ -17,7 +17,7 @@ from refstack.common import *
class Vendor: class Vendor:
""" Vendor functions""" """Vendor functions"""
def __init__(self, id): def __init__(self, id):
""" init method loads specified id or fails""" """ init method loads specified id or fails"""

2
refstack/schema.sql
View File

@ -1,3 +1,3 @@
CREATE TABLE IF NOT EXISTS VENDORS CREATE TABLE IF NOT EXISTS VENDORS
( vendor_id integer primary key asc autoincrement, (vendor_id integer primary key asc autoincrement,
vendor_name TEXT NOT NULL); vendor_name TEXT NOT NULL);