openstack/rally-openstack
Code Issues Proposed changes

Merge "[Sahara] Move sahara data to sahara context"

Browse Source
This commit is contained in:
Jenkins 2015-11-19 17:02:32 +00:00 committed by Gerrit Code Review
commit 08bcbd40c9
14 changed files with 120 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
rally/plugins/openstack/context/sahara/sahara_cluster.py
View File

@ -91,7 +91,8 @@ class SaharaCluster(context.Context):
@logging.log_task_wrapper(LOG.info, _("Enter context: `Sahara Cluster`"))
def setup(self):
self.context["sahara_clusters"] = {}
utils.init_sahara_context(self)
self.context["sahara"]["clusters"] = {}
wait_dict = {}
@ -129,7 +130,8 @@ class SaharaCluster(context.Context):
wait_active=False
)
self.context["tenants"][tenant_id]["sahara_cluster"] = cluster.id
self.context["tenants"][tenant_id]["sahara"]["cluster"] = (
cluster.id)
# Need to save the client instance to poll for active status
wait_dict[cluster] = scenario.clients("sahara")

14
rally/plugins/openstack/context/sahara/sahara_image.py
View File

@ -20,6 +20,7 @@ from rally import exceptions
from rally import osclients
from rally.plugins.openstack.context.cleanup import manager as resource_manager
from rally.plugins.openstack.scenarios.glance import utils as glance_utils
from rally.plugins.openstack.scenarios.sahara import utils
from rally.task import context
@ -75,13 +76,14 @@ class SaharaImage(context.Context):
@logging.log_task_wrapper(LOG.info, _("Enter context: `Sahara Image`"))
def setup(self):
self.context["sahara_images"] = {}
utils.init_sahara_context(self)
self.context["sahara"]["images"] = {}
# The user may want to use the existing image. In this case he should
# make sure that the image is public and has all required metadata.
image_uuid = self.config.get("image_uuid")
self.context["need_sahara_image_cleanup"] = not image_uuid
self.context["sahara"]["need_image_cleanup"] = not image_uuid
if image_uuid:
# Using the first user to check the existing image.
@ -97,7 +99,8 @@ class SaharaImage(context.Context):
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
self.context["tenants"][tenant_id]["sahara_image"] = image_id
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
else:
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
@ -109,12 +112,13 @@ class SaharaImage(context.Context):
user=user,
user_name=self.config["username"])
self.context["tenants"][tenant_id]["sahara_image"] = image_id
self.context["tenants"][tenant_id]["sahara"]["image"] = (
image_id)
@logging.log_task_wrapper(LOG.info, _("Exit context: `Sahara Image`"))
def cleanup(self):
# TODO(boris-42): Delete only resources created by this context
if self.context["need_sahara_image_cleanup"]:
if self.context["sahara"]["need_image_cleanup"]:
resource_manager.cleanup(names=["glance.images"],
users=self.context.get("users", []))

21
rally/plugins/openstack/context/sahara/sahara_input_data_sources.py
View File

@ -23,6 +23,7 @@ from rally import consts
from rally import osclients
from rally.plugins.openstack.context.cleanup import manager as resource_manager
from rally.plugins.openstack.context.cleanup import resources as res_cleanup
from rally.plugins.openstack.scenarios.sahara import utils
from rally.plugins.openstack.scenarios.swift import utils as swift_utils
from rally.task import context
@ -68,8 +69,9 @@ class SaharaInputDataSources(context.Context):
@logging.log_task_wrapper(LOG.info,
_("Enter context: `Sahara Input Data Sources`"))
def setup(self):
self.context["swift_objects"] = []
self.context["container_name"] = None
utils.init_sahara_context(self)
self.context["sahara"]["swift_objects"] = []
self.context["sahara"]["container_name"] = None
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
@ -92,7 +94,7 @@ class SaharaInputDataSources(context.Context):
data_source_type=input_type,
url=input_url)
self.context["tenants"][tenant_id]["sahara_input"] = input_ds.id
self.context["tenants"][tenant_id]["sahara"]["input"] = input_ds.id
def setup_inputs_swift(self, clients, tenant_id, input_url,
swift_files, username, password):
@ -100,29 +102,30 @@ class SaharaInputDataSources(context.Context):
context=self.context)
container_name = "rally_" + parse.urlparse(input_url).netloc.rstrip(
".sahara")
self.context["container_name"] = (
self.context["sahara"]["container_name"] = (
swift_scenario._create_container(container_name=container_name))
for swift_file in swift_files:
content = requests.get(swift_file["download_url"]).content
self.context["swift_objects"].append(
self.context["sahara"]["swift_objects"].append(
swift_scenario._upload_object(
self.context["container_name"], content,
self.context["sahara"]["container_name"], content,
object_name=swift_file["name"]))
input_ds_swift = clients.sahara().data_sources.create(
name=self.generate_random_name(), description="",
data_source_type="swift", url=input_url,
credential_user=username, credential_pass=password)
self.context["tenants"][tenant_id]["sahara_input"] = (
self.context["tenants"][tenant_id]["sahara"]["input"] = (
input_ds_swift.id)
@logging.log_task_wrapper(LOG.info, _("Exit context: `Sahara Input Data"
"Sources`"))
def cleanup(self):
resources = ["data_sources"]
for swift_object in self.context["swift_objects"]:
for swift_object in self.context["sahara"]["swift_objects"]:
res_cleanup.SwiftObject(resource=swift_object[1])
res_cleanup.SwiftContainer(resource=self.context["container_name"])
res_cleanup.SwiftContainer(
resource=self.context["sahara"]["container_name"])
# TODO(boris-42): Delete only resources created by this context
resource_manager.cleanup(

16
rally/plugins/openstack/context/sahara/sahara_job_binaries.py
View File

@ -22,6 +22,7 @@ from rally import consts
from rally import exceptions
from rally import osclients
from rally.plugins.openstack.context.cleanup import manager as resource_manager
from rally.plugins.openstack.scenarios.sahara import utils
from rally.task import context
@ -79,20 +80,20 @@ class SaharaJobBinaries(context.Context):
@logging.log_task_wrapper(LOG.info,
_("Enter context: `Sahara Job Binaries`"))
def setup(self):
utils.init_sahara_context(self)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
clients = osclients.Clients(user["endpoint"])
sahara = clients.sahara()
self.context["tenants"][tenant_id]["sahara_mains"] = []
self.context["tenants"][tenant_id]["sahara_libs"] = []
self.context["tenants"][tenant_id]["sahara"]["mains"] = []
self.context["tenants"][tenant_id]["sahara"]["libs"] = []
for main in self.config.get("mains", []):
self.download_and_save_lib(
sahara=sahara,
lib_type="sahara_mains",
lib_type="mains",
name=main["name"],
download_url=main["download_url"],
tenant_id=tenant_id)
@ -100,7 +101,7 @@ class SaharaJobBinaries(context.Context):
for lib in self.config.get("libs", []):
self.download_and_save_lib(
sahara=sahara,
lib_type="sahara_libs",
lib_type="libs",
name=lib["name"],
download_url=lib["download_url"],
tenant_id=tenant_id)
@ -116,7 +117,7 @@ class SaharaJobBinaries(context.Context):
data_source_type=input_type,
url=input_url)
self.context["tenants"][tenant_id]["sahara_input"] = input_ds.id
self.context["tenants"][tenant_id]["sahara"]["input"] = input_ds.id
def download_and_save_lib(self, sahara, lib_type, name, download_url,
tenant_id):
@ -136,7 +137,8 @@ class SaharaJobBinaries(context.Context):
description="",
extra={})
self.context["tenants"][tenant_id][lib_type].append(job_binary.id)
self.context["tenants"][tenant_id]["sahara"][lib_type].append(
job_binary.id)
@logging.log_task_wrapper(LOG.info,
_("Exit context: `Sahara Job Binaries`"))

19
rally/plugins/openstack/context/sahara/sahara_output_data_sources.py
View File

@ -20,6 +20,7 @@ from rally import consts
from rally import osclients
from rally.plugins.openstack.context.cleanup import manager as resource_manager
from rally.plugins.openstack.context.cleanup import resources as res_cleanup
from rally.plugins.openstack.scenarios.sahara import utils
from rally.plugins.openstack.scenarios.swift import utils as swift_utils
from rally.task import context
@ -49,6 +50,7 @@ class SaharaOutputDataSources(context.Context):
@logging.log_task_wrapper(LOG.info,
_("Enter context: `Sahara Output Data Sources`"))
def setup(self):
utils.init_sahara_context(self)
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
@ -60,7 +62,7 @@ class SaharaOutputDataSources(context.Context):
context=self.context)
container_name = rutils.generate_random_name(
prefix=self.config["output_url_prefix"])
self.context["tenants"][tenant_id]["sahara_container"] = {
self.context["tenants"][tenant_id]["sahara"]["container"] = {
"name": swift._create_container(
container_name=container_name),
"output_swift_objects": []
@ -80,7 +82,7 @@ class SaharaOutputDataSources(context.Context):
data_source_type="hdfs",
url=output_url)
self.context["tenants"][tenant_id]["sahara_output"] = output_ds.id
self.context["tenants"][tenant_id]["sahara"]["output"] = output_ds.id
def setup_outputs_swift(self, swift, sahara, tenant_id, container_name,
username, password):
@ -92,7 +94,7 @@ class SaharaOutputDataSources(context.Context):
credential_user=username,
credential_pass=password)
self.context["tenants"][tenant_id]["sahara_output"] = (
self.context["tenants"][tenant_id]["sahara"]["output"] = (
output_ds_swift.id
)
@ -101,16 +103,15 @@ class SaharaOutputDataSources(context.Context):
def cleanup(self):
for user, tenant_id in rutils.iterate_per_tenants(
self.context["users"]):
if self.context["tenants"][tenant_id].get("sahara_container",
{}).get(
"name") is not None:
if self.context["tenants"][tenant_id].get(
"sahara", {}).get("container", {}).get("name") is not None:
for swift_object in (
self.context["tenants"][tenant_id]["sahara_container"][
self.context["tenants"][tenant_id]["sahara"]["container"][
"output_swift_objects"]):
res_cleanup.SwiftObject(swift_object[1])
res_cleanup.SwiftContainer(
self.context["tenants"][tenant_id].get("sahara_container",
{}).get("name"))
self.context["tenants"][tenant_id].get(
"sahara", {}).get("container", {}).get("name"))
resources = ["data_sources"]
resource_manager.cleanup(
names=["sahara.%s" % res for res in resources],

4
rally/plugins/openstack/scenarios/sahara/clusters.py
View File

@ -79,7 +79,7 @@ class SaharaClusters(utils.SaharaScenario):
do not assign floating ips to workers.
"""
image_id = self.context["tenant"]["sahara_image"]
image_id = self.context["tenant"]["sahara"]["image"]
LOG.debug("Using Image: %s" % image_id)
@ -160,7 +160,7 @@ class SaharaClusters(utils.SaharaScenario):
do not assign floating ips to workers.
"""
image_id = self.context["tenant"]["sahara_image"]
image_id = self.context["tenant"]["sahara"]["image"]
LOG.debug("Using Image: %s" % image_id)

10
rally/plugins/openstack/scenarios/sahara/jobs.py
View File

@ -42,8 +42,8 @@ class SaharaJob(utils.SaharaScenario):
in a sequence
"""
mains = self.context["tenant"]["sahara_mains"]
libs = self.context["tenant"]["sahara_libs"]
mains = self.context["tenant"]["sahara"]["mains"]
libs = self.context["tenant"]["sahara"]["libs"]
name = self.generate_random_name()
job = self.clients("sahara").jobs.create(name=name,
@ -52,13 +52,13 @@ class SaharaJob(utils.SaharaScenario):
mains=mains,
libs=libs)
cluster_id = self.context["tenant"]["sahara_cluster"]
cluster_id = self.context["tenant"]["sahara"]["cluster"]
if job_type.lower() == "java":
input_id = None
output_id = None
else:
input_id = self.context["tenant"]["sahara_input"]
input_id = self.context["tenant"]["sahara"]["input"]
output_id = self._create_output_ds().id
self._run_job_execution(job_id=job.id,
@ -101,7 +101,7 @@ class SaharaJob(utils.SaharaScenario):
remove worker nodes from the cluster
"""
cluster_id = self.context["tenant"]["sahara_cluster"]
cluster_id = self.context["tenant"]["sahara"]["cluster"]
# Executing the sequence before the first scaling
self.create_launch_job_sequence(jobs)

14
rally/plugins/openstack/scenarios/sahara/utils.py
View File

@ -21,6 +21,7 @@ from saharaclient.api import base as sahara_base
from rally.common.i18n import _
from rally.common import log as logging
from rally.common import utils as rutils
from rally import consts
from rally import exceptions
from rally.plugins.openstack import scenario
@ -453,8 +454,8 @@ class SaharaScenario(scenario.OpenStackScenario):
:returns: The created Data Source
"""
ds_type = self.context["sahara_output_conf"]["output_type"]
url_prefix = self.context["sahara_output_conf"]["output_url_prefix"]
ds_type = self.context["sahara"]["output_conf"]["output_type"]
url_prefix = self.context["sahara"]["output_conf"]["output_url_prefix"]
if ds_type == "swift":
raise exceptions.RallyException(
@ -554,3 +555,12 @@ class SaharaScenario(scenario.OpenStackScenario):
LOG.debug("Using neutron router %s." % net["router_id"])
return neutron_net_id
def init_sahara_context(context_instance):
context_instance.context["sahara"] = context_instance.context.get("sahara",
{})
for user, tenant_id in rutils.iterate_per_tenants(
context_instance.context["users"]):
context_instance.context["tenants"][tenant_id]["sahara"] = (
context_instance.context["tenants"][tenant_id].get("sahara", {}))

8
tests/unit/plugins/openstack/context/sahara/test_sahara_image.py
View File

@ -38,7 +38,8 @@ class SaharaImageTestCase(test.ScenarioTestCase):
self.users_key = []
for i in range(self.tenants_num):
self.tenants[str(i)] = {"id": str(i), "name": str(i)}
self.tenants[str(i)] = {"id": str(i), "name": str(i),
"sahara_image": "42"}
for j in range(self.users_per_tenant):
self.users_key.append({"id": "%s_%s" % (str(i), str(j)),
"tenant_id": str(i),
@ -79,7 +80,7 @@ class SaharaImageTestCase(test.ScenarioTestCase):
},
"admin": {"endpoint": mock.MagicMock()},
"users": self.users_key,
"tenants": self.tenants
"tenants": self.tenants,
})
return self.context
@ -140,7 +141,8 @@ class SaharaImageTestCase(test.ScenarioTestCase):
sahara_ctx.setup()
for tenant_id in sahara_ctx.context["tenants"]:
image_id = sahara_ctx.context["tenants"][tenant_id]["sahara_image"]
image_id = (
sahara_ctx.context["tenants"][tenant_id]["sahara"]["image"])
self.assertEqual("some_id", image_id)
self.assertEqual(False, mock_glance_scenario__create_image.called)

13
tests/unit/plugins/openstack/context/sahara/test_sahara_job_binaries.py
View File

@ -90,13 +90,13 @@ class SaharaJobBinariesTestCase(test.ScenarioTestCase):
for i in range(self.tenants_num):
download_calls.append(mock.call(
sahara=mock_sahara,
lib_type="sahara_mains",
lib_type="mains",
name="test.jar",
download_url="http://example.com/test.jar",
tenant_id=str(i)))
download_calls.append(mock.call(
sahara=mock_sahara,
lib_type="sahara_libs",
lib_type="libs",
name="test.jar",
download_url="http://example.com/test.jar",
tenant_id=str(i)))
@ -122,17 +122,18 @@ class SaharaJobBinariesTestCase(test.ScenarioTestCase):
mock.MagicMock(id=42))
sahara_ctx = sahara_job_binaries.SaharaJobBinaries(self.context)
sahara_ctx.context["tenants"]["0"]["sahara_mains"] = []
sahara_ctx.context["tenants"]["0"]["sahara_libs"] = []
sahara_ctx.context["tenants"]["0"]["sahara"] = {"mains": []}
sahara_ctx.context["tenants"]["0"]["sahara"]["libs"] = []
sahara_ctx.download_and_save_lib(sahara=mock_sahara,
lib_type="sahara_mains",
lib_type="mains",
name="test_binary",
download_url="http://somewhere",
tenant_id="0")
sahara_ctx.download_and_save_lib(sahara=mock_sahara,
lib_type="sahara_libs",
lib_type="libs",
name="test_binary_2",
download_url="http://somewhere",
tenant_id="0")

2
tests/unit/plugins/openstack/context/sahara/test_sahara_output_data_sources.py
View File

@ -70,7 +70,7 @@ class SaharaOutputDataSourcesTestCase(test.ScenarioTestCase):
def check_setup(self):
context = sahara_output_data_sources.SaharaOutputDataSources.context[
"sahara_output_conf"]
"sahara"]["output_conf"]
self.assertIsNotNone(context.get("output_type"))
self.assertIsNotNone(context.get("output_url_prefix"))

8
tests/unit/plugins/openstack/scenarios/sahara/test_clusters.py
View File

@ -34,7 +34,9 @@ class SaharaClustersTestCase(test.ScenarioTestCase):
clusters_scenario.context = {
"tenant": {
"sahara_image": "test_image"
"sahara": {
"image": "test_image",
}
}
}
clusters_scenario.create_and_delete_cluster(
@ -76,7 +78,9 @@ class SaharaClustersTestCase(test.ScenarioTestCase):
clusters_scenario.context = {
"tenant": {
"sahara_image": "test_image"
"sahara": {
"image": "test_image",
}
}
}

48
tests/unit/plugins/openstack/scenarios/sahara/test_jobs.py
View File

@ -41,11 +41,13 @@ class SaharaJobTestCase(test.ScenarioTestCase):
self.context.update({
"tenant": {
"sahara_image": "test_image",
"sahara_mains": ["main_42"],
"sahara_libs": ["lib_42"],
"sahara_cluster": "cl_42",
"sahara_input": "in_42"
"sahara": {
"image": "test_image",
"mains": ["main_42"],
"libs": ["lib_42"],
"cluster": "cl_42",
"input": "in_42"
}
}
})
jobs_scenario = jobs.SaharaJob(self.context)
@ -83,11 +85,13 @@ class SaharaJobTestCase(test.ScenarioTestCase):
self.context.update({
"tenant": {
"sahara_image": "test_image",
"sahara_mains": ["main_42"],
"sahara_libs": ["lib_42"],
"sahara_cluster": "cl_42",
"sahara_input": "in_42"
"sahara": {
"image": "test_image",
"mains": ["main_42"],
"libs": ["lib_42"],
"cluster": "cl_42",
"input": "in_42"
}
}
})
jobs_scenario = jobs.SaharaJob(self.context)
@ -122,11 +126,13 @@ class SaharaJobTestCase(test.ScenarioTestCase):
self.context.update({
"tenant": {
"sahara_image": "test_image",
"sahara_mains": ["main_42"],
"sahara_libs": ["lib_42"],
"sahara_cluster": "cl_42",
"sahara_input": "in_42"
"sahara": {
"image": "test_image",
"mains": ["main_42"],
"libs": ["lib_42"],
"cluster": "cl_42",
"input": "in_42"
}
}
})
jobs_scenario = jobs.SaharaJob(self.context)
@ -181,11 +187,13 @@ class SaharaJobTestCase(test.ScenarioTestCase):
self.context.update({
"tenant": {
"sahara_image": "test_image",
"sahara_mains": ["main_42"],
"sahara_libs": ["lib_42"],
"sahara_cluster": "cl_42",
"sahara_input": "in_42"
"sahara": {
"image": "test_image",
"mains": ["main_42"],
"libs": ["lib_42"],
"cluster": "cl_42",
"input": "in_42"
}
}
})
jobs_scenario = jobs.SaharaJob(self.context)

16
tests/unit/plugins/openstack/scenarios/sahara/test_utils.py
View File

@ -437,9 +437,11 @@ class SaharaScenarioTestCase(test.ScenarioTestCase):
return_value="42")
def test_create_output_ds(self, mock_generate_random_name):
self.context.update({
"sahara_output_conf": {
"output_type": "hdfs",
"output_url_prefix": "hdfs://test_out/"
"sahara": {
"output_conf": {
"output_type": "hdfs",
"output_url_prefix": "hdfs://test_out/"
}
}
})
@ -457,9 +459,11 @@ class SaharaScenarioTestCase(test.ScenarioTestCase):
return_value="42")
def test_create_output_ds_swift(self, mock_generate_random_name):
self.context.update({
"sahara_output_conf": {
"output_type": "swift",
"output_url_prefix": "swift://test_out/"
"sahara": {
"output_conf": {
"output_type": "swift",
"output_url_prefix": "swift://test_out/"
}
}
})