129 lines
5.1 KiB
Python
129 lines
5.1 KiB
Python
# Copyright 2015: Mirantis Inc.
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import requests
|
|
from six.moves.urllib import parse
|
|
|
|
from rally.common import utils as rutils
|
|
from rally.common import validation
|
|
from rally import consts
|
|
from rally.task import context
|
|
|
|
from rally_openstack.cleanup import manager as resource_manager
|
|
from rally_openstack import osclients
|
|
from rally_openstack.scenarios.sahara import utils
|
|
from rally_openstack.scenarios.swift import utils as swift_utils
|
|
|
|
|
|
@validation.add("required_platform", platform="openstack", users=True)
|
|
@context.configure(name="sahara_input_data_sources", platform="openstack",
|
|
order=443)
|
|
class SaharaInputDataSources(context.Context):
|
|
"""Context class for setting up Input Data Sources for an EDP job."""
|
|
|
|
CONFIG_SCHEMA = {
|
|
"type": "object",
|
|
"$schema": consts.JSON_SCHEMA,
|
|
"properties": {
|
|
"input_type": {
|
|
"enum": ["swift", "hdfs"],
|
|
},
|
|
"input_url": {
|
|
"type": "string",
|
|
},
|
|
"swift_files": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"name": {
|
|
"type": "string"
|
|
},
|
|
"download_url": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"additionalProperties": False,
|
|
"required": ["name", "download_url"]
|
|
}
|
|
}
|
|
},
|
|
"additionalProperties": False,
|
|
"required": ["input_type", "input_url"]
|
|
}
|
|
|
|
def setup(self):
|
|
utils.init_sahara_context(self)
|
|
self.context["sahara"]["swift_objects"] = []
|
|
self.context["sahara"]["container_name"] = None
|
|
|
|
for user, tenant_id in rutils.iterate_per_tenants(
|
|
self.context["users"]):
|
|
clients = osclients.Clients(user["credential"])
|
|
if self.config["input_type"] == "swift":
|
|
self.setup_inputs_swift(clients, tenant_id,
|
|
self.config["input_url"],
|
|
self.config["swift_files"],
|
|
user["credential"].username,
|
|
user["credential"].password)
|
|
else:
|
|
self.setup_inputs(clients, tenant_id,
|
|
self.config["input_type"],
|
|
self.config["input_url"])
|
|
|
|
def setup_inputs(self, clients, tenant_id, input_type, input_url):
|
|
input_ds = clients.sahara().data_sources.create(
|
|
name=self.generate_random_name(),
|
|
description="",
|
|
data_source_type=input_type,
|
|
url=input_url)
|
|
|
|
self.context["tenants"][tenant_id]["sahara"]["input"] = input_ds.id
|
|
|
|
def setup_inputs_swift(self, clients, tenant_id, input_url,
|
|
swift_files, username, password):
|
|
swift_scenario = swift_utils.SwiftScenario(clients=clients,
|
|
context=self.context)
|
|
# TODO(astudenov): use self.generate_random_name()
|
|
container_name = "rally_" + parse.urlparse(input_url).netloc.rstrip(
|
|
".sahara")
|
|
self.context["sahara"]["container_name"] = (
|
|
swift_scenario._create_container(container_name=container_name))
|
|
for swift_file in swift_files:
|
|
content = requests.get(swift_file["download_url"]).content
|
|
self.context["sahara"]["swift_objects"].append(
|
|
swift_scenario._upload_object(
|
|
self.context["sahara"]["container_name"], content,
|
|
object_name=swift_file["name"]))
|
|
input_ds_swift = clients.sahara().data_sources.create(
|
|
name=self.generate_random_name(), description="",
|
|
data_source_type="swift", url=input_url,
|
|
credential_user=username, credential_pass=password)
|
|
|
|
self.context["tenants"][tenant_id]["sahara"]["input"] = (
|
|
input_ds_swift.id)
|
|
|
|
def cleanup(self):
|
|
resource_manager.cleanup(
|
|
names=["swift.object", "swift.container"],
|
|
users=self.context.get("users", []),
|
|
superclass=swift_utils.SwiftScenario,
|
|
task_id=self.get_owner_id())
|
|
resource_manager.cleanup(
|
|
names=["sahara.data_sources"],
|
|
users=self.context.get("users", []),
|
|
superclass=self.__class__,
|
|
task_id=self.get_owner_id())
|