x/scalpels
Code Issues Proposed changes

enable pep8

Browse Source 
Change-Id: Ie74349f65747cbe9703e58f071b5a12777d1e981
This commit is contained in:
Kun Huang 2015-12-07 13:51:11 +08:00
parent 736a006fe2
commit 577a1920cb
20 changed files with 357 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
scalpels/agents/base.py
View File

@ -4,16 +4,20 @@
import subprocess import subprocess
def run_agent(task_uuid, ag): def run_agent(task_uuid, ag):
""" """Example
python <path-to-dir>/agent.py <uuid> mysql python <path-to-dir>/agent.py <uuid> mysql
""" """
cmd = "sca-tracer %s %s" % (task_uuid, ag) cmd = "sca-tracer %s %s" % (task_uuid, ag)
ag = subprocess.Popen(cmd.split()) ag = subprocess.Popen(cmd.split())
return ag.pid return ag.pid
def _parse_traffic(out, name): def _parse_traffic(out, name):
""" """Example
in: in:
ts, 123.00 pkts 2312 bytes ts, 123.00 pkts 2312 bytes
... ...
@ -31,11 +35,11 @@ def _parse_traffic(out, name):
pkts_ret = {"name": ag_name, pkts_ret = {"name": ag_name,
"unit": "pkts", "unit": "pkts",
"data": [], "data": [],
"rtype": "stream",} "rtype": "stream"}
bytes_ret = {"name": ag_name, bytes_ret = {"name": ag_name,
"unit": "bytes", "unit": "bytes",
"data": [], "data": [],
"rtype": "stream",} "rtype": "stream"}
for ts, _t in out: for ts, _t in out:
pkts, pkts_unit, bytes, bytes_unit = _t.split(" ", 3) pkts, pkts_unit, bytes, bytes_unit = _t.split(" ", 3)
pkts_ret["data"].append((ts, pkts)) pkts_ret["data"].append((ts, pkts))
@ -43,16 +47,24 @@ def _parse_traffic(out, name):
return (pkts_ret, bytes_ret) return (pkts_ret, bytes_ret)
def parse_rpc(out): def parse_rpc(out):
return _parse_traffic(out, "Port") return _parse_traffic(out, "Port")
def parse_traffic(out): def parse_traffic(out):
return _parse_traffic(out, "Device") return _parse_traffic(out, "Device")
def parse_rabbit(out): def parse_rabbit(out):
""" """Example
in: in:
ts, {u'_unique_id': u'xxx', u'failure': None, u'ending': True, u'result': None, u'_msg_id': u'xxx'} ts, {u'_unique_id': u'xxx',
u'failure': None,
u'ending': True,
u'result': None,
u'_msg_id': u'xxx'}
out: out:
name: RabbitMQ name: RabbitMQ
unit: None unit: None
@ -64,6 +76,7 @@ def parse_rabbit(out):
"data": out} "data": out}
return (rbt_ret, ) return (rbt_ret, )
def _parse_count_stream(out, name): def _parse_count_stream(out, name):
ret = {"name": name, ret = {"name": name,
"unit": "count", "unit": "count",
@ -71,8 +84,10 @@ def _parse_count_stream(out, name):
"data": out} "data": out}
return (ret, ) return (ret, )
def parse_oslolock(out): def parse_oslolock(out):
""" """Example
in: in:
ts, 4 ts, 4
ts, 0 ts, 0
@ -84,11 +99,14 @@ def parse_oslolock(out):
""" """
return _parse_count_stream(out, "Oslo-Lock") return _parse_count_stream(out, "Oslo-Lock")
def parse_modelsave(out): def parse_modelsave(out):
return _parse_count_stream(out, "Model-Save") return _parse_count_stream(out, "Model-Save")
def parse_sqlaexec(out): def parse_sqlaexec(out):
return _parse_count_stream(out, "Sqlalchemy-Execute") return _parse_count_stream(out, "Sqlalchemy-Execute")
def parse_rpccount(out): def parse_rpccount(out):
return _parse_count_stream(out, "RPC-Count") return _parse_count_stream(out, "RPC-Count")

85
scalpels/agents/server.py
View File

@ -1,13 +1,16 @@
from oslo_config import cfg
import oslo_messaging
from scalpels.db import api as db_api
from scalpels.agents.base import run_agent
import psutil
import signal import signal
from oslo_config import cfg
from oslo_log import log as logging from oslo_log import log as logging
import oslo_messaging
import psutil
from scalpels.agents import base
from scalpels.db import api as db_api
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
class ServerControlEndpoint(object): class ServerControlEndpoint(object):
target = oslo_messaging.Target(topic="test", version='1.0') target = oslo_messaging.Target(topic="test", version='1.0')
@ -19,6 +22,7 @@ class ServerControlEndpoint(object):
if server: if server:
self.server.stop() self.server.stop()
class TracerEndpoint(object): class TracerEndpoint(object):
target = oslo_messaging.Target(topic="test", version='1.0') target = oslo_messaging.Target(topic="test", version='1.0')
@ -27,16 +31,17 @@ class TracerEndpoint(object):
tracers = db_api.tracer_list() tracers = db_api.tracer_list()
ret = list() ret = list()
for tr in tracers: for tr in tracers:
ret.append({"name":tr.name, ret.append({"name": tr.name,
"tpl":tr.template, "tpl": tr.template,
"running":tr.is_running, "running": tr.is_running,
"pid":tr.pid, "pid": tr.pid,
"results":tr.results}) "results": tr.results})
return ret return ret
def start_tracers(self, ctx, tracers): def start_tracers(self, ctx, tracers):
all_tr = self.tracer_list(ctx) all_tr = self.tracer_list(ctx)
running_tr = map(lambda t:t["name"], filter(lambda t:t["running"], all_tr)) running_tr = map(lambda t: t["name"],
filter(lambda t: t["running"], all_tr))
task = db_api.task_create(results=[], pids=[]) task = db_api.task_create(results=[], pids=[])
pids = [] pids = []
@ -44,7 +49,7 @@ class TracerEndpoint(object):
if tr in running_tr: if tr in running_tr:
LOG.info("%s is running, skipped" % tr) LOG.info("%s is running, skipped" % tr)
else: else:
pid = run_agent(task.uuid, tr) pid = base.run_agent(task.uuid, tr)
LOG.debug("saving pid %s" % pid) LOG.debug("saving pid %s" % pid)
self.set_tracer_pid(ctx, tr, pid) self.set_tracer_pid(ctx, tr, pid)
pids.append(pid) pids.append(pid)
@ -53,7 +58,7 @@ class TracerEndpoint(object):
print "[LOG] task <%s> runs successfully!" % task.uuid print "[LOG] task <%s> runs successfully!" % task.uuid
def stop_tracers(self, ctx, tracers): def stop_tracers(self, ctx, tracers):
all_tr = {t["name"]:t for t in self.tracer_list(ctx)} all_tr = {t["name"]: t for t in self.tracer_list(ctx)}
for tr in tracers: for tr in tracers:
if tr not in all_tr.keys(): if tr not in all_tr.keys():
LOG.info("%s is not registered" % tr) LOG.info("%s is not registered" % tr)
@ -66,20 +71,20 @@ class TracerEndpoint(object):
p = psutil.Process(int(pid)) p = psutil.Process(int(pid))
p.send_signal(signal.SIGINT) p.send_signal(signal.SIGINT)
def register_tracer(self, ctx, tracer_opts): def register_tracer(self, ctx, tracer_opts):
db_api.register_tracer(name=tracer_opts["name"], template=tracer_opts["tpl"]) db_api.register_tracer(name=tracer_opts["name"],
template=tracer_opts["tpl"])
print "[LOG] registering tracer %(name)s: %(tpl)s" % tracer_opts print "[LOG] registering tracer %(name)s: %(tpl)s" % tracer_opts
def set_tracer_stat(self, ctx, tracer, running): def set_tracer_stat(self, ctx, tracer, running):
running=bool(running) running = bool(running)
print "[LOG] setting tracer: %s running: %s" % (tracer, running) print "[LOG] setting tracer: %s running: %s" % (tracer, running)
db_api.tracer_update(tracer, running=running) db_api.tracer_update(tracer, running=running)
def set_tracer_pid(self, ctx, tracer, pid): def set_tracer_pid(self, ctx, tracer, pid):
db_api.tracer_update(tracer, pid=pid) db_api.tracer_update(tracer, pid=pid)
class TaskEndpoint(object): class TaskEndpoint(object):
target = oslo_messaging.Target(topic="test", version='1.0') target = oslo_messaging.Target(topic="test", version='1.0')
@ -95,16 +100,17 @@ class TaskEndpoint(object):
def get_task(self, ctx, uuid, fuzzy): def get_task(self, ctx, uuid, fuzzy):
print "[LOG] reading task: %s" % uuid print "[LOG] reading task: %s" % uuid
task = db_api.task_get(uuid, fuzzy) task = db_api.task_get(uuid, fuzzy)
# TODO object # TODO(kun) object
return {"uuid":task.uuid, return {"uuid": task.uuid,
"results":task.results,} "results": task.results}
def get_latest_task(self, ctx): def get_latest_task(self, ctx):
task = db_api.task_get_last() task = db_api.task_get_last()
print "[LOG] reading latest task: %s" % task.uuid print "[LOG] reading latest task: %s" % task.uuid
# TODO object # TODO(kun) object
return {"uuid":task.uuid, return {"uuid": task.uuid,
"results":task.results,} "results": task.results}
class ResultEndpoint(object): class ResultEndpoint(object):
@ -113,36 +119,37 @@ class ResultEndpoint(object):
def get_result(self, ctx, uuid): def get_result(self, ctx, uuid):
print "[LOG] reading result: %s" % uuid print "[LOG] reading result: %s" % uuid
ret = db_api.result_get(uuid) ret = db_api.result_get(uuid)
# TODO object # TODO(kun) object
return { return {"id": ret.id,
"id":ret.id, "uuid": ret.uuid,
"uuid":ret.uuid, "name": ret.name,
"name":ret.name, "unit": ret.unit,
"unit":ret.unit, "data": ret.data,
"data":ret.data, "rtype": ret.rtype}
"rtype":ret.rtype,
}
def get_all_results(self, ctx): def get_all_results(self, ctx):
print "[LOG] reading all results" print "[LOG] reading all results"
rets = db_api.get_all_results() rets = db_api.get_all_results()
# TODO object # TODO(kun) object
return [{"id":ret.id, return [{"id": ret.id,
"uuid":ret.uuid, "uuid": ret.uuid,
"name":ret.name, "name": ret.name,
"unit":ret.unit, "unit": ret.unit,
"data":ret.data, "data": ret.data,
"rtype":ret.rtype} for ret in rets] "rtype": ret.rtype} for ret in rets]
class ConfigEndpoint(object): class ConfigEndpoint(object):
target = oslo_messaging.Target(topic="test", version='1.0') target = oslo_messaging.Target(topic="test", version='1.0')
def update_config(self, ctx, data_opts): def update_config(self, ctx, data_opts):
db_api.update_config(data_opts) db_api.update_config(data_opts)
def get_config(self, ctx): def get_config(self, ctx):
return db_api.get_config() return db_api.get_config()
transport = oslo_messaging.get_transport(cfg.CONF) transport = oslo_messaging.get_transport(cfg.CONF)
target = oslo_messaging.Target(topic='test', server='localhost') target = oslo_messaging.Target(topic='test', server='localhost')
endpoints = [ endpoints = [

16
scalpels/client/actions/load.py
View File

@ -2,9 +2,11 @@
#-*- coding:utf-8 -*- #-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
from novaclient import client
import os import os
from novaclient import client
def run(config): def run(config):
loads = (k for k in config if config[k]) loads = (k for k in config if config[k])
for load in loads: for load in loads:
@ -15,13 +17,15 @@ def run(config):
continue continue
loadcall(config) loadcall(config)
def _get_creds_from_env(): def _get_creds_from_env():
user = os.environ.get("OS_USERNAME") user = os.environ.get("OS_USERNAME")
pw = os.environ.get("OS_PASSWORD") pw = os.environ.get("OS_PASSWORD")
tenant = os.environ.get("OS_TENANT_NAME") tenant = os.environ.get("OS_TENANT_NAME")
auth_url= os.environ.get("OS_AUTH_URL") auth_url = os.environ.get("OS_AUTH_URL")
return (user, pw, tenant, auth_url) return (user, pw, tenant, auth_url)
def nova_boot_bulk(): def nova_boot_bulk():
creds = _get_creds_from_env() creds = _get_creds_from_env()
if None in creds: if None in creds:
@ -30,11 +34,13 @@ def nova_boot_bulk():
image = nova.images.find(name="cirros-0.3.4-x86_64-uec") image = nova.images.find(name="cirros-0.3.4-x86_64-uec")
flavor = nova.flavors.get("3") flavor = nova.flavors.get("3")
net = nova.networks.find(label="private") net = nova.networks.find(label="private")
nics = [{"net-id":net.id}] nics = [{"net-id": net.id}]
ret = nova.servers.create(name="bulk-foo", image=image, flavor=flavor, min_count=2, max_count=2, nics=nics) ret = nova.servers.create(name="bulk-foo", image=image, flavor=flavor,
min_count=2, max_count=2, nics=nics)
print "we got %s" % ret print "we got %s" % ret
return return
def load_storm(config): def load_storm(config):
#TODO use novaclient python api to do this #TODO(kun) use novaclient python api to do this
nova_boot_bulk() nova_boot_bulk()

18
scalpels/client/actions/report.py
View File

@ -2,10 +2,12 @@
#-*- coding:utf-8 -*- #-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
from scalpels.client.api import api as agent_api from scalpels.client.actions.start import _parse_agents_from_args # noqa
from scalpels.client.utils import generate_multiple_result_html from scalpels.client import api
from scalpels.client.utils import pprint_result from scalpels.client import utils
from scalpels.client.actions.start import _parse_agents_from_args
agent_api = api.api
def run(config): def run(config):
@ -13,11 +15,11 @@ def run(config):
raise ValueError("Can't stop both a task and a tracer") raise ValueError("Can't stop both a task and a tracer")
elif config["agent"]: elif config["agent"]:
req_tr = _parse_agents_from_args(config) req_tr = _parse_agents_from_args(config)
all_tr = {t["name"]:t for t in agent_api.get_tracer_list()} all_tr = {t["name"]: t for t in agent_api.get_tracer_list()}
for tr in req_tr: for tr in req_tr:
for ret_uuid in all_tr[tr]["results"]: for ret_uuid in all_tr[tr]["results"]:
ret = agent_api.get_result(ret_uuid) ret = agent_api.get_result(ret_uuid)
pprint_result(ret) utils.pprint_result(ret)
else: else:
task = agent_api.try_get_task_from_config(config) task = agent_api.try_get_task_from_config(config)
@ -28,6 +30,6 @@ def run(config):
ret = agent_api.get_result(ret_uuid) ret = agent_api.get_result(ret_uuid)
rets.append(ret) rets.append(ret)
if config.get("html"): if config.get("html"):
generate_multiple_result_html(rets) utils.generate_multiple_result_html(rets)
else: else:
map(pprint_result, rets) map(utils.pprint_result, rets)

21
scalpels/client/actions/result.py
View File

@ -2,13 +2,16 @@
#-*- coding:utf-8 -*- #-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
from scalpels.client.utils import generate_multiple_result_html from scalpels.client import api
from scalpels.client.utils import pprint_result from scalpels.client import utils
from scalpels.client.utils import generate_result_html
from scalpels.client.api import api as agent_api
agent_api = api.api
def run(config): def run(config):
""" """Run Command
uuid: pprint it uuid: pprint it
list: pprint all list: pprint all
uuid and html: generate_result_html uuid and html: generate_result_html
@ -17,15 +20,15 @@ def run(config):
if config.get("list"): if config.get("list"):
rets = agent_api.get_all_results() rets = agent_api.get_all_results()
if config.get("html"): if config.get("html"):
generate_multiple_result_html(rets) utils.generate_multiple_result_html(rets)
elif config.get("short"): elif config.get("short"):
for ret in rets: for ret in rets:
print ret["uuid"] print ret["uuid"]
else: else:
map(pprint_result, rets) map(utils.pprint_result, rets)
elif config.get("uuid"): elif config.get("uuid"):
ret = agent_api.get_result(config["uuid"]) ret = agent_api.get_result(config["uuid"])
if config.get("html"): if config.get("html"):
generate_result_html(ret) utils.generate_result_html(ret)
else: else:
pprint_result(ret) utils.pprint_result(ret)

12
scalpels/client/actions/start.py
View File

@ -2,9 +2,14 @@
#-*- coding:utf-8 -*- #-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
import os
import json import json
from scalpels.client.api import api as agent_api import os
from scalpels.client import api
agent_api = api.api
def _parse_agents_from_args(config): def _parse_agents_from_args(config):
parsed_agents = set() parsed_agents = set()
@ -12,7 +17,7 @@ def _parse_agents_from_args(config):
return parsed_agents return parsed_agents
for ag in config.get("agent"): for ag in config.get("agent"):
parsed_agents |= set(ag.split(",")) parsed_agents |= set(ag.split(","))
config.update({"agent":parsed_agents}) config.update({"agent": parsed_agents})
return parsed_agents return parsed_agents
@ -29,6 +34,7 @@ def _parse_agents_from_file(config):
parsed_agents.add(ag["name"]) parsed_agents.add(ag["name"])
return parsed_agents return parsed_agents
def run(config): def run(config):
print "command start: %s" % config print "command start: %s" % config
agents = _parse_agents_from_args(config) agents = _parse_agents_from_args(config)

7
scalpels/client/actions/stop.py
View File

@ -2,8 +2,11 @@
#-*- coding:utf-8 -*- #-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
from scalpels.client.api import api as agent_api from scalpels.client.actions.start import _parse_agents_from_args # noqa
from scalpels.client.actions.start import _parse_agents_from_args from scalpels.client import api
agent_api = api.api
def run(config): def run(config):

12
scalpels/client/actions/tracer.py
View File

@ -2,13 +2,17 @@
#-*- coding:utf-8 -*- #-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
from scalpels.client.api import api as agent_api import prettytable
from prettytable import PrettyTable
from scalpels.client import api
agent_api = api.api
def run(config): def run(config):
tracers = agent_api.get_tracer_list() tracers = agent_api.get_tracer_list()
t = PrettyTable(["tracer", "tracer template", "is running", "pid"]) t = prettytable.PrettyTable(["tracer", "tracer template",
"is running", "pid"])
for tr in tracers: for tr in tracers:
t.add_row([tr["name"],tr["tpl"],tr["running"],tr["pid"]]) t.add_row([tr["name"], tr["tpl"], tr["running"], tr["pid"]])
print t print t

12
scalpels/client/api.py
View File

@ -2,9 +2,12 @@
#-*- coding:utf-8 -*- #-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
from scalpels.client import rpcapi
from scalpels.client import utils
UUID_LOWEST_SUPPORT = utils.UUID_LOWEST_SUPPORT
rpcapi = rpcapi.rpcapi
from scalpels.client.rpcapi import rpcapi
from scalpels.client.utils import UUID_LOWEST_SUPPORT
class API(object): class API(object):
def __init__(self): def __init__(self):
@ -24,7 +27,8 @@ class API(object):
def get_task(self, uuid, fuzzy=False): def get_task(self, uuid, fuzzy=False):
if fuzzy and len(uuid) < UUID_LOWEST_SUPPORT: if fuzzy and len(uuid) < UUID_LOWEST_SUPPORT:
raise ValueError("fuzzy uuid query must get %s length" % UUID_LOWEST_SUPPORT) raise ValueError("fuzzy uuid query must get %s length" %
UUID_LOWEST_SUPPORT)
return rpcapi.get_task(uuid=uuid, fuzzy=fuzzy) return rpcapi.get_task(uuid=uuid, fuzzy=fuzzy)
def get_latest_task(self): def get_latest_task(self):
@ -44,7 +48,7 @@ class API(object):
raise ValueError("can't assign last and uuid togther") raise ValueError("can't assign last and uuid togther")
elif uuid: elif uuid:
return self.get_task(uuid, fuzzy=True) return self.get_task(uuid, fuzzy=True)
else: # no matter whether last is set else: # no matter whether last is set
return self.get_latest_task() return self.get_latest_task()
def register_tracer(self, tracer_opts): def register_tracer(self, tracer_opts):

6
scalpels/client/rpcapi.py
View File

@ -1,5 +1,6 @@
import oslo_messaging as messaging
from oslo_config import cfg from oslo_config import cfg
import oslo_messaging as messaging
class RPCAPI(object): class RPCAPI(object):
@ -41,7 +42,8 @@ class RPCAPI(object):
return self._client.call(ctxt, "get_config") return self._client.call(ctxt, "get_config")
def set_tracer_stat(self, ctxt={}, tracer=None, running=None): def set_tracer_stat(self, ctxt={}, tracer=None, running=None):
self._client.cast(ctxt, "set_tracer_stat", tracer=tracer, running=running) self._client.cast(ctxt, "set_tracer_stat", tracer=tracer,
running=running)
def set_tracer_pid(self, ctxt={}, tracer=None, pid=None): def set_tracer_pid(self, ctxt={}, tracer=None, pid=None):
self._client.cast(ctxt, "set_tracer_pid", tracer=tracer, pid=pid) self._client.cast(ctxt, "set_tracer_pid", tracer=tracer, pid=pid)

52
scalpels/client/shell.py
View File

@ -13,50 +13,64 @@ def run(parser):
func = getattr(mod, "run") func = getattr(mod, "run")
return func(config) return func(config)
def main(): def main():
rootparser = argparse.ArgumentParser(description="main entry point for scalpels") rootparser = argparse.ArgumentParser(description="main entry of scalpels")
subparsers = rootparser.add_subparsers(title="actions", dest="action") subparsers = rootparser.add_subparsers(title="actions", dest="action")
# setup load actions # setup load actions
load = subparsers.add_parser("load") load = subparsers.add_parser("load")
load.add_argument("--storm", action="store_true", dest="storm", help="run concurrency nova boot") load.add_argument("--storm", action="store_true", dest="storm",
help="run concurrency nova boot")
# setup start actions # setup start actions
start = subparsers.add_parser("start") start = subparsers.add_parser("start")
start.add_argument("-f", "--file", action="store", dest="file", help="config file for this task", required=False) start.add_argument("-f", "--file", action="store", dest="file",
start.add_argument("-a", "--agent", action="append", dest="agent", help="agent(s) to run", required=False) help="config file for this task", required=False)
start.add_argument("-a", "--agent", action="append", dest="agent",
help="agent(s) to run", required=False)
# setup stop actions # setup stop actions
stop = subparsers.add_parser("stop") stop = subparsers.add_parser("stop")
stop.add_argument("--last", action="store_true", dest="last", help="report the last task") stop.add_argument("--last", action="store_true", dest="last",
stop.add_argument("uuid", type=str, default="", nargs="?", help="report the last task") help="report the last task")
stop.add_argument("-a", "--agent", action="append", dest="agent", help="agent(s) to stop", required=False) stop.add_argument("uuid", type=str, default="", nargs="?",
help="report the last task")
stop.add_argument("-a", "--agent", action="append", dest="agent",
help="agent(s) to stop", required=False)
# setup report actions # setup report actions
report = subparsers.add_parser("report") report = subparsers.add_parser("report")
report.add_argument("--last", action="store_true", dest="last", help="report the last task") report.add_argument("--last", action="store_true", dest="last",
report.add_argument("--html", action="store_true", dest="html", help="report html to stdout instead of pretty print") help="report the last task")
report.add_argument("uuid", type=str, default="", nargs="?", help="report the last task") report.add_argument("--html", action="store_true", dest="html",
report.add_argument("-a", "--agent", action="append", dest="agent", help="agent(s) to stop", required=False) help="report html to stdout instead of pretty print")
report.add_argument("uuid", type=str, default="", nargs="?",
help="report the last task")
report.add_argument("-a", "--agent", action="append", dest="agent",
help="agent(s) to stop", required=False)
# setup sca result --list # setup sca result --list
result = subparsers.add_parser("result") result = subparsers.add_parser("result")
result.add_argument("-l", "--list", action="store_true", dest="list", help="list all results from db") result.add_argument("-l", "--list", action="store_true", dest="list",
result.add_argument("uuid", type=str, default="", nargs="?", help="report the last task") help="list all results from db")
result.add_argument("--html", action="store_true", dest="html", help="report html to stdout instead of pretty print") result.add_argument("uuid", type=str, default="", nargs="?",
result.add_argument("--short", action="store_true", dest="short", help="report uuid only") help="report the last task")
result.add_argument("--html", action="store_true", dest="html",
help="report html to stdout instead of pretty print")
result.add_argument("--short", action="store_true", dest="short",
help="report uuid only")
# agent command # agent command
tracer = subparsers.add_parser("tracer") tracer = subparsers.add_parser("tracer")
tracer.add_argument("-l", "--list", action="store_true", dest="list", help="list all agents") tracer.add_argument("-l", "--list", action="store_true", dest="list",
help="list all agents")
parser = rootparser.parse_args() parser = rootparser.parse_args()
try: try:
run(parser) run(parser)
except Exception as e: except Exception as e:
raise raise e
return 1
else: else:
return 0 return 0

36
scalpels/client/utils.py
View File

@ -2,44 +2,50 @@
#-*- coding:utf-8 -*- #-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
from mako.lookup import TemplateLookup
from prettytable import PrettyTable
from scalpels import templates
import os import os
from mako import lookup as lookup_m
import prettytable
from scalpels import templates
UUID_LOWEST_SUPPORT = 8 UUID_LOWEST_SUPPORT = 8
# TODO this map should be saved in a config file
# TODO refar to pre/exec/post # TODO(kun) this map should be saved in a config file
# TODO(kun) refar to pre/exec/post
tracers_map = { tracers_map = {
"mysql": "bash %s/mysql-live.sh", #XXX doesn't work now, needs works on interapt pipeline "mysql": "bash %s/mysql-live.sh", #XXX doesn't work now, see bug 1512276
"rabbit": "python %s/rbt-trace.py", "rabbit": "python %s/rbt-trace.py",
"rpc": "bash %s/port-input-traffic.sh 5672", "rpc": "bash %s/port-input-traffic.sh 5672",
"traffic": "bash %s/device-input-traffic.sh eth0", "traffic": "bash %s/device-input-traffic.sh eth0",
"oslolock": "stap %s/oslo-lock.stp", # with sudo, need add current user to stapdev group "oslolock": "stap %s/oslo-lock.stp",
"modelsave": "stap %s/model-save.stp", # with sudo, need add current user to stapdev group "modelsave": "stap %s/model-save.stp",
"sqlaexec": "stap %s/sqla-exec.stp", # with sudo, need add current user to stapdev group "sqlaexec": "stap %s/sqla-exec.stp",
"rpccount": "stap %s/rpc-count.stp", # with sudo, need add current user to stapdev group "rpccount": "stap %s/rpc-count.stp",
} }
def generate_result_html(result): def generate_result_html(result):
if result["rtype"] == "stream": if result["rtype"] == "stream":
tmpl_dir = os.path.dirname(templates.__file__) tmpl_dir = os.path.dirname(templates.__file__)
lookup = TemplateLookup(directories=[tmpl_dir]) lookup = lookup_m.TemplateLookup(directories=[tmpl_dir])
t = lookup.get_template("line-chart.mako") t = lookup.get_template("line-chart.mako")
print t.render(**result) print t.render(**result)
def generate_multiple_result_html(results): def generate_multiple_result_html(results):
tmpl_dir = os.path.dirname(templates.__file__) tmpl_dir = os.path.dirname(templates.__file__)
lookup = TemplateLookup(directories=[tmpl_dir]) lookup = lookup_m.TemplateLookup(directories=[tmpl_dir])
t = lookup.get_template("multi-line-chart.mako") t = lookup.get_template("multi-line-chart.mako")
d = {"results": results} d = {"results": results}
print t.render(**d) print t.render(**d)
def pprint_result(result): def pprint_result(result):
print "<result %s>" % result["uuid"] print "<result %s>" % result["uuid"]
t = PrettyTable(["timestamp", "%s (%s)" % (result["name"], result["unit"])]) t = prettytable.PrettyTable(["timestamp", "%s (%s)" % (result["name"],
result["unit"])])
for data in result["data"]: for data in result["data"]:
t.add_row([data[0], data[1][:100]]) t.add_row([data[0], data[1][:100]])
print t print t

13
scalpels/cmd/agent.py
View File

@ -2,16 +2,19 @@
#-*- coding:utf-8 -*- #-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
from scalpels.agents.server import server
from oslo_log import log as logging
from oslo_config import cfg from oslo_config import cfg
from oslo_log import log as logging
from scalpels.agents import server
def main(): def main():
# TODO handle stop later # TODO(kun) handle stop later
logging.register_options(cfg.CONF) logging.register_options(cfg.CONF)
logging.setup(cfg.CONF, "scalpels") logging.setup(cfg.CONF, "scalpels")
server.start() server.server.start()
server.wait() server.server.wait()
if __name__ == "__main__": if __name__ == "__main__":
main() main()

47
scalpels/cmd/manage.py
View File

@ -2,26 +2,33 @@
#-*- coding:utf-8 -*- #-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
import os
import argparse import argparse
import psutil import os
import signal import signal
import psutil
from scalpels.client import api
from scalpels.db import api as db_api from scalpels.db import api as db_api
from scalpels.client.api import api as agent_api
agent_api = api.api
def get_default_tracer_dir(): def get_default_tracer_dir():
# /opt/stack/data/ for devstack case # /opt/stack/data/ for devstack case
# /opt/stack/ # /opt/stack/
# TODO replace these ugly codes # TODO(kun) replace these ugly codes
cmd = os.path.dirname(__file__) cmd = os.path.dirname(__file__)
scalpels_py = os.path.dirname(cmd) scalpels_py = os.path.dirname(cmd)
scalpels_package = os.path.dirname(scalpels_py) scalpels_package = os.path.dirname(scalpels_py)
default_data_dir = os.path.join(scalpels_package, "scripts") default_data_dir = os.path.join(scalpels_package, "scripts")
return default_data_dir return default_data_dir
def do_db(parser): def do_db(parser):
setup_config = {"tracer_path":get_default_tracer_dir()} setup_config = {"tracer_path": get_default_tracer_dir()}
if parser.force: if parser.force:
print "recreating database" print "recreating database"
db_api.db_drop() db_api.db_drop()
@ -30,6 +37,7 @@ def do_db(parser):
print "creating database" print "creating database"
db_api.db_create(setup_config) db_api.db_create(setup_config)
def do_setup(parser): def do_setup(parser):
data_opts = dict(parser.data_opts) if parser.data_opts else None data_opts = dict(parser.data_opts) if parser.data_opts else None
@ -44,14 +52,15 @@ def do_setup(parser):
if parser.stat: if parser.stat:
config = agent_api.get_config() config = agent_api.get_config()
from prettytable import PrettyTable import prettytable
t = PrettyTable(["key", "value"]) t = prettytable.PrettyTable(["key", "value"])
for k,v in config.items(): for k, v in config.items():
t.add_row([k, v]) t.add_row([k, v])
print t print t
def do_stop(parser): def do_stop(parser):
# TODO call rpc server's stop API instead # TODO(kun) call rpc server's stop API instead
for p in psutil.process_iter(): for p in psutil.process_iter():
if p.as_dict()["cmdline"] and "sca-agent" in " ".join(p.cmdline): if p.as_dict()["cmdline"] and "sca-agent" in " ".join(p.cmdline):
print "killing process %d, %s" % (p.pid, p.cmdline) print "killing process %d, %s" % (p.pid, p.cmdline)
@ -60,21 +69,28 @@ def do_stop(parser):
print "Can't find sca-agent process" print "Can't find sca-agent process"
return return
def main(): def main():
rootparser = argparse.ArgumentParser(description="main entry point for scalpels") rootparser = argparse.ArgumentParser(description="entry point of scalpels")
subparsers = rootparser.add_subparsers(title="actions", dest="action") subparsers = rootparser.add_subparsers(title="actions", dest="action")
# db actions # db actions
db = subparsers.add_parser("db-create") db = subparsers.add_parser("db-create")
db.add_argument("-f", "--force", action="store_true", dest="force", help="re-create db") db.add_argument("-f", "--force", action="store_true",
dest="force", help="re-create db")
# setup re-setup actions # setup re-setup actions
setup = subparsers.add_parser("setup") setup = subparsers.add_parser("setup")
setup.add_argument("-d", "--data_opts", action="append", dest="data_opts", type=lambda kv:kv.split("="), help="data opts for tracer variables", required=False) setup.add_argument("-d", "--data_opts", action="append", dest="data_opts",
setup.add_argument("-t", "--tracer_opts", action="append", dest="tracer_opts", type=lambda kv:kv.split("="), help="tracer opts for registering", required=False) type=lambda kv: kv.split("="),
setup.add_argument("-s", "--stat", action="store_true", dest="stat", help="setup stats for this agent") help="data opts for tracer variables", required=False)
setup.add_argument("-t", "--tracer_opts", action="append",
dest="tracer_opts", type=lambda kv: kv.split("="),
help="tracer opts for registering", required=False)
setup.add_argument("-s", "--stat", action="store_true", dest="stat",
help="setup stats for this agent")
stop = subparsers.add_parser("stop") stop = subparsers.add_parser("stop") # noqa
parser = rootparser.parse_args() parser = rootparser.parse_args()
@ -87,5 +103,6 @@ def main():
if parser.action == "stop": if parser.action == "stop":
do_stop(parser) do_stop(parser)
if __name__ == "__main__": if __name__ == "__main__":
main() main()

24
scalpels/cmd/tracer.py
View File

@ -2,16 +2,17 @@
#-*- coding:utf-8 -*- #-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
import subprocess
import psutil
import sys
from scalpels.db import api as db_api
from copy import deepcopy as copy
import signal import signal
from tooz import coordination import subprocess
import sys
import time import time
import psutil
from tooz import coordination
from scalpels.agents import base from scalpels.agents import base
from scalpels.client.api import api as agent_api from scalpels.client import api
from scalpels.db import api as db_api
""" """
example: example:
@ -21,11 +22,13 @@ TODO:
config key-word arguments for each tracer config key-word arguments for each tracer
""" """
agent_api = api.api
worker_pid = None worker_pid = None
task_uuid = None task_uuid = None
out = None out = None
ag = None ag = None
def read_from_ag(ag): def read_from_ag(ag):
# wrong impl. here, need read from config or db instead # wrong impl. here, need read from config or db instead
config = agent_api.get_config() config = agent_api.get_config()
@ -35,6 +38,7 @@ def read_from_ag(ag):
return tr["tpl"] % config return tr["tpl"] % config
raise ValueError("tracer %s is not found" % ag) raise ValueError("tracer %s is not found" % ag)
def handle_int(signal, frame): def handle_int(signal, frame):
stop_tracer() stop_tracer()
save_result_to_task() save_result_to_task()
@ -42,6 +46,7 @@ def handle_int(signal, frame):
agent_api.set_tracer_stat(ag, running=False) agent_api.set_tracer_stat(ag, running=False)
sys.exit(0) sys.exit(0)
def stop_tracer(): def stop_tracer():
global worker_pid global worker_pid
# psutil is much more professional... I have to use it instead # psutil is much more professional... I have to use it instead
@ -49,12 +54,13 @@ def stop_tracer():
worker_p = psutil.Process(worker_pid) worker_p = psutil.Process(worker_pid)
worker_p.send_signal(signal.SIGINT) worker_p.send_signal(signal.SIGINT)
def save_result_to_task(): def save_result_to_task():
global task_uuid global task_uuid
global out global out
parse_func = getattr(base, "parse_%s" % ag) parse_func = getattr(base, "parse_%s" % ag)
# TODO file lock is okay in localhost, here need redis for distributed # TODO(kun) file lock is okay in localhost, here need redis for distributed
# lock istead # lock istead
co = coordination.get_coordinator("file:///tmp", b"localhost") co = coordination.get_coordinator("file:///tmp", b"localhost")
co.start() co.start()
@ -69,6 +75,7 @@ def save_result_to_task():
time.sleep(2) time.sleep(2)
co.stop() co.stop()
def main(): def main():
global worker_pid global worker_pid
global task_uuid global task_uuid
@ -93,6 +100,5 @@ def main():
out.append(_t) out.append(_t)
if __name__ == "__main__": if __name__ == "__main__":
main() main()

43
scalpels/db/api.py
View File

@ -3,83 +3,106 @@
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
from oslo_config import cfg
from oslo_db import api as db_api from oslo_db import api as db_api
from oslo_db import options as db_options from oslo_db import options as db_options
from oslo_config import cfg
CONF = cfg.CONF CONF = cfg.CONF
backend_mapping = {"sqlalchemy": "scalpels.db.sqlalchemy.api"}
IMPL = db_api.DBAPI("sqlalchemy", backend_mapping={"sqlalchemy":"scalpels.db.sqlalchemy.api"}) IMPL = db_api.DBAPI("sqlalchemy", backend_mapping)
db_options.set_defaults(CONF,
connection="sqlite:////tmp/scalpels.sqlite",
sqlite_db="scalpels.sqlite")
db_options.set_defaults(CONF, connection="sqlite:////tmp/scalpels.sqlite", sqlite_db="scalpels.sqlite")
def db_create(sc): def db_create(sc):
IMPL.db_create(sc) IMPL.db_create(sc)
def db_drop(): def db_drop():
IMPL.db_drop() IMPL.db_drop()
def result_create(name="", unit="", data=None, rtype=None): def result_create(name="", unit="", data=None, rtype=None):
""" """Create result
:param data: a list :) :param data: a list :)
:returns: result model obj :returns: result model obj
""" """
return IMPL.result_create(name, unit, data, rtype) return IMPL.result_create(name, unit, data, rtype)
def task_create(results, pids): def task_create(results, pids):
""" """Create task
:param results: a list contains result.uuid :param results: a list contains result.uuid
:returns: task model obj :returns: task model obj
""" """
return IMPL.task_create(results, pids) return IMPL.task_create(results, pids)
def task_get(task_uuid, fuzzy=False): def task_get(task_uuid, fuzzy=False):
return IMPL.task_get(task_uuid, fuzzy) return IMPL.task_get(task_uuid, fuzzy)
def task_update(task_uuid, results=None, pids=None): def task_update(task_uuid, results=None, pids=None):
return IMPL.task_update(task_uuid, results, pids) return IMPL.task_update(task_uuid, results, pids)
def task_append_result(task_uuid, result_uuid): def task_append_result(task_uuid, result_uuid):
return IMPL.task_append_result(task_uuid, result_uuid) return IMPL.task_append_result(task_uuid, result_uuid)
def task_get_last(): def task_get_last():
return IMPL.task_get_last() return IMPL.task_get_last()
def result_get(result_uuid): def result_get(result_uuid):
""" """Get result
:returns : dict, with data and its metadata :returns : dict, with data and its metadata
""" """
return IMPL.result_get(result_uuid) return IMPL.result_get(result_uuid)
def setup_config_get(): def setup_config_get():
"""
:returns : dict
"""
return IMPL.setup_config_get() return IMPL.setup_config_get()
def get_all_results(): def get_all_results():
return IMPL.get_all_results() return IMPL.get_all_results()
def register_tracer(name, template): def register_tracer(name, template):
return IMPL.register_tracer(name, template) return IMPL.register_tracer(name, template)
def tracer_list(): def tracer_list():
return IMPL.tracer_list() return IMPL.tracer_list()
def update_config(data_opts): def update_config(data_opts):
return IMPL.update_config(data_opts) return IMPL.update_config(data_opts)
def get_config(): def get_config():
return IMPL.get_config() return IMPL.get_config()
def tracer_get(tracer): def tracer_get(tracer):
""" """Get tracer
param tracer: tracer name, like, 'rpc' param tracer: tracer name, like, 'rpc'
""" """
return IMPL.tracer_get(tracer) return IMPL.tracer_get(tracer)
def tracer_update(tracer, running=None, pid=None): def tracer_update(tracer, running=None, pid=None):
return IMPL.tracer_update(tracer, running, pid) return IMPL.tracer_update(tracer, running, pid)
def tracer_append_result(tracer, result_uuid): def tracer_append_result(tracer, result_uuid):
return IMPL.tracer_append_result(tracer, result_uuid) return IMPL.tracer_append_result(tracer, result_uuid)

71
scalpels/db/sqlalchemy/api.py
View File

@ -2,18 +2,23 @@
#-*- coding:utf-8 -*- #-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
from oslo_config import cfg import copy
import sys import sys
from scalpels.db.sqlalchemy import BASE
from scalpels.db.sqlalchemy import models from oslo_config import cfg
from oslo_db.sqlalchemy import session as db_session from oslo_db.sqlalchemy import session as db_session
from oslo_db.sqlalchemy import utils as oslodbsqa_utils from oslo_db.sqlalchemy import utils as oslodbsqa_utils
from copy import deepcopy as copy
from scalpels.db import sqlalchemy
from scalpels.db.sqlalchemy import models
CONF = cfg.CONF CONF = cfg.CONF
_FACADE = None _FACADE = None
BASE = sqlalchemy.BASE
def _create_facade_lazily(): def _create_facade_lazily():
global _FACADE global _FACADE
@ -22,52 +27,61 @@ def _create_facade_lazily():
return _FACADE return _FACADE
def get_engine(): def get_engine():
facade = _create_facade_lazily() facade = _create_facade_lazily()
return facade.get_engine() return facade.get_engine()
def get_session(**kwargs): def get_session(**kwargs):
facade = _create_facade_lazily() facade = _create_facade_lazily()
return facade.get_session(**kwargs) return facade.get_session(**kwargs)
def get_backend(): def get_backend():
return sys.modules[__name__] return sys.modules[__name__]
def db_create(sc): def db_create(sc):
BASE.metadata.create_all(get_engine()) BASE.metadata.create_all(get_engine())
setup = models.Setup() setup = models.Setup()
setup.update({"config":sc}) setup.update({"config": sc})
setup.save() setup.save()
return setup return setup
def db_drop(): def db_drop():
BASE.metadata.drop_all(get_engine()) BASE.metadata.drop_all(get_engine())
def model_query(model, session=None): def model_query(model, session=None):
if session is None: if session is None:
session = get_session() session = get_session()
query = oslodbsqa_utils.model_query(model, session) query = oslodbsqa_utils.model_query(model, session)
return query return query
def result_create(name="", unit="", data=None, rtype=None): def result_create(name="", unit="", data=None, rtype=None):
result = models.Result() result = models.Result()
result.update({"name":name, result.update({"name": name,
"unit": unit, "unit": unit,
"data":data, "data": data,
"rtype": rtype if rtype else "unknown", "rtype": rtype if rtype else "unknown"})
})
result.save() result.save()
return result return result
def task_create(results, pids): def task_create(results, pids):
task = models.Task() task = models.Task()
task.update({"results":results, "pids":pids}) task.update({"results": results, "pids": pids})
task.save() task.save()
return task return task
def task_update(task_uuid, results=None, pids=None): def task_update(task_uuid, results=None, pids=None):
session = get_session() session = get_session()
task = model_query(models.Task, session=session).filter_by(uuid=task_uuid).first() task = (model_query(models.Task, session=session).
filter_by(uuid=task_uuid).first())
_update = dict() _update = dict()
if results: if results:
_update["results"] = results _update["results"] = results
@ -77,15 +91,18 @@ def task_update(task_uuid, results=None, pids=None):
task.save(session=session) task.save(session=session)
return task return task
def task_append_result(task_uuid, result_uuid): def task_append_result(task_uuid, result_uuid):
session = get_session() session = get_session()
task = model_query(models.Task, session=session).filter_by(uuid=task_uuid).first() task = (model_query(models.Task, session=session).
new = copy(task.results) filter_by(uuid=task_uuid).first())
new = copy.deepcopy(task.results)
new.append(result_uuid) new.append(result_uuid)
task.update({"results":new}) task.update({"results": new})
task.save(session=session) task.save(session=session)
return task return task
def task_get(task_uuid, fuzzy=False): def task_get(task_uuid, fuzzy=False):
if not fuzzy: if not fuzzy:
task = model_query(models.Task).filter_by(uuid=task_uuid).first() task = model_query(models.Task).filter_by(uuid=task_uuid).first()
@ -107,12 +124,14 @@ def result_get(result_uuid):
ret = model_query(models.Result).filter_by(uuid=result_uuid).first() ret = model_query(models.Result).filter_by(uuid=result_uuid).first()
return ret return ret
def task_get_last(): def task_get_last():
tasks = model_query(models.Task).all() tasks = model_query(models.Task).all()
if len(tasks) > 0: if len(tasks) > 0:
return tasks[-1] return tasks[-1]
return None return None
def setup_config_get(): def setup_config_get():
setups = model_query(models.Setup).all() setups = model_query(models.Setup).all()
if len(setups) > 0: if len(setups) > 0:
@ -120,40 +139,48 @@ def setup_config_get():
return setup.config return setup.config
return None return None
def get_all_results(): def get_all_results():
rets = model_query(models.Result).all() rets = model_query(models.Result).all()
return rets return rets
def register_tracer(name, template): def register_tracer(name, template):
tracer = models.Tracer() tracer = models.Tracer()
tracer.update({"name":name, "template": template, "results":[]}) tracer.update({"name": name, "template": template, "results": []})
tracer.save() tracer.save()
return tracer return tracer
def tracer_list(): def tracer_list():
tracers = model_query(models.Tracer).all() tracers = model_query(models.Tracer).all()
return tracers return tracers
def update_config(data_opts): def update_config(data_opts):
session = get_session() session = get_session()
config = model_query(models.Setup, session=session).first() config = model_query(models.Setup, session=session).first()
new = copy(config.config) new = copy.deepcopy(config.config)
new.update(data_opts) new.update(data_opts)
config.update({"config":new}) config.update({"config": new})
config.save(session=session) config.save(session=session)
return config return config
def get_config(): def get_config():
config = model_query(models.Setup).first() config = model_query(models.Setup).first()
return config.config return config.config
def tracer_get(tracer): def tracer_get(tracer):
tracer = model_query(models.Tracer).filter_by(name=tracer).first() tracer = model_query(models.Tracer).filter_by(name=tracer).first()
return tracer return tracer
def tracer_update(tracer_name, running=None, pid=None): def tracer_update(tracer_name, running=None, pid=None):
session = get_session() session = get_session()
tracer = model_query(models.Tracer, session=session).filter_by(name=tracer_name).first() tracer = (model_query(models.Tracer, session=session).
filter_by(name=tracer_name).first())
_update = {} _update = {}
if running is not None: if running is not None:
_update["is_running"] = running _update["is_running"] = running
@ -163,11 +190,13 @@ def tracer_update(tracer_name, running=None, pid=None):
tracer.save(session=session) tracer.save(session=session)
return tracer return tracer
def tracer_append_result(tracer_name, result_uuid): def tracer_append_result(tracer_name, result_uuid):
session = get_session() session = get_session()
tracer = model_query(models.Tracer, session=session).filter_by(name=tracer_name).first() tracer = (model_query(models.Tracer, session=session).
new = copy(tracer.results) filter_by(name=tracer_name).first())
new = copy.deepcopy(tracer.results)
new.append(result_uuid) new.append(result_uuid)
tracer.update({"results":new}) tracer.update({"results": new})
tracer.save(session=session) tracer.save(session=session)
return tracer return tracer

28
scalpels/db/sqlalchemy/models.py
View File

@ -2,26 +2,32 @@
#-*- coding:utf-8 -*- #-*- coding:utf-8 -*-
# Author: Kun Huang <academicgareth@gmail.com> # Author: Kun Huang <academicgareth@gmail.com>
import uuid
import json import json
import uuid
from sqlalchemy import types as sqa_types
from scalpels.db.sqlalchemy import BASE
from sqlalchemy import (Column, Integer, String, Boolean)
from oslo_db.sqlalchemy import models as oslodbmodels from oslo_db.sqlalchemy import models as oslodbmodels
from sqlalchemy import types as sqa_types
from sqlalchemy import (Column, Integer, String, Boolean)
from scalpels.db import sqlalchemy
BASE = sqlalchemy.BASE
class JSONEncodedData(sqa_types.TypeDecorator): class JSONEncodedData(sqa_types.TypeDecorator):
impl = sqa_types.Text impl = sqa_types.Text
def process_bind_param(self, value, dialect): def process_bind_param(self, value, dialect):
if value is not None: if value is not None:
value = json.dumps(value) value = json.dumps(value)
return value return value
def process_result_value(self, value, dialect): def process_result_value(self, value, dialect):
if value is not None: if value is not None:
value = json.loads(value) value = json.loads(value)
return value return value
class ScalpelsBase(oslodbmodels.ModelBase, oslodbmodels.TimestampMixin): class ScalpelsBase(oslodbmodels.ModelBase, oslodbmodels.TimestampMixin):
def save(self, session=None): def save(self, session=None):
if session is None: if session is None:
@ -30,27 +36,33 @@ class ScalpelsBase(oslodbmodels.ModelBase, oslodbmodels.TimestampMixin):
super(ScalpelsBase, self).save(session=session) super(ScalpelsBase, self).save(session=session)
class Task(BASE, ScalpelsBase): class Task(BASE, ScalpelsBase):
__tablename__ = "task" __tablename__ = "task"
id = Column(Integer, primary_key=True, autoincrement=True) id = Column(Integer, primary_key=True, autoincrement=True)
uuid = Column(String(36), default=lambda : str(uuid.uuid4()), nullable=False) uuid = Column(String(36), default=lambda: str(uuid.uuid4()),
nullable=False)
results = Column(JSONEncodedData, nullable=False) results = Column(JSONEncodedData, nullable=False)
pids = Column(JSONEncodedData, nullable=False) pids = Column(JSONEncodedData, nullable=False)
class Result(BASE, ScalpelsBase): class Result(BASE, ScalpelsBase):
__tablename__ = "result" __tablename__ = "result"
id = Column(Integer, primary_key=True, autoincrement=True) id = Column(Integer, primary_key=True, autoincrement=True)
uuid = Column(String(36), default=lambda : str(uuid.uuid4()), nullable=False) uuid = Column(String(36), default=lambda: str(uuid.uuid4()),
nullable=False)
data = Column(JSONEncodedData, nullable=False) data = Column(JSONEncodedData, nullable=False)
unit = Column(String(20), nullable=True) unit = Column(String(20), nullable=True)
name = Column(String(20), nullable=True) name = Column(String(20), nullable=True)
rtype = Column(String(20), nullable=False) rtype = Column(String(20), nullable=False)
class Setup(BASE, ScalpelsBase): class Setup(BASE, ScalpelsBase):
__tablename__ = "setup" __tablename__ = "setup"
id = Column(Integer, primary_key=True, autoincrement=True) id = Column(Integer, primary_key=True, autoincrement=True)
config = Column(JSONEncodedData, nullable=False) config = Column(JSONEncodedData, nullable=False)
class Tracer(BASE, ScalpelsBase): class Tracer(BASE, ScalpelsBase):
__tablename__ = "tracer" __tablename__ = "tracer"
id = Column(Integer, primary_key=True, autoincrement=True) id = Column(Integer, primary_key=True, autoincrement=True)

1
tests/unit/test_utils.py
View File

@ -4,6 +4,7 @@
from oslotest import base from oslotest import base
class TestCase(base.BaseTestCase): class TestCase(base.BaseTestCase):
def setUp(self): def setUp(self):
super(TestCase, self).setUp() super(TestCase, self).setUp()

8
tox.ini
View File

@ -20,8 +20,8 @@ distribute = false
basepython = python2.7 basepython = python2.7
[testenv:pep8] [testenv:pep8]
#commands = flake8 commands = flake8
commands = {toxinidir}/tools/fake_pep8.sh #commands = {toxinidir}/tools/fake_pep8.sh
distribute = false distribute = false
[testenv:py26] [testenv:py26]
@ -44,6 +44,6 @@ commands = make html
downloadcache = ~/cache/pip downloadcache = ~/cache/pip
[flake8] [flake8]
ignore = H703 ignore = H703,H102,E265,E262,H233
show-source = true show-source = true
exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,build,setup.py exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,tools,build,setup.py,tests/ci/*,scripts/*