x/synergy-scheduler-manager
Code Issues Proposed changes

Added queue usage to project

Browse Source 
This commit adds some info about the queue usage to
each project

Change-Id: If14ca606148b0dbd2b22bfa470196b2db7c47d07
Sem-Ver: bugfix
This commit is contained in:
Lisa Zangrando 2017-08-03 14:48:41 +02:00
parent 5f28805ca7
commit f8ca81d631
12 changed files with 543 additions and 738 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
setup.cfg
View File

@ -30,10 +30,8 @@ synergy.managers =
ProjectManager = synergy_scheduler_manager.project_manager:ProjectManager ProjectManager = synergy_scheduler_manager.project_manager:ProjectManager
synergy.commands = synergy.commands =
user = synergy_scheduler_manager.client.command:UserCommand
project = synergy_scheduler_manager.client.command:ProjectCommand project = synergy_scheduler_manager.client.command:ProjectCommand
quota = synergy_scheduler_manager.client.command:QuotaCommand
queue = synergy_scheduler_manager.client.command:QueueCommand
usage = synergy_scheduler_manager.client.command:UsageCommand
[build_sphinx] [build_sphinx]
source-dir = doc/source source-dir = doc/source

319
synergy_scheduler_manager/client/command.py
View File

@ -1,8 +1,6 @@
from synergy.client.command import ExecuteCommand from synergy.client.command import ExecuteCommand
from synergy.client.tabulate import tabulate from synergy.client.tabulate import tabulate
from synergy_scheduler_manager.common.project import Project from synergy_scheduler_manager.common.project import Project
from synergy_scheduler_manager.common.quota import SharedQuota
from synergy_scheduler_manager.common.user import User
__author__ = "Lisa Zangrando" __author__ = "Lisa Zangrando"
@ -202,272 +200,121 @@ class ProjectCommand(ExecuteCommand):
data["effective_memory"] * 100) data["effective_memory"] * 100)
row.append(usage) row.append(usage)
table.append(row)
print(tabulate(table, headers, tablefmt="fancy_grid")) if attribute == "queue":
data = project.getData()
q_usage = data.get("queue_usage", 0)
q_size = data.get("queue_size", 0)
if q_size:
class QueueCommand(ExecuteCommand): usage = float(q_usage) / float(q_size) * 100
row.append("{:.2f}%".format(usage))
def __init__(self): else:
super(QueueCommand, self).__init__("QueueCommand") row.append("0%")
def configureParser(self, subparser):
queue_parser = subparser.add_parser('queue')
queue_subparsers = queue_parser.add_subparsers(dest="command")
queue_subparsers.add_parser("show", add_help=True,
help="shows the queue info")
def execute(self, synergy_url, args):
if args.command == "show":
command = "GET_QUEUE"
cmd_args = {"name": "DYNAMIC"}
queue = super(QueueCommand, self).execute(synergy_url,
"QueueManager",
command,
args=cmd_args)
table = []
headers = ["name", "size", "is open"]
row = []
row.append(queue.getName())
row.append(queue.getSize())
row.append(str(queue.isOpen()).lower())
table.append(row)
print(tabulate(table, headers, tablefmt="fancy_grid"))
class QuotaCommand(ExecuteCommand):
def __init__(self):
super(QuotaCommand, self).__init__("QuotaCommand")
def configureParser(self, subparser):
quota_parser = subparser.add_parser('quota')
quota_subparsers = quota_parser.add_subparsers(dest="command")
show_parser = quota_subparsers.add_parser("show", add_help=True,
help="shows the quota info")
group = show_parser.add_mutually_exclusive_group()
group.add_argument("-i", "--project_id", metavar="<id>")
group.add_argument("-n", "--project_name", metavar="<name>")
group.add_argument("-a", "--all_projects", action="store_true")
group.add_argument("-s", "--shared", action="store_true")
def execute(self, synergy_url, args):
if args.command == "show":
command = "show"
cmd_args = {"shared": args.shared,
"project_id": args.project_id,
"project_name": args.project_name,
"all_projects": args.all_projects}
result = super(QuotaCommand, self).execute(synergy_url,
"QuotaManager",
command,
args=cmd_args)
if isinstance(result, SharedQuota):
self.printSharedQuota(result)
elif isinstance(result, Project):
self.printProjects([result])
else:
self.printProjects(result)
def printProjects(self, projects):
table = []
headers = ["project", "private quota", "shared quota", "share", "TTL"]
for project in projects:
share = project.getShare()
norm_share = share.getNormalizedValue()
quota = project.getQuota()
vcpus_size = quota.getSize("vcpus", private=False)
vcpus_usage = quota.getUsage("vcpus", private=False)
memory_size = quota.getSize("memory", private=False)
memory_usage = quota.getUsage("memory", private=False)
row = []
row.append(project.getName())
private = "vcpus: {:.2f} of {:.2f} | memory: {:.2f} of "\
"{:.2f}".format(quota.getUsage("vcpus"),
quota.getSize("vcpus"),
quota.getUsage("memory"),
quota.getSize("memory"))
shared = "vcpus: {:.2f} of {:.2f} | memory: {:.2f} of {:.2f} | "\
"share: {:.2f}% | TTL: {:.2f}".format(vcpus_usage,
vcpus_size,
memory_usage,
memory_size,
norm_share * 100,
project.getTTL())
row.append(private)
row.append(shared)
table.append(row) table.append(row)
print(tabulate(table, headers, tablefmt="fancy_grid")) print(tabulate(table, headers, tablefmt="fancy_grid"))
def printSharedQuota(self, quota):
table = []
headers = ["resource", "used", "size"]
resources = ["vcpus", "memory", "instances"]
for resource in resources: class UserCommand(ExecuteCommand):
row = [resource, quota.getUsage(resource), quota.getSize(resource)]
table.append(row)
print(tabulate(table, headers, tablefmt="fancy_grid"))
class UsageCommand(ExecuteCommand):
def __init__(self): def __init__(self):
super(UsageCommand, self).__init__("UsageCommand") super(UserCommand, self).__init__("UserCommand")
def configureParser(self, subparser): def configureParser(self, subparser):
usage_parser = subparser.add_parser('usage') usr_parser = subparser.add_parser('user')
usage_subparsers = usage_parser.add_subparsers(dest="command") usr_subparsers = usr_parser.add_subparsers(dest="command")
show_parser = usage_subparsers.add_parser("show", add_help=True,
help="shows the usage info")
subparsers = show_parser.add_subparsers() show_parser = usr_subparsers.add_parser("show", add_help=True,
parser_a = subparsers.add_parser('project', help='project help') help="shows the user info")
group = show_parser.add_mutually_exclusive_group(required=True)
group.add_argument("-i", "--id", metavar="<id>")
group.add_argument("-n", "--name", metavar="<name>")
group.add_argument("-a", "--all", action="store_true")
group2 = show_parser.add_mutually_exclusive_group(required=True)
group2.add_argument("-j", "--prj_id", metavar="<id>")
group2.add_argument("-m", "--prj_name", metavar="<name>")
group = parser_a.add_mutually_exclusive_group() show_parser.add_argument("-s", "--share", action="store_true")
group.add_argument("-d", "--project_id", metavar="<id>") show_parser.add_argument("-u", "--usage", action="store_true")
group.add_argument("-m", "--project_name", metavar="<name>") show_parser.add_argument("-p", "--priority", action="store_true")
group.add_argument("-a", "--all_projects", action="store_true") show_parser.add_argument("-l", "--long", action="store_true")
parser_b = subparsers.add_parser('user', help='user help')
group = parser_b.add_mutually_exclusive_group(required=True)
group.add_argument("-d", "--project_id", metavar="<id>")
group.add_argument("-m", "--project_name", metavar="<name>")
group = parser_b.add_mutually_exclusive_group(required=True)
group.add_argument("-i", "--user_id", metavar="<id>")
group.add_argument("-n", "--user_name", metavar="<name>")
group.add_argument("-a", "--all_users", action="store_true")
def execute(self, synergy_url, args): def execute(self, synergy_url, args):
if args.command == "show": usr_id = getattr(args, 'id', None)
command = "show" usr_name = getattr(args, 'name', None)
user_id = None prj_id = getattr(args, 'prj_id', None)
if hasattr(args, "user_id"): prj_name = getattr(args, 'prj_name', None)
user_id = args.user_id command = getattr(args, 'command', None)
headers = ["name"]
user_name = None if command == "show":
if hasattr(args, "user_name"): if args.long:
user_name = args.user_name headers.insert(0, "id")
if args.share:
headers.append("share")
if args.usage:
headers.append("usage")
if args.priority:
headers.append("priority")
all_users = False cmd_args = {"id": prj_id, "name": prj_name}
if hasattr(args, "all_users"): result = super(UserCommand, self).execute(synergy_url,
all_users = args.all_users "ProjectManager",
"GET_PROJECT",
args=cmd_args)
project_id = None if not result:
if hasattr(args, "project_id"): print("project not found!")
project_id = args.project_id
project_name = None
if hasattr(args, "project_name"):
project_name = args.project_name
all_projects = False
if hasattr(args, "all_projects"):
all_projects = args.all_projects
cmd_args = {"user_id": user_id,
"user_name": user_name,
"all_users": all_users,
"project_id": project_id,
"project_name": project_name,
"all_projects": all_projects}
result = super(UsageCommand, self).execute(synergy_url,
"SchedulerManager",
command,
args=cmd_args)
if isinstance(result, Project):
self.printProjects([result])
elif isinstance(result, User):
self.printUsers([result])
elif isinstance(result, list):
if all(isinstance(n, Project) for n in result):
self.printProjects(result)
else:
self.printUsers(result)
def printProjects(self, projects):
if not projects:
return return
data = projects[0].getData() self.printProject(result, headers, usr_id, usr_name)
date_format = "{:%d %b %Y %H:%M:%S}"
from_date = date_format.format(data["time_window_from_date"])
to_date = date_format.format(data["time_window_to_date"])
headers = ["project", def printProject(self, project, headers, usr_id, usr_name):
"shared quota (%s - %s)" % (from_date, to_date), if not project:
"share"]
table = []
for project in projects:
data = project.getData()
share = project.getShare()
row = []
row.append(project.getName())
shared = "vcpus: {:.2f}% | memory: {:.2f}%".format(
data["effective_vcpus"] * 100, data["effective_memory"] * 100)
row.append(shared)
row.append("{:.2f}%".format(share.getNormalizedValue() * 100))
table.append(row)
print(tabulate(table, headers, tablefmt="fancy_grid"))
def printUsers(self, users):
if not users:
return return
table = [] table = []
users = None
date_format = "{:%d %b %Y %H:%M:%S}" if usr_id or usr_name:
data = users[0].getData() user = project.getUser(id=usr_id, name=usr_name)
if not user:
print("user not found!")
return
from_date = date_format.format(data["time_window_from_date"]) users = [user]
to_date = date_format.format(data["time_window_to_date"]) else:
users = project.getUsers()
headers = ["user",
"shared quota (%s - %s)" % (from_date, to_date),
"share",
"priority"]
for user in users: for user in users:
share = user.getShare()
data = user.getData()
priority = user.getPriority()
row = [] row = []
row.append(user.getName())
row.append("vcpus: {:.2f}% | memory: {:.2f}%".format( for attribute in headers:
data["actual_rel_vcpus"] * 100, if attribute == "id":
data["actual_rel_memory"] * 100)) row.append(user.getId())
row.append("{:.2f}%".format(share.getNormalizedValue() * 100)) if attribute == "name":
row.append("{:.2f}".format(priority.getValue())) row.append(user.getName())
if attribute == "share":
share = user.getShare()
share_value = share.getValue()
share_norm = share.getNormalizedValue()
row.append("{:.2f}% | {:.2f}%".format(share_value,
share_norm * 100))
if attribute == "priority":
row.append(user.getPriority().getValue())
if attribute == "usage":
data = user.getData()
usage = "vcpus: {:.2f}% | ram: {:.2f}%".format(
data["actual_vcpus"] * 100,
data["actual_memory"] * 100)
row.append(usage)
table.append(row) table.append(row)

15
synergy_scheduler_manager/common/project.py
View File

@ -40,6 +40,12 @@ class Project(SynergyObject):
def getQuota(self): def getQuota(self):
return self.get("quota") return self.get("quota")
def setQueue(self, queue):
self.set("queue", queue)
def getQueue(self):
return self.get("queue")
def getShare(self): def getShare(self):
return self.get("share") return self.get("share")
@ -75,3 +81,12 @@ class Project(SynergyObject):
def setEnabled(self, enabled=True): def setEnabled(self, enabled=True):
self.set("enabled", enabled) self.set("enabled", enabled)
def serialize(self):
queue = self.getQueue()
usage = queue.getUsage(self.getId())
self.getData()["queue_usage"] = usage
self.getData()["queue_size"] = queue.getSize()
return super(Project, self).serialize()

447
synergy_scheduler_manager/common/queue.py
View File

@ -1,10 +1,10 @@
import heapq
import json import json
import threading import Queue as queue
from datetime import datetime from datetime import datetime
from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.exc import SQLAlchemyError
from synergy.common.serializer import SynergyObject from synergy.common.serializer import SynergyObject
from synergy.exception import SynergyError
__author__ = "Lisa Zangrando" __author__ = "Lisa Zangrando"
@ -28,16 +28,15 @@ permissions and limitations under the License."""
class QueueItem(object): class QueueItem(object):
def __init__(self, id, user_id, prj_id, priority, def __init__(self):
retry_count, creation_time, last_update, data=None): self.id = -1
self.id = id self.priority = 0
self.user_id = user_id self.retry_count = 0
self.prj_id = prj_id self.user_id = None
self.priority = priority self.prj_id = None
self.retry_count = retry_count self.data = None
self.creation_time = creation_time self.creation_time = datetime.now()
self.last_update = last_update self.last_update = self.creation_time
self.data = data
def getId(self): def getId(self):
return self.id return self.id
@ -91,47 +90,32 @@ class QueueItem(object):
self.data = data self.data = data
class PriorityQueue(object):
def __init__(self):
self._heap = []
def __len__(self):
return len(self._heap)
def __iter__(self):
"""Get all elements ordered by asc. priority. """
return self
def put(self, priority, item):
heapq.heappush(self._heap, (-priority, item.getCreationTime(), item))
def get(self):
return heapq.heappop(self._heap)[2]
def size(self):
return len(self._heap)
def items(self):
return [heapq.heappop(self._heap)[2] for i in range(len(self._heap))]
def smallest(self, x):
result = heapq.nsmallest(x, self._heap, key=lambda s: -s[0])
return [item[2] for item in result]
def largest(self, x):
result = heapq.nlargest(x, self._heap, key=lambda s: -s[0])
return [item[2] for item in result]
class Queue(SynergyObject): class Queue(SynergyObject):
def __init__(self): def __init__(self, name="default", type="PRIORITY", db_engine=None):
super(Queue, self).__init__() super(Queue, self).__init__()
self.setName("N/A") if type == "FIFO":
self.queue = queue.Queue()
elif type == "LIFO":
self.queue = queue.LifoQueue()
elif type == "PRIORITY":
self.queue = queue.PriorityQueue()
else:
raise SynergyError("queue type %r not supported" % type)
self.set("type", type)
self.set("is_closed", False) self.set("is_closed", False)
self.set("size", 0) self.set("size", 0)
self.setName(name)
self.db_engine = db_engine
self._createTable()
self._buildFromDB()
def _setSize(self, value):
size = self.get("size")
self.set("size", size + value)
def isOpen(self): def isOpen(self):
return not self.get("is_closed") return not self.get("is_closed")
@ -142,69 +126,144 @@ class Queue(SynergyObject):
def setClosed(self, is_closed): def setClosed(self, is_closed):
self.set("is_closed", is_closed) self.set("is_closed", is_closed)
def isEmpty(self):
return self.queue.empty()
def close(self):
self.setClosed(True)
def enqueue(self, user, data, priority=0):
if self.isClosed():
raise SynergyError("the queue is closed!")
if not user:
raise SynergyError("user not specified!")
if not data:
raise SynergyError("data not specified!")
item = QueueItem()
item.setUserId(user.getId())
item.setProjectId(user.getProjectId())
item.setPriority(priority)
item.setData(data)
self._insertItemDB(item)
if self.getType() == "PRIORITY":
self.queue.put((-priority, item.getCreationTime(), item))
else:
self.queue.put(item)
self._setSize(1)
def dequeue(self, block=True, timeout=None, delete=False):
if self.isClosed():
raise SynergyError("the queue is closed!")
if self.queue.empty() and not block:
return None
item = self.queue.get(block=block, timeout=timeout)
if not item:
return None
if self.getType() == "PRIORITY":
item = item[2]
self._getItemDataDB(item)
if delete:
self.delete(item)
return item
def restore(self, item):
if self.isClosed():
raise SynergyError("the queue is closed!")
if self.getType() == "PRIORITY":
self.queue.put((-item.getPriority(), item.getCreationTime(), item))
else:
self.queue.put(item)
self._updateItemDB(item)
def getType(self):
return self.get("type")
def getSize(self): def getSize(self):
return self.get("size") return self.get("size")
def setSize(self, size): def getUsage(self, prj_id):
self.set("size", size) result = 0
class QueueDB(Queue):
def __init__(self, name, db_engine, fairshare_manager=None):
super(QueueDB, self).__init__()
self.setName(name)
self.db_engine = db_engine
self.fairshare_manager = fairshare_manager
self.priority_updater = None
self.condition = threading.Condition()
self.pqueue = PriorityQueue()
self.createTable()
self.buildFromDB()
self.updatePriority()
def getSize(self):
connection = self.db_engine.connect() connection = self.db_engine.connect()
try: try:
QUERY = "select count(*) from `%s`" % self.getName() QUERY = "select count(*) from `%s` " % self.getName()
result = connection.execute(QUERY) QUERY += "where prj_id=%s"
row = result.fetchone() qresult = connection.execute(QUERY, [prj_id])
row = qresult.fetchone()
return row[0] result = row[0]
except SQLAlchemyError as ex: except SQLAlchemyError as ex:
raise Exception(ex.message) raise SynergyError(ex.message)
finally:
connection.close()
return result
def delete(self, item):
if self.isClosed():
raise SynergyError("the queue is closed!")
if not item or not self.db_engine:
return
connection = self.db_engine.connect()
trans = connection.begin()
try:
QUERY = "delete from `%s`" % self.getName()
QUERY += " where id=%s"
connection.execute(QUERY, [item.getId()])
trans.commit()
except SQLAlchemyError as ex:
trans.rollback()
raise SynergyError(ex.message)
finally: finally:
connection.close() connection.close()
def createTable(self): self._setSize(-1)
self.queue.task_done()
def _createTable(self):
if not self.db_engine:
return
TABLE = """CREATE TABLE IF NOT EXISTS `%s` (`id` BIGINT NOT NULL \ TABLE = """CREATE TABLE IF NOT EXISTS `%s` (`id` BIGINT NOT NULL \
AUTO_INCREMENT PRIMARY KEY, `priority` INT DEFAULT 0, user_id CHAR(40) \ AUTO_INCREMENT PRIMARY KEY, `priority` INT DEFAULT 0, user_id CHAR(40) \
NOT NULL, prj_id CHAR(40) NOT NULL, `retry_count` INT DEFAULT 0, \ NOT NULL, prj_id CHAR(40) NOT NULL, `retry_count` INT DEFAULT 0, \
`creation_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, `last_update` \ `creation_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, `last_update` \
TIMESTAMP NULL, `data` TEXT NOT NULL ) ENGINE=InnoDB""" % self.getName() TIMESTAMP NULL, `data` TEXT NOT NULL) ENGINE=InnoDB""" % self.getName()
connection = self.db_engine.connect() connection = self.db_engine.connect()
try: try:
connection.execute(TABLE) connection.execute(TABLE)
except SQLAlchemyError as ex: except SQLAlchemyError as ex:
raise Exception(ex.message) raise SynergyError(ex.message)
except Exception as ex: except Exception as ex:
raise Exception(ex.message) raise SynergyError(ex.message)
finally: finally:
connection.close() connection.close()
def close(self): def _buildFromDB(self):
if not self.isClosed(): if not self.db_engine:
self.setClosed(True) return
with self.condition:
self.condition.notifyAll()
def buildFromDB(self):
connection = self.db_engine.connect() connection = self.db_engine.connect()
try: try:
@ -213,182 +272,98 @@ TIMESTAMP NULL, `data` TEXT NOT NULL ) ENGINE=InnoDB""" % self.getName()
result = connection.execute(QUERY) result = connection.execute(QUERY)
for row in result: for row in result:
queue_item = QueueItem(row[0], row[1], row[2], item = QueueItem()
row[3], row[4], row[5], row[6]) item.setId(row[0])
item.setUserId(row[1])
item.setProjectId(row[2])
item.setPriority(row[3])
item.setRetryCount(row[4])
item.setCreationTime(row[5])
item.setLastUpdate(row[6])
self.pqueue.put(row[3], queue_item) self.restore(item)
self._setSize(1)
except SQLAlchemyError as ex: except SQLAlchemyError as ex:
raise Exception(ex.message) raise SynergyError(ex.message)
finally: finally:
connection.close() connection.close()
with self.condition: def _insertItemDB(self, item):
self.condition.notifyAll() if not item or not self.db_engine:
def insertItem(self, user_id, prj_id, priority, data):
with self.condition:
idRecord = -1
QUERY = "insert into `%s` (user_id, prj_id, priority, " \
"data) values" % self.getName()
QUERY += "(%s, %s, %s, %s)"
connection = self.db_engine.connect()
trans = connection.begin()
try:
result = connection.execute(QUERY,
[user_id, prj_id, priority,
json.dumps(data)])
idRecord = result.lastrowid
trans.commit()
except SQLAlchemyError as ex:
trans.rollback()
raise Exception(ex.message)
finally:
connection.close()
now = datetime.now()
queue_item = QueueItem(idRecord, user_id, prj_id,
priority, 0, now, now)
self.pqueue.put(priority, queue_item)
self.condition.notifyAll()
def reinsertItem(self, queue_item):
with self.condition:
self.pqueue.put(queue_item.getPriority(), queue_item)
self.condition.notifyAll()
def getItem(self, blocking=True):
item = None
queue_item = None
with self.condition:
while (queue_item is None and not self.isClosed()):
if len(self.pqueue):
queue_item = self.pqueue.get()
elif blocking:
self.condition.wait()
elif queue_item is None:
break
if (not self.isClosed() and queue_item is not None):
connection = self.db_engine.connect()
try:
QUERY = """select user_id, prj_id, priority, \
retry_count, creation_time, last_update, data from `%s`""" % self.getName()
QUERY += " where id=%s"
result = connection.execute(QUERY, [queue_item.getId()])
row = result.fetchone()
item = QueueItem(queue_item.getId(), row[0], row[1],
row[2], row[3], row[4], row[5],
json.loads(row[6]))
except SQLAlchemyError as ex:
raise Exception(ex.message)
finally:
connection.close()
self.condition.notifyAll()
return item
def deleteItem(self, queue_item):
if not queue_item:
return return
with self.condition: QUERY = "insert into `%s` (user_id, prj_id, priority, " \
connection = self.db_engine.connect() "data) values" % self.getName()
trans = connection.begin() QUERY += "(%s, %s, %s, %s)"
try: connection = self.db_engine.connect()
QUERY = "delete from `%s`" % self.getName() trans = connection.begin()
QUERY += " where id=%s"
connection.execute(QUERY, [queue_item.getId()]) try:
result = connection.execute(QUERY,
[item.getUserId(),
item.getProjectId(),
item.getPriority(),
json.dumps(item.getData())])
trans.commit() idRecord = result.lastrowid
except SQLAlchemyError as ex:
trans.rollback()
raise Exception(ex.message) trans.commit()
finally:
connection.close()
self.condition.notifyAll()
def updateItem(self, queue_item): item.setId(idRecord)
if not queue_item: except SQLAlchemyError as ex:
trans.SynergyError()
raise SynergyError(ex.message)
finally:
connection.close()
def _getItemDataDB(self, item):
if not item or not self.db_engine:
return return
with self.condition: data = None
connection = self.db_engine.connect() connection = self.db_engine.connect()
trans = connection.begin()
try: try:
queue_item.setLastUpdate(datetime.now()) QUERY = "select data from `%s`" % self.getName()
QUERY += " where id=%s"
QUERY = "update `%s`" % self.getName() result = connection.execute(QUERY, [item.getId()])
QUERY += " set priority=%s, retry_count=%s, " \
"last_update=%s where id=%s"
connection.execute(QUERY, [queue_item.getPriority(), row = result.fetchone()
queue_item.getRetryCount(),
queue_item.getLastUpdate(),
queue_item.getId()])
trans.commit() data = json.loads(row[0])
except SQLAlchemyError as ex: except SQLAlchemyError as ex:
trans.rollback() raise SynergyError(ex.message)
finally:
connection.close()
raise Exception(ex.message) item.setData(data)
finally: return data
connection.close()
self.pqueue.put(queue_item.getPriority(), queue_item) def _updateItemDB(self, item):
self.condition.notifyAll() if not item or not self.db_engine:
def updatePriority(self):
if self.fairshare_manager is None:
return return
queue_items = [] connection = self.db_engine.connect()
trans = connection.begin()
with self.condition: try:
while len(self.pqueue) > 0: item.setLastUpdate(datetime.now())
queue_item = self.pqueue.get()
priority = queue_item.getPriority()
try: QUERY = "update `%s`" % self.getName()
priority = self.fairshare_manager.calculatePriority( QUERY += " set priority=%s, retry_count=%s, " \
user_id=queue_item.getUserId(), "last_update=%s where id=%s"
prj_id=queue_item.getProjectId(),
timestamp=queue_item.getCreationTime(),
retry=queue_item.getRetryCount())
queue_item.setPriority(priority) connection.execute(QUERY, [item.getPriority(),
except Exception: item.getRetryCount(),
continue item.getLastUpdate(),
finally: item.getId()])
queue_items.append(queue_item)
if len(queue_items) > 0: trans.commit()
for queue_item in queue_items: except SQLAlchemyError as ex:
self.pqueue.put(queue_item.getPriority(), queue_item) trans.rollback()
del queue_items raise SynergyError(ex.message)
finally:
self.condition.notifyAll() connection.close()
def serialize(self):
queue = Queue()
queue.setName(self.getName())
queue.setSize(self.getSize())
queue.setClosed(self.isClosed())
return queue.serialize()

18
synergy_scheduler_manager/fairshare_manager.py
View File

@ -72,7 +72,7 @@ class FairShareManager(Manager):
def task(self): def task(self):
try: try:
self._calculateFairShare() self._calculateFairShare()
except SynergyError as ex: except Exception as ex:
LOG.error(ex) LOG.error(ex)
raise ex raise ex
@ -144,7 +144,7 @@ class FairShareManager(Manager):
total_prj_share = float(0) total_prj_share = float(0)
projects = self.project_manager.getProjects() projects = self.project_manager.getProjects()
for project in projects.values(): for project in projects:
prj_share = project.getShare() prj_share = project.getShare()
# check the share for each user and update the usage_record # check the share for each user and update the usage_record
@ -168,7 +168,7 @@ class FairShareManager(Manager):
total_prj_share += prj_share.getValue() total_prj_share += prj_share.getValue()
for project in projects.values(): for project in projects:
prj_share = project.getShare() prj_share = project.getShare()
prj_share.setSiblingValue(total_prj_share) prj_share.setSiblingValue(total_prj_share)
prj_share.setNormalizedValue( prj_share.setNormalizedValue(
@ -196,7 +196,7 @@ class FairShareManager(Manager):
time_window_from_date = to_date time_window_from_date = to_date
time_window_to_date = to_date time_window_to_date = to_date
for prj_id, project in projects.items(): for project in projects:
prj_data = project.getData() prj_data = project.getData()
prj_data["actual_vcpus"] = float(0) prj_data["actual_vcpus"] = float(0)
prj_data["actual_memory"] = float(0) prj_data["actual_memory"] = float(0)
@ -223,9 +223,9 @@ class FairShareManager(Manager):
from_date = to_date - timedelta(days=(self.period_length)) from_date = to_date - timedelta(days=(self.period_length))
time_window_from_date = from_date time_window_from_date = from_date
for prj_id, project in projects.items(): for project in projects:
usages = self.nova_manager.getProjectUsage( usages = self.nova_manager.getProjectUsage(
prj_id, from_date, to_date) project.getId(), from_date, to_date)
for user_id, usage_rec in usages.items(): for user_id, usage_rec in usages.items():
decay_vcpus = decay * usage_rec["vcpus"] decay_vcpus = decay * usage_rec["vcpus"]
@ -243,7 +243,7 @@ class FairShareManager(Manager):
to_date = from_date to_date = from_date
for prj_id, project in projects.items(): for project in projects:
prj_data = project.getData() prj_data = project.getData()
prj_data["time_window_to_date"] = time_window_to_date prj_data["time_window_to_date"] = time_window_to_date
@ -262,7 +262,7 @@ class FairShareManager(Manager):
prj_data["actual_memory"] += usr_data["actual_memory"] prj_data["actual_memory"] += usr_data["actual_memory"]
prj_data["actual_vcpus"] += usr_data["actual_vcpus"] prj_data["actual_vcpus"] += usr_data["actual_vcpus"]
for project in projects.values(): for project in projects:
prj_data = project.getData() prj_data = project.getData()
prj_data["effective_memory"] = prj_data["actual_memory"] prj_data["effective_memory"] = prj_data["actual_memory"]
prj_data["effective_vcpus"] = prj_data["actual_vcpus"] prj_data["effective_vcpus"] = prj_data["actual_vcpus"]
@ -294,7 +294,7 @@ class FairShareManager(Manager):
share / sibling_share)) share / sibling_share))
usr_data["effective_memory"] = effective_memory usr_data["effective_memory"] = effective_memory
usr_data["effective_cores"] = effective_vcpus usr_data["effective_vcpus"] = effective_vcpus
f_memory = 2 ** (-effective_memory / norm_share) f_memory = 2 ** (-effective_memory / norm_share)
usr_priority.setFairShare("memory", f_memory) usr_priority.setFairShare("memory", f_memory)

98
synergy_scheduler_manager/nova_manager.py
View File

@ -18,6 +18,7 @@ from oslo_config import cfg
from sqlalchemy import create_engine from sqlalchemy import create_engine
from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.exc import SQLAlchemyError
from synergy.common.manager import Manager from synergy.common.manager import Manager
from synergy.exception import SynergyError
__author__ = "Lisa Zangrando" __author__ = "Lisa Zangrando"
__email__ = "lisa.zangrando[AT]pd.infn.it" __email__ = "lisa.zangrando[AT]pd.infn.it"
@ -321,10 +322,10 @@ class NovaManager(Manager):
self.timeout = CONF.NovaManager.timeout self.timeout = CONF.NovaManager.timeout
if self.getManager("KeystoneManager") is None: if self.getManager("KeystoneManager") is None:
raise Exception("KeystoneManager not found!") raise SynergyError("KeystoneManager not found!")
if self.getManager("SchedulerManager") is None: if self.getManager("SchedulerManager") is None:
raise Exception("SchedulerManager not found!") raise SynergyError("SchedulerManager not found!")
self.keystone_manager = self.getManager("KeystoneManager") self.keystone_manager = self.getManager("KeystoneManager")
@ -399,7 +400,7 @@ class NovaManager(Manager):
raise ex raise ex
def execute(self, command, *args, **kargs): def execute(self, command, *args, **kargs):
raise Exception("command %r not supported!" % command) raise SynergyError("command %r not supported!" % command)
def task(self): def task(self):
pass pass
@ -414,8 +415,8 @@ class NovaManager(Manager):
return result return result
if fallback is True: if fallback is True:
raise Exception("No attribute %r found in [NovaManager] " raise SynergyError("No attribute %r found in [NovaManager] "
"section of synergy.conf" % name) "section of synergy.conf" % name)
else: else:
return None return None
@ -426,8 +427,8 @@ class NovaManager(Manager):
secret = CONF.NovaManager.metadata_proxy_shared_secret secret = CONF.NovaManager.metadata_proxy_shared_secret
if not secret: if not secret:
return Exception("'metadata_proxy_shared_secret' " raise SynergyError("'metadata_proxy_shared_secret' "
"attribute not defined in synergy.conf") "attribute not defined in synergy.conf")
digest = hmac.new(secret, server.getId(), hashlib.sha256).hexdigest() digest = hmac.new(secret, server.getId(), hashlib.sha256).hexdigest()
@ -436,12 +437,12 @@ class NovaManager(Manager):
service = token.getService("nova") service = token.getService("nova")
if not service: if not service:
raise Exception("nova service not found!") raise SynergyError("nova service not found!")
endpoint = service.getEndpoint("public") endpoint = service.getEndpoint("public")
if not endpoint: if not endpoint:
raise Exception("nova endpoint not found!") raise SynergyError("nova endpoint not found!")
url = endpoint.getURL() url = endpoint.getURL()
url = url[:url.rfind(":") + 1] + "8775/openstack/2015-10-15/user_data" url = url[:url.rfind(":") + 1] + "8775/openstack/2015-10-15/user_data"
@ -463,9 +464,9 @@ class NovaManager(Manager):
return None return None
elif request.status_code == 403: elif request.status_code == 403:
if "Invalid proxy request signature" in request._content: if "Invalid proxy request signature" in request._content:
raise Exception("cannot retrieve the 'userdata' value: " raise SynergyError("cannot retrieve the 'userdata' value: "
"check the 'metadata_proxy_shared_secret'" "check the 'metadata_proxy_shared_"
" attribute value") "secret' attribute value")
else: else:
request.raise_for_status() request.raise_for_status()
else: else:
@ -483,8 +484,8 @@ class NovaManager(Manager):
response_data = self.getResource(url, method="GET") response_data = self.getResource(url, method="GET")
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
response = ex.response.json() response = ex.response.json()
raise Exception("error on retrieving the flavors list: %s" raise SynergyError("error on retrieving the flavors list: %s"
% response) % response)
flavors = [] flavors = []
@ -505,8 +506,8 @@ class NovaManager(Manager):
try: try:
response_data = self.getResource("flavors/" + id, "GET") response_data = self.getResource("flavors/" + id, "GET")
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
raise Exception("error on retrieving the flavor info (id=%r)" raise SynergyError("error on retrieving the flavor info (id=%r)"
": %s" % (id, ex.response.json())) ": %s" % (id, ex.response.json()))
flavor = None flavor = None
@ -533,8 +534,8 @@ class NovaManager(Manager):
response_data = self.getResource(url, "GET", params) response_data = self.getResource(url, "GET", params)
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
response = ex.response.json() response = ex.response.json()
raise Exception("error on retrieving the servers list" raise SynergyError("error on retrieving the servers list"
": %s" % (id, response)) ": %s" % (id, response))
servers = [] servers = []
@ -571,8 +572,8 @@ class NovaManager(Manager):
try: try:
response_data = self.getResource("servers/" + id, "GET") response_data = self.getResource("servers/" + id, "GET")
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
raise Exception("error on retrieving the server info (id=%r)" raise SynergyError("error on retrieving the server info (id=%r)"
": %s" % (id, ex.response.json())) ": %s" % (id, ex.response.json()))
server = None server = None
@ -619,8 +620,8 @@ class NovaManager(Manager):
try: try:
response_data = self.getResource(url, "DELETE") response_data = self.getResource(url, "DELETE")
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
raise Exception("error on deleting the server (id=%r)" raise SynergyError("error on deleting the server (id=%r)"
": %s" % (id, ex.response.json())) ": %s" % (id, ex.response.json()))
if response_data: if response_data:
response_data = response_data["server"] response_data = response_data["server"]
@ -638,8 +639,8 @@ class NovaManager(Manager):
try: try:
response_data = self.getResource(url, "POST", data) response_data = self.getResource(url, "POST", data)
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
raise Exception("error on starting the server %s" raise SynergyError("error on starting the server %s"
": %s" % (id, ex.response.json())) ": %s" % (id, ex.response.json()))
if response_data: if response_data:
response_data = response_data["server"] response_data = response_data["server"]
@ -657,8 +658,8 @@ class NovaManager(Manager):
try: try:
response_data = self.getResource(url, "POST", data) response_data = self.getResource(url, "POST", data)
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
raise Exception("error on stopping the server info (id=%r)" raise SynergyError("error on stopping the server info (id=%r)"
": %s" % (id, ex.response.json())) ": %s" % (id, ex.response.json()))
if response_data: if response_data:
response_data = response_data["server"] response_data = response_data["server"]
@ -674,8 +675,8 @@ class NovaManager(Manager):
response_data = self.getResource(url, "GET", data) response_data = self.getResource(url, "GET", data)
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
response = ex.response.json() response = ex.response.json()
raise Exception("error on retrieving the hypervisors list: %s" raise SynergyError("error on retrieving the hypervisors list: %s"
% response["badRequest"]["message"]) % response["badRequest"]["message"])
if response_data: if response_data:
response_data = response_data["hosts"] response_data = response_data["hosts"]
@ -690,8 +691,9 @@ class NovaManager(Manager):
response_data = self.getResource(url, "GET", data) response_data = self.getResource(url, "GET", data)
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
response = ex.response.json() response = ex.response.json()
raise Exception("error on retrieving the hypervisor info (id=%r)" raise SynergyError("error on retrieving the hypervisor info (id=%r"
": %s" % (id, response["badRequest"]["message"])) "): %s" % (id,
response["badRequest"]["message"]))
if response_data: if response_data:
response_data = response_data["host"] response_data = response_data["host"]
@ -707,8 +709,8 @@ class NovaManager(Manager):
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
LOG.info(ex) LOG.info(ex)
response = ex.response.json() response = ex.response.json()
raise Exception("error on retrieving the hypervisors list: %s" raise SynergyError("error on retrieving the hypervisors list: %s"
% response["badRequest"]["message"]) % response["badRequest"]["message"])
hypervisors = [] hypervisors = []
@ -743,8 +745,8 @@ class NovaManager(Manager):
try: try:
response_data = self.getResource(url, "GET", data) response_data = self.getResource(url, "GET", data)
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
raise Exception("error on retrieving the hypervisor info (id=%r)" raise SynergyError("error on retrieving the hypervisor info (id=%r"
": %s" % (id, ex.response.json())) "): %s" % (id, ex.response.json()))
hypervisor = None hypervisor = None
@ -775,8 +777,8 @@ class NovaManager(Manager):
url = "os-quota-sets/defaults" url = "os-quota-sets/defaults"
response_data = self.getResource(url, "GET") response_data = self.getResource(url, "GET")
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
raise Exception("error on retrieving the quota defaults" raise SynergyError("error on retrieving the quota defaults"
": %s" % ex.response.json()) ": %s" % ex.response.json())
elif id is not None: elif id is not None:
if is_class: if is_class:
url = "os-quota-class-sets/%s" % id url = "os-quota-class-sets/%s" % id
@ -791,10 +793,10 @@ class NovaManager(Manager):
else: else:
quota_data = response_data["quota_set"] quota_data = response_data["quota_set"]
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
raise Exception("error on retrieving the quota info (id=%r)" raise SynergyError("error on retrieving the quota info (id=%r)"
": %s" % (id, ex.response.json())) ": %s" % (id, ex.response.json()))
else: else:
raise Exception("wrong arguments") raise SynergyError("wrong arguments")
quota = None quota = None
@ -825,8 +827,8 @@ class NovaManager(Manager):
try: try:
self.getResource(url, "PUT", qs) self.getResource(url, "PUT", qs)
except requests.exceptions.HTTPError as ex: except requests.exceptions.HTTPError as ex:
raise Exception("error on updating the quota info (id=%r)" raise SynergyError("error on updating the quota info (id=%r)"
": %s" % (id, ex.response.json())) ": %s" % (id, ex.response.json()))
def getResource(self, resource, method, data=None): def getResource(self, resource, method, data=None):
self.keystone_manager.authenticate() self.keystone_manager.authenticate()
@ -834,12 +836,12 @@ class NovaManager(Manager):
service = token.getService("nova") service = token.getService("nova")
if not service: if not service:
raise Exception("nova service not found!") raise SynergyError("nova service not found!")
endpoint = service.getEndpoint("public") endpoint = service.getEndpoint("public")
if not endpoint: if not endpoint:
raise Exception("nova endpoint not found!") raise SynergyError("nova endpoint not found!")
url = endpoint.getURL() + "/" + resource url = endpoint.getURL() + "/" + resource
@ -883,7 +885,7 @@ class NovaManager(Manager):
verify=self.ssl_ca_file, verify=self.ssl_ca_file,
cert=self.ssl_cert_file) cert=self.ssl_cert_file)
else: else:
raise Exception("wrong HTTP method: %s" % method) raise SynergyError("wrong HTTP method: %s" % method)
if request.status_code != requests.codes.ok: if request.status_code != requests.codes.ok:
request.raise_for_status() request.raise_for_status()
@ -943,7 +945,7 @@ a.launched_at<='%(to_date)s' and (a.terminated_at>='%(from_date)s' or \
"vcpus": float(row[2])} "vcpus": float(row[2])}
except SQLAlchemyError as ex: except SQLAlchemyError as ex:
raise Exception(ex.message) raise SynergyError(ex.message)
finally: finally:
connection.close() connection.close()
@ -990,7 +992,7 @@ where instance_uuid='%(id)s' and deleted_at is NULL""" % {"id": server.getId()}
servers.append(server) servers.append(server)
except SQLAlchemyError as ex: except SQLAlchemyError as ex:
raise Exception(ex.message) raise SynergyError(ex.message)
finally: finally:
connection.close() connection.close()
@ -1046,7 +1048,7 @@ where instance_uuid='%(id)s' and deleted_at is NULL""" % {"id": server.getId()}
servers.append(server) servers.append(server)
except SQLAlchemyError as ex: except SQLAlchemyError as ex:
raise Exception(ex.message) raise SynergyError(ex.message)
finally: finally:
connection.close() connection.close()
@ -1123,7 +1125,7 @@ nova.block_device_mapping where instance_uuid='%(server_id)s'
blockDeviceMapList.append(blockDeviceMap) blockDeviceMapList.append(blockDeviceMap)
except SQLAlchemyError as ex: except SQLAlchemyError as ex:
raise Exception(ex.message) raise SynergyError(ex.message)
finally: finally:
connection.close() connection.close()

9
synergy_scheduler_manager/project_manager.py
View File

@ -70,8 +70,11 @@ class ProjectManager(Manager):
def task(self): def task(self):
if not self.configured: if not self.configured:
self.buildFromDB() try:
self.configured = True self.buildFromDB()
self.configured = True
except Exception as ex:
LOG.error(ex)
def destroy(self): def destroy(self):
pass pass
@ -296,7 +299,7 @@ class ProjectManager(Manager):
return project return project
def getProjects(self): def getProjects(self):
return self.projects return self.projects.values()
def createTable(self): def createTable(self):
TABLE = """CREATE TABLE IF NOT EXISTS project (`id` VARCHAR(64) \ TABLE = """CREATE TABLE IF NOT EXISTS project (`id` VARCHAR(64) \

60
synergy_scheduler_manager/queue_manager.py
View File

@ -1,9 +1,10 @@
import logging import logging
from common.queue import QueueDB from common.queue import Queue
from oslo_config import cfg from oslo_config import cfg
from sqlalchemy import create_engine from sqlalchemy import create_engine
from synergy.common.manager import Manager from synergy.common.manager import Manager
from synergy.exception import SynergyError
__author__ = "Lisa Zangrando" __author__ = "Lisa Zangrando"
@ -39,17 +40,9 @@ class QueueManager(Manager):
cfg.IntOpt('db_pool_recycle', default=30, required=False), cfg.IntOpt('db_pool_recycle', default=30, required=False),
cfg.IntOpt('db_max_overflow', default=5, required=False) cfg.IntOpt('db_max_overflow', default=5, required=False)
] ]
self.queue_list = {} self.queues = {}
def setup(self): def setup(self):
if self.getManager("FairShareManager") is None:
raise Exception("FairShareManager not found!")
self.fairshare_manager = self.getManager("FairShareManager")
if self.fairshare_manager is None:
raise Exception("FairShareManager not found!")
db_connection = CONF.QueueManager.db_connection db_connection = CONF.QueueManager.db_connection
pool_size = CONF.QueueManager.db_pool_size pool_size = CONF.QueueManager.db_pool_size
pool_recycle = CONF.QueueManager.db_pool_recycle pool_recycle = CONF.QueueManager.db_pool_recycle
@ -61,49 +54,42 @@ class QueueManager(Manager):
pool_recycle=pool_recycle, pool_recycle=pool_recycle,
max_overflow=max_overflow) max_overflow=max_overflow)
except Exception as ex: except Exception as ex:
LOG.error(ex) raise SynergyError(ex.message)
raise ex
def execute(self, command, *args, **kargs): def execute(self, command, *args, **kargs):
if command == "CREATE_QUEUE": if command == "GET_QUEUE":
return self.createQueue(*args, **kargs) return self.getQueue(kargs.get("name", None))
elif command == "DELETE_QUEUE":
return self.deleteQueue(*args, **kargs)
elif command == "GET_QUEUE":
queue = self.getQueue(kargs.get("name", None))
return queue
else: else:
raise Exception("command=%r not supported!" % command) raise SynergyError("command %r not supported!" % command)
def task(self): def task(self):
try: try:
for queue in self.queue_list.values(): for queue in self.queues.values():
queue.updatePriority() queue.updatePriority()
except Exception as ex: except Exception as ex:
LOG.error("Exception has occured", exc_info=1)
LOG.error(ex) LOG.error(ex)
def destroy(self): def destroy(self):
for queue in self.queue_list.values(): for queue in self.queues.values():
queue.close() queue.close()
def createQueue(self, name): def createQueue(self, name, type):
if name not in self.queue_list: if name in self.queues:
queue = QueueDB(name, self.db_engine, self.fairshare_manager) raise SynergyError("the queue %r already exists!" % name)
self.queue_list[name] = queue
return queue queue = Queue(name, type, db_engine=self.db_engine)
else: self.queues[name] = queue
raise Exception("the queue %r already exists!" % name)
return queue
def deleteQueue(self, name): def deleteQueue(self, name):
if name not in self.queue_list: if name not in self.queues:
raise Exception("queue %r not found!" % name) raise SynergyError("queue %r not found!" % name)
del self.queue_list[name] del self.queues[name]
def getQueue(self, name): def getQueue(self, name):
if name not in self.queue_list: if name not in self.queues:
raise Exception("queue %r not found!" % name) raise SynergyError("queue %r not found!" % name)
return self.queue_list[name] return self.queues[name]

27
synergy_scheduler_manager/quota_manager.py
View File

@ -77,7 +77,7 @@ class QuotaManager(Manager):
try: try:
self.updateSharedQuota() self.updateSharedQuota()
self.deleteExpiredServers() self.deleteExpiredServers()
except SynergyError as ex: except Exception as ex:
LOG.error(ex) LOG.error(ex)
def doOnEvent(self, event_type, *args, **kwargs): def doOnEvent(self, event_type, *args, **kwargs):
@ -109,13 +109,15 @@ class QuotaManager(Manager):
quota.setSize("vcpus", class_quota.getSize("vcpus")) quota.setSize("vcpus", class_quota.getSize("vcpus"))
quota.setSize("memory", class_quota.getSize("memory")) quota.setSize("memory", class_quota.getSize("memory"))
quota.setSize("instances", class_quota.getSize("instances")) quota.setSize("instances", class_quota.getSize("instances"))
quota.setSize( quota.setSize("vcpus",
"vcpus", SharedQuota.getSize("vcpus"), private=False) SharedQuota.getSize("vcpus"),
quota.setSize( private=False)
"memory", SharedQuota.getSize("memory"), private=False) quota.setSize("memory",
quota.setSize( SharedQuota.getSize("memory"),
"instances", SharedQuota.getSize("instances"), private=False)
private=False) quota.setSize("instances",
SharedQuota.getSize("instances"),
private=False)
servers = self.nova_manager.getProjectServers(project.getId()) servers = self.nova_manager.getProjectServers(project.getId())
@ -155,8 +157,8 @@ class QuotaManager(Manager):
quota = self.nova_manager.getQuota(project.getId()) quota = self.nova_manager.getQuota(project.getId())
if quota.getSize("vcpus") <= -1 and \ if quota.getSize("vcpus") <= -1 and \
quota.getSize("memory") <= -1 and \ quota.getSize("memory") <= -1 and \
quota.getSize("instances") <= -1: quota.getSize("instances") <= -1:
qc = self.nova_manager.getQuota(project.getId(), is_class=True) qc = self.nova_manager.getQuota(project.getId(), is_class=True)
self.nova_manager.updateQuota(qc) self.nova_manager.updateQuota(qc)
@ -176,7 +178,8 @@ class QuotaManager(Manager):
raise ex raise ex
def deleteExpiredServers(self): def deleteExpiredServers(self):
for prj_id, project in self.project_manager.getProjects().items(): for project in self.project_manager.getProjects():
prj_id = project.getId()
TTL = project.getTTL() TTL = project.getTTL()
quota = project.getQuota() quota = project.getQuota()
@ -322,7 +325,7 @@ class QuotaManager(Manager):
SharedQuota.setSize("vcpus", 0) SharedQuota.setSize("vcpus", 0)
SharedQuota.setSize("memory", 0) SharedQuota.setSize("memory", 0)
for project in self.project_manager.getProjects().values(): for project in self.project_manager.getProjects():
quota = project.getQuota() quota = project.getQuota()
quota.setSize("vcpus", shared_vcpus, private=False) quota.setSize("vcpus", shared_vcpus, private=False)
quota.setSize("memory", shared_memory, private=False) quota.setSize("memory", shared_memory, private=False)

49
synergy_scheduler_manager/scheduler_manager.py
View File

@ -68,20 +68,20 @@ class Worker(Thread):
last_release_time = SharedQuota.getLastReleaseTime() last_release_time = SharedQuota.getLastReleaseTime()
while queue_items: while queue_items:
self.queue.reinsertItem(queue_items.pop(0)) self.queue.restore(queue_items.pop(0))
if len(queue_items) >= self.backfill_depth: if len(queue_items) >= self.backfill_depth:
SharedQuota.wait() SharedQuota.wait()
continue continue
queue_item = self.queue.getItem(blocking=False) queue_item = self.queue.dequeue(block=False)
if queue_item is None: if queue_item is None:
if self.queue.getSize(): if self.queue.getSize():
SharedQuota.wait() SharedQuota.wait()
continue continue
else: else:
queue_item = self.queue.getItem(blocking=True) queue_item = self.queue.dequeue(block=True)
if queue_item is None: if queue_item is None:
continue continue
@ -100,7 +100,7 @@ class Worker(Thread):
if s.getState() != "building": if s.getState() != "building":
# or server["OS-EXT-STS:task_state"] != "scheduling": # or server["OS-EXT-STS:task_state"] != "scheduling":
self.queue.deleteItem(queue_item) self.queue.delete(queue_item)
continue continue
except SynergyError as ex: except SynergyError as ex:
LOG.warn("the server %s is not anymore available!" LOG.warn("the server %s is not anymore available!"
@ -147,7 +147,7 @@ class Worker(Thread):
% (server.getId(), ex)) % (server.getId(), ex))
if found: if found:
self.queue.deleteItem(queue_item) self.queue.delete(queue_item)
else: else:
quota.release(server) quota.release(server)
queue_items.append(queue_item) queue_items.append(queue_item)
@ -158,7 +158,7 @@ class Worker(Thread):
LOG.error("Exception has occured", exc_info=1) LOG.error("Exception has occured", exc_info=1)
LOG.error("Worker %s: %s" % (self.name, ex)) LOG.error("Worker %s: %s" % (self.name, ex))
self.queue.deleteItem(queue_item) self.queue.delete(queue_item)
LOG.info("Worker %s destroyed!" % self.name) LOG.info("Worker %s destroyed!" % self.name)
@ -212,22 +212,25 @@ class SchedulerManager(Manager):
self.quota_manager.updateSharedQuota() self.quota_manager.updateSharedQuota()
try: try:
self.dynamic_queue = self.queue_manager.createQueue("DYNAMIC") self.queue = self.queue_manager.createQueue("DYNAMIC", "PRIORITY")
except SynergyError as ex: except SynergyError as ex:
LOG.error("Exception has occured", exc_info=1) LOG.error("Exception has occured", exc_info=1)
LOG.error(ex) LOG.error(ex)
self.dynamic_queue = self.queue_manager.getQueue("DYNAMIC") self.queue = self.queue_manager.getQueue("DYNAMIC")
dynamic_worker = Worker("DYNAMIC", for project in self.project_manager.getProjects():
self.dynamic_queue, project.setQueue(self.queue)
self.project_manager,
self.nova_manager,
self.keystone_manager,
self.backfill_depth)
dynamic_worker.start()
self.workers.append(dynamic_worker) worker = Worker("DYNAMIC",
self.queue,
self.project_manager,
self.nova_manager,
self.keystone_manager,
self.backfill_depth)
worker.start()
self.workers.append(worker)
self.quota_manager.deleteExpiredServers() self.quota_manager.deleteExpiredServers()
@ -246,6 +249,14 @@ class SchedulerManager(Manager):
self._processServerEvent(server, event, state) self._processServerEvent(server, event, state)
elif event_type == "SERVER_CREATE": elif event_type == "SERVER_CREATE":
self._processServerCreate(kwargs["request"]) self._processServerCreate(kwargs["request"])
elif event_type == "PROJECT_ADDED":
if not self.configured:
return
project = kwargs.get("project", None)
if self.queue and project:
project.setQueue(self.queue)
def _processServerEvent(self, server, event, state): def _processServerEvent(self, server, event, state):
if event == "compute.instance.create.end" and state == "active": if event == "compute.instance.create.end" and state == "active":
@ -356,11 +367,9 @@ class SchedulerManager(Manager):
token_admin.getUser().getId(), token_user) token_admin.getUser().getId(), token_user)
context["trust_id"] = trust.getId() context["trust_id"] = trust.getId()
user = project.getUser(id=request.getUserId())
self.dynamic_queue.insertItem(request.getUserId(), self.queue.enqueue(user, request.toDict(), priority)
request.getProjectId(),
priority=priority,
data=request.toDict())
LOG.info("new request: id=%s user_id=%s prj_id=%s priority" LOG.info("new request: id=%s user_id=%s prj_id=%s priority"
"=%s quota=shared" % (request.getId(), "=%s quota=shared" % (request.getId(),

10
synergy_scheduler_manager/tests/functional/test_scheduler_manager.py
View File

@ -14,7 +14,7 @@ from mock import create_autospec
from mock import MagicMock from mock import MagicMock
from sqlalchemy.engine.base import Engine from sqlalchemy.engine.base import Engine
from synergy_scheduler_manager.common.project import Project from synergy_scheduler_manager.common.project import Project
from synergy_scheduler_manager.common.queue import QueueDB from synergy_scheduler_manager.common.queue import Queue
from synergy_scheduler_manager.common.queue import QueueItem from synergy_scheduler_manager.common.queue import QueueItem
from synergy_scheduler_manager.project_manager import ProjectManager from synergy_scheduler_manager.project_manager import ProjectManager
from synergy_scheduler_manager.scheduler_manager import Worker from synergy_scheduler_manager.scheduler_manager import Worker
@ -124,6 +124,7 @@ class TestWorker(base.TestCase):
# TO COMPLETE # TO COMPLETE
def setUp(self): def setUp(self):
super(TestWorker, self).setUp() super(TestWorker, self).setUp()
self.db_engine_mock = create_autospec(Engine)
self.nova_manager_mock = MagicMock() self.nova_manager_mock = MagicMock()
self.keystone_manager_mock = MagicMock() self.keystone_manager_mock = MagicMock()
db_engine_mock = create_autospec(Engine) db_engine_mock = create_autospec(Engine)
@ -156,7 +157,7 @@ class TestWorker(base.TestCase):
self.worker = Worker( self.worker = Worker(
name="test", name="test",
queue=QueueDB("testq", db_engine_mock), queue=Queue("testq", db_engine=db_engine_mock),
project_manager=self.project_manager, project_manager=self.project_manager,
nova_manager=self.nova_manager_mock, nova_manager=self.nova_manager_mock,
keystone_manager=self.keystone_manager_mock) keystone_manager=self.keystone_manager_mock)
@ -171,7 +172,6 @@ class TestWorker(base.TestCase):
def test_run_build_server(self): def test_run_build_server(self):
def nova_exec_side_effect(command, *args, **kwargs): def nova_exec_side_effect(command, *args, **kwargs):
"""Mock nova.execute to do a successful build."""
if command == "GET_SERVER": if command == "GET_SERVER":
res = {"OS-EXT-STS:vm_state": "building", res = {"OS-EXT-STS:vm_state": "building",
"OS-EXT-STS:task_state": "scheduling"} "OS-EXT-STS:task_state": "scheduling"}
@ -188,7 +188,7 @@ class TestWorker(base.TestCase):
# Mock QueueItem in the queue # Mock QueueItem in the queue
qitem_mock = create_autospec(QueueItem) qitem_mock = create_autospec(QueueItem)
get_item_mock = create_autospec(self.worker.queue.getItem) get_item_mock = create_autospec(self.worker.queue.dequeue)
get_item_mock.return_value = qitem_mock get_item_mock.return_value = qitem_mock
self.worker.queue.getItem = get_item_mock self.worker.queue.getItem = get_item_mock
@ -203,5 +203,5 @@ class TestWorker(base.TestCase):
project.getQuota().allocate = quota_allocate_mock project.getQuota().allocate = quota_allocate_mock
# Delete item from the queue # Delete item from the queue
delete_item_mock = create_autospec(self.worker.queue.deleteItem) delete_item_mock = create_autospec(self.worker.queue.delete)
self.worker.queue.deleteItem = delete_item_mock self.worker.queue.deleteItem = delete_item_mock

225
synergy_scheduler_manager/tests/unit/test_queue.py
View File

@ -9,30 +9,32 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import heapq
import json
from datetime import datetime from datetime import datetime
from mock import call
from mock import create_autospec from mock import create_autospec
from sqlalchemy.engine.base import Engine from sqlalchemy.engine.base import Engine
from synergy_scheduler_manager.common.queue import PriorityQueue from synergy_scheduler_manager.common.queue import Queue
from synergy_scheduler_manager.common.queue import QueueDB
from synergy_scheduler_manager.common.queue import QueueItem from synergy_scheduler_manager.common.queue import QueueItem
from synergy_scheduler_manager.common.user import User
from synergy_scheduler_manager.tests.unit import base from synergy_scheduler_manager.tests.unit import base
import logging
LOG = logging.getLogger(__name__)
class TestQueueItem(base.TestCase): class TestQueueItem(base.TestCase):
def setUp(self): def setUp(self):
super(TestQueueItem, self).setUp() super(TestQueueItem, self).setUp()
self.qitem = QueueItem(id=1, self.qitem = QueueItem()
user_id=100, self.qitem.setId(1)
prj_id=1, self.qitem.setPriority(1000)
priority=1000, self.qitem.setUserId(100)
retry_count=1, self.qitem.setProjectId(1)
creation_time='now', self.qitem.setRetryCount(1)
last_update='now', self.qitem.setData(1)
data=1)
def test_get_set_id(self): def test_get_set_id(self):
self.assertEqual(1, self.qitem.getId()) self.assertEqual(1, self.qitem.getId())
@ -67,16 +69,20 @@ class TestQueueItem(base.TestCase):
self.assertEqual(11, self.qitem.getRetryCount()) self.assertEqual(11, self.qitem.getRetryCount())
def test_get_set_creation_time(self): def test_get_set_creation_time(self):
self.assertEqual("now", self.qitem.getCreationTime()) self.assertEqual(self.qitem.getCreationTime(),
self.qitem.getLastUpdate())
self.qitem.setCreationTime("later") now = datetime.now()
self.assertEqual("later", self.qitem.getCreationTime()) self.qitem.setCreationTime(now)
self.assertEqual(now, self.qitem.getCreationTime())
def test_get_set_last_update(self): def test_get_set_last_update(self):
self.assertEqual("now", self.qitem.getLastUpdate()) self.assertEqual(self.qitem.getCreationTime(),
self.qitem.getLastUpdate())
self.qitem.setLastUpdate("later") now = datetime.now()
self.assertEqual("later", self.qitem.getLastUpdate()) self.qitem.setLastUpdate(now)
self.assertEqual(now, self.qitem.getLastUpdate())
def test_get_set_data(self): def test_get_set_data(self):
self.assertEqual(1, self.qitem.getData()) self.assertEqual(1, self.qitem.getData())
@ -85,143 +91,104 @@ class TestQueueItem(base.TestCase):
self.assertEqual(2, self.qitem.getData()) self.assertEqual(2, self.qitem.getData())
class TestPriorityQueue(base.TestCase): class TestQueue(base.TestCase):
def setUp(self):
super(TestPriorityQueue, self).setUp()
self.pq = PriorityQueue()
now = datetime.now()
for i in range(0, 3):
item = QueueItem(id=i,
user_id=100,
prj_id=1,
priority=1000,
retry_count=1,
creation_time=now,
last_update=now,
data=i)
self.pq.put(i, item)
def test_get(self):
self.assertEqual(2, self.pq.get().getId())
self.assertEqual(1, self.pq.get().getId())
self.assertEqual(0, self.pq.get().getId())
def test_size(self):
self.assertEqual(3, self.pq.size())
def test_items(self):
self.assertEqual(3, len(self.pq.items()))
def test_smallest(self):
self.assertEqual(0, self.pq.smallest(1)[0].getId())
def test_largest(self):
self.assertEqual(2, self.pq.largest(1)[0].getId())
class TestQueueDB(base.TestCase):
def setUp(self): def setUp(self):
super(TestQueueDB, self).setUp() super(TestQueue, self).setUp()
# Create a Queue that mocks database interaction # Create a Queue that mocks database interaction
self.db_engine_mock = create_autospec(Engine) self.db_engine_mock = create_autospec(Engine)
self.q = QueueDB(name="test", db_engine=self.db_engine_mock) self.queue = Queue(name="test", db_engine=self.db_engine_mock)
def test_get_name(self): def test_get_name(self):
self.assertEqual('test', self.q.getName()) self.assertEqual('test', self.queue.getName())
def test_close(self): def test_close(self):
self.q.close() self.queue.close()
self.assertEqual(True, self.q.isClosed()) self.assertEqual(True, self.queue.isClosed())
def test_insert_item(self): def test_getSize(self):
self.assertEqual(0, self.queue.getSize())
self.q.insertItem(user_id=1, prj_id=2, priority=10, data="mydata") def test_enqueue(self):
self.assertEqual(0, self.queue.getSize())
user = User()
user.setId(2)
user.setProjectId(100)
# Check the db call of the item insert self.queue.enqueue(user=user, data="mydata", priority=10)
insert_call = call.connect().execute(
'insert into `test` (user_id, prj_id, priority, data) '
'values(%s, %s, %s, %s)', [1, 2, 10, '"mydata"'])
self.assertIn(insert_call, self.db_engine_mock.mock_calls)
# Check the item existence and values in the in-memory queue self.assertEqual(1, self.queue.getSize())
priority, timestamp, item = heapq.heappop(self.q.pqueue._heap)
self.assertEqual(-10, priority)
self.assertEqual(item.getCreationTime(), timestamp)
self.assertEqual(1, item.user_id)
self.assertEqual(2, item.prj_id)
self.assertEqual(10, item.priority)
self.assertEqual(0, item.retry_count)
self.assertIsNone(item.data) # TODO(vincent): should it be "mydata"?
def test_get_size(self): def test_dequeue(self):
execute_mock = self.db_engine_mock.connect().execute self.assertEqual(0, self.queue.getSize())
execute_call = call('select count(*) from `test`') user = User()
user.setId(2)
user.setProjectId(100)
data = json.dumps("mydata")
self.queue.enqueue(user=user, data=data, priority=10)
fetchone_mock = execute_mock().fetchone self.assertEqual(1, self.queue.getSize())
fetchone_mock.return_value = [3]
# Check that getSize() uses the correct sqlalchemy method # Mock the DB
self.assertEqual(3, self.q.getSize())
# Check that getSize() uses the correct SQL statement
self.assertEqual(execute_call, execute_mock.call_args)
def test_get_item(self):
# Insert the item and mock its DB insertion
execute_mock = self.db_engine_mock.connect().execute execute_mock = self.db_engine_mock.connect().execute
execute_mock().lastrowid = 123 execute_mock().lastrowid = 123
self.q.insertItem(user_id=1, prj_id=2, priority=10, data="mydata") fetchone_mock = execute_mock().fetchone
fetchone_mock.return_value = [data]
# Mock the DB select by returning the same things we inserted before qitem = self.queue.dequeue()
select_mock = self.db_engine_mock.connect().execute
select_call = call("select user_id, prj_id, priority, retry_count, "
"creation_time, last_update, data from `test` "
"where id=%s", [123])
fetchone_mock = select_mock().fetchone
fetchone_mock.return_value = [1, 2, 10, 0, "now", "now", '"mydata"']
item = self.q.getItem() self.assertIsNotNone(qitem)
self.assertEqual(select_call, select_mock.call_args) self.assertEqual(2, qitem.getUserId())
self.assertEqual(123, item.id) self.assertEqual(100, qitem.getProjectId())
self.assertEqual(1, item.user_id) self.assertEqual("mydata", qitem.getData())
self.assertEqual(2, item.prj_id)
self.assertEqual(10, item.priority)
self.assertEqual(0, item.retry_count)
self.assertEqual("now", item.creation_time)
self.assertEqual("now", item.last_update)
self.assertEqual("mydata", item.data)
def test_delete_item(self): def test_delete(self):
# Mock QueueItem to be deleted self.assertEqual(0, self.queue.getSize())
qitem = create_autospec(QueueItem) user = User()
qitem.getId.return_value = 123 user.setId(2)
user.setProjectId(100)
data = json.dumps("mydata")
# Mock the DB delete self.queue.enqueue(user=user, data=data, priority=10)
# Mock the DB
execute_mock = self.db_engine_mock.connect().execute execute_mock = self.db_engine_mock.connect().execute
execute_call = call("delete from `test` where id=%s", [123]) execute_mock().lastrowid = 123
fetchone_mock = execute_mock().fetchone
fetchone_mock.return_value = [data]
qitem = self.queue.dequeue()
self.q.deleteItem(qitem) execute_mock = self.db_engine_mock.connect().execute
self.assertEqual(execute_call, execute_mock.call_args) execute_mock().lastrowid = 123
self.queue.delete(qitem)
def test_update_item(self): def test_insertItemDB(self):
# Mock QueueItem to be updated qitem = QueueItem()
qitem = create_autospec(QueueItem) qitem.setPriority(1000)
qitem.getPriority.return_value = 10 qitem.setUserId(100)
qitem.getRetryCount.return_value = 20 qitem.setProjectId(1)
qitem.getLastUpdate.return_value = "right_now" qitem.setRetryCount(1)
qitem.getId.return_value = 123 qitem.setData(1)
# Check the db call of the item insert
execute_mock = self.db_engine_mock.connect().execute
execute_mock().lastrowid = 123
self.queue._insertItemDB(qitem)
self.assertEqual(123, qitem.getId())
def test_updateItemDB(self):
qitem = QueueItem()
qitem.setPriority(1000)
qitem.setUserId(100)
qitem.setProjectId(1)
qitem.setRetryCount(1)
qitem.setData(1)
lastUpdate = qitem.getLastUpdate()
# Mock the DB update # Mock the DB update
execute_mock = self.db_engine_mock.connect().execute execute_mock = self.db_engine_mock.connect().execute
execute_call = call("update `test` set priority=%s, retry_count=%s, " execute_mock().lastrowid = 123
"last_update=%s where id=%s",
[10, 20, "right_now", 123])
# Check the DB call and that the new QueueItem is in the queue self.queue._updateItemDB(qitem)
self.q.updateItem(qitem) self.assertNotEqual(lastUpdate, qitem.getLastUpdate())
self.assertEqual(execute_call, execute_mock.call_args)
self.assertIn((-10, qitem.getCreationTime(), qitem),
self.q.pqueue._heap)