From cb29fb70220f6668aa2ca245dc31bfcd60684284 Mon Sep 17 00:00:00 2001 From: Andrew Melton Date: Thu, 27 Feb 2014 11:15:41 -0500 Subject: [PATCH] Exists stats api --- docs/dbapi.rst | 97 +++++++++++++++++++++- etc/pip-requires.txt | 4 +- stacktach/dbapi.py | 59 ++++++++++---- stacktach/urls.py | 36 ++++++--- tests/unit/test_dbapi.py | 171 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 335 insertions(+), 32 deletions(-) diff --git a/docs/dbapi.rst b/docs/dbapi.rst index e6d596f..d23cac4 100644 --- a/docs/dbapi.rst +++ b/docs/dbapi.rst @@ -41,7 +41,7 @@ Write APIs ********** db/confirm/usage/exists/batch/ -===================================== +============================== .. http:put:: http://example.com/db/confirm/usage/exists/batch/ @@ -100,6 +100,101 @@ Uses the provided message_id's and http status codes to update image and instanc Read APIs ********* +db/stats/nova/exists/ +===================== + +.. http:get:: http://example.com/db/stats/nova/exists + +Returns a list of status combinations and count of events with those status combinations. + +Note: Only status combinations with >0 count will show up. + + **Query Parameters** + + * ``audit_period_beginning_min``: datetime (yyyy-mm-dd hh:mm:ss) + * ``audit_period_beginning_max``: datetime (yyyy-mm-dd hh:mm:ss) + * ``audit_period_ending_min``: datetime (yyyy-mm-dd hh:mm:ss) + * ``audit_period_ending_max``: datetime (yyyy-mm-dd hh:mm:ss) + * ``launched_at_min``: datetime (yyyy-mm-dd hh:mm:ss) + * ``launched_at_max``: datetime (yyyy-mm-dd hh:mm:ss) + * ``deleted_at_min``: datetime (yyyy-mm-dd hh:mm:ss) + * ``deleted_at_max``: datetime (yyyy-mm-dd hh:mm:ss) + * ``received_min``: datetime (yyyy-mm-dd hh:mm:ss) + * ``received_max``: datetime (yyyy-mm-dd hh:mm:ss) + + **Example request**: + + .. sourcecode:: http + + GET /db/stats/nova/exists/ HTTP/1.1 + Host: example.com + Accept: application/json + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept + Content-Type: application/json + + { + "stats": + [ + {"status": "pending", "send_status": 0, "event_count": 1}, + {"status": "verified", "send_status": 200, "event_count": 100}, + {"status": "reconciled", "send_status": 200, "event_count": 2}, + {"status": "failed", "send_status": 0, "event_count": 1}, + ] + } + +db/stats/glance/exists/ +======================= + +.. http:get:: http://example.com/db/status/usage/glance/exists + +Returns a list of status combinations and count of events with those status combinations. + +Note: Only status combinations with >0 count will show up. + + **Query Parameters** + + * ``audit_period_beginning_min``: datetime (yyyy-mm-dd hh:mm:ss) + * ``audit_period_beginning_max``: datetime (yyyy-mm-dd hh:mm:ss) + * ``audit_period_ending_min``: datetime (yyyy-mm-dd hh:mm:ss) + * ``audit_period_ending_max``: datetime (yyyy-mm-dd hh:mm:ss) + * ``created_at_min``: datetime (yyyy-mm-dd hh:mm:ss) + * ``created_at_max``: datetime (yyyy-mm-dd hh:mm:ss) + * ``deleted_at_min``: datetime (yyyy-mm-dd hh:mm:ss) + * ``deleted_at_max``: datetime (yyyy-mm-dd hh:mm:ss) + * ``received_min``: datetime (yyyy-mm-dd hh:mm:ss) + * ``received_max``: datetime (yyyy-mm-dd hh:mm:ss) + + **Example request**: + + .. sourcecode:: http + + GET /db/stats/nova/exists/ HTTP/1.1 + Host: example.com + Accept: application/json + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Vary: Accept + Content-Type: application/json + + { + "stats": + [ + {"status": "verified", "send_status": 200, "event_count": 200}, + {"status": "failed", "send_status": 0, "event_count": 2}, + ] + } + + db/usage/launches/ ================== diff --git a/etc/pip-requires.txt b/etc/pip-requires.txt index f14c1f6..72c109e 100644 --- a/etc/pip-requires.txt +++ b/etc/pip-requires.txt @@ -1,4 +1,4 @@ -Django>=1.4.2 +Django>=1.4.2, <1.6.0 MySQL-python>=1.2.3 eventlet>=0.9.17 kombu>=2.4.7 @@ -9,4 +9,4 @@ Pympler requests south sphinxcontrib-httpdomain -pbr \ No newline at end of file +pbr diff --git a/stacktach/dbapi.py b/stacktach/dbapi.py index 1d0cc4d..174c50b 100644 --- a/stacktach/dbapi.py +++ b/stacktach/dbapi.py @@ -23,6 +23,7 @@ import functools import json from django.db import transaction +from django.db.models import Count from django.db.models import FieldDoesNotExist from django.forms.models import model_to_dict from django.http import HttpResponse @@ -199,22 +200,7 @@ def list_usage_exists_glance(request): def list_usage_exists_with_service(request, service): model = _exists_model_factory(service) - try: - custom_filters = {} - if 'received_min' in request.GET: - received_min = request.GET['received_min'] - custom_filters['received_min'] = {} - custom_filters['received_min']['raw__when__gte'] = \ - utils.str_time_to_unix(received_min) - if 'received_max' in request.GET: - received_max = request.GET['received_max'] - custom_filters['received_max'] = {} - custom_filters['received_max']['raw__when__lte'] = \ - utils.str_time_to_unix(received_max) - except AttributeError: - msg = "Range filters must be dates." - raise BadRequestException(message=msg) - + custom_filters = _get_exists_filter_args(request) objects = get_db_objects(model['klass'], request, 'id', custom_filters=custom_filters) dicts = _convert_model_list(objects, _exists_extra_values) @@ -232,6 +218,28 @@ def get_usage_exist_glance(request, exist_id): _exists_extra_values)} +@api_call +def get_usage_exist_stats(request): + return {'stats': _get_exist_stats(request, 'nova')} + + +@api_call +def get_usage_exist_stats_glance(request): + return {'stats': _get_exist_stats(request, 'glance')} + + +def _get_exist_stats(request, service): + klass = _exists_model_factory(service)['klass'] + exists_filters = _get_exists_filter_args(request) + filters = _get_filter_args(klass, request, + custom_filters=exists_filters) + for value in exists_filters.values(): + filters.update(value) + query = klass.objects.filter(**filters) + values = query.values('status', 'send_status') + stats = values.annotate(event_count=Count('send_status')) + return stats + @api_call def exists_send_status(request, message_id): if request.method not in ['PUT', 'POST']: @@ -330,6 +338,25 @@ def _check_has_field(klass, field_name): raise BadRequestException(msg) +def _get_exists_filter_args(request): + try: + custom_filters = {} + if 'received_min' in request.GET: + received_min = request.GET['received_min'] + custom_filters['received_min'] = {} + custom_filters['received_min']['raw__when__gte'] = \ + utils.str_time_to_unix(received_min) + if 'received_max' in request.GET: + received_max = request.GET['received_max'] + custom_filters['received_max'] = {} + custom_filters['received_max']['raw__when__lte'] = \ + utils.str_time_to_unix(received_max) + except AttributeError: + msg = "Range filters must be dates." + raise BadRequestException(message=msg) + return custom_filters + + def _get_filter_args(klass, request, custom_filters=None): filter_args = {} if 'instance' in request.GET: diff --git a/stacktach/urls.py b/stacktach/urls.py index e49c217..b5d471c 100644 --- a/stacktach/urls.py +++ b/stacktach/urls.py @@ -7,8 +7,22 @@ web_logger = stacklog.get_logger('stacktach-web') web_logger_listener = stacklog.LogListener(web_logger) web_logger_listener.start() -urlpatterns = patterns('', +web_urls = ( url(r'^$', 'stacktach.views.welcome', name='welcome'), + url(r'^(?P\d+)/$', 'stacktach.views.home', name='home'), + url(r'^(?P\d+)/details/(?P\w+)/(?P\d+)/$', + 'stacktach.views.details', name='details'), + url(r'^(?P\d+)/search/$', + 'stacktach.views.search', name='search'), + url(r'^(?P\d+)/expand/(?P\d+)/$', + 'stacktach.views.expand', name='expand'), + url(r'^(?P\d+)/latest_raw/$', + 'stacktach.views.latest_raw', name='latest_raw'), + url(r'^(?P\d+)/instance_status/$', + 'stacktach.views.instance_status', name='instance_status'), +) + +stacky_urls = ( url(r'stacky/deployments/$', 'stacktach.stacky_server.do_deployments'), url(r'stacky/events/$', 'stacktach.stacky_server.do_events'), url(r'stacky/hosts/$', 'stacktach.stacky_server.do_hosts'), @@ -35,7 +49,9 @@ urlpatterns = patterns('', 'stacktach.stacky_server.do_list_usage_deletes'), url(r'stacky/usage/exists/$', 'stacktach.stacky_server.do_list_usage_exists'), +) +dbapi_urls = ( url(r'db/usage/launches/$', 'stacktach.dbapi.list_usage_launches'), url(r'db/usage/nova/launches/$', @@ -71,16 +87,10 @@ urlpatterns = patterns('', 'stacktach.dbapi.get_usage_exist_glance'), url(r'db/confirm/usage/exists/(?P[\w\-]+)/$', 'stacktach.dbapi.exists_send_status'), - - url(r'^(?P\d+)/$', 'stacktach.views.home', name='home'), - url(r'^(?P\d+)/details/(?P\w+)/(?P\d+)/$', - 'stacktach.views.details', name='details'), - url(r'^(?P\d+)/search/$', - 'stacktach.views.search', name='search'), - url(r'^(?P\d+)/expand/(?P\d+)/$', - 'stacktach.views.expand', name='expand'), - url(r'^(?P\d+)/latest_raw/$', - 'stacktach.views.latest_raw', name='latest_raw'), - url(r'^(?P\d+)/instance_status/$', - 'stacktach.views.instance_status', name='instance_status'), + url(r'db/stats/nova/exists$', + 'stacktach.dbapi.get_usage_exist_stats'), + url(r'db/stats/glance/exists$', + 'stacktach.dbapi.get_usage_exist_stats_glance'), ) + +urlpatterns = patterns('', *(web_urls + stacky_urls + dbapi_urls)) diff --git a/tests/unit/test_dbapi.py b/tests/unit/test_dbapi.py index 4b335a5..3de9b11 100644 --- a/tests/unit/test_dbapi.py +++ b/tests/unit/test_dbapi.py @@ -21,6 +21,7 @@ import datetime import json +from django.db.models import Count from django.db.models import FieldDoesNotExist from django.db import transaction import mox @@ -43,6 +44,10 @@ class DBAPITestCase(StacktachBaseTestCase): mor_exception = models.InstanceExists.MultipleObjectsReturned self.mox.StubOutWithMock(models, 'InstanceExists', use_mock_anything=True) + self.mox.StubOutWithMock(models, 'ImageExists', + use_mock_anything=True) + models.InstanceExists._meta = self.mox.CreateMockAnything() + models.ImageExists._meta = self.mox.CreateMockAnything() models.InstanceExists.objects = self.mox.CreateMockAnything() models.ImageExists.objects = self.mox.CreateMockAnything() models.InstanceExists.DoesNotExist = dne_exception @@ -921,3 +926,169 @@ class DBAPITestCase(StacktachBaseTestCase): self.assertEqual(resp.status_code, 200) self.assertEqual(json.loads(resp.content), {'deletes': deletes}) self.mox.VerifyAll() + + def test_get_usage_exist_stats_nova(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + fake_request.GET = {} + query = self.mox.CreateMockAnything() + models.InstanceExists.objects.filter().AndReturn(query) + query.values('status', 'send_status').AndReturn(query) + result = [ + {'status': 'verified', 'send_status': 201L, 'event_count': 2}, + {'status': 'failed', 'send_status': 0L, 'event_count': 1} + ] + query.annotate(event_count=mox.IsA(Count)).AndReturn(result) + self.mox.ReplayAll() + response = dbapi.get_usage_exist_stats(fake_request) + self.assertEqual(response.status_code, 200) + expected_response = json.dumps({'stats': result}) + self.assertEqual(expected_response, response.content) + self.mox.VerifyAll() + + def test_get_usage_exist_stats_nova_received_min(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + now = datetime.datetime.utcnow() + fake_request.GET = {'received_min': str(now)} + query = self.mox.CreateMockAnything() + filters = {'raw__when__gte': utils.decimal_utc(now)} + models.InstanceExists.objects.filter(**filters).AndReturn(query) + query.values('status', 'send_status').AndReturn(query) + result = [ + {'status': 'verified', 'send_status': 201L, 'event_count': 2}, + {'status': 'failed', 'send_status': 0L, 'event_count': 1} + ] + query.annotate(event_count=mox.IsA(Count)).AndReturn(result) + self.mox.ReplayAll() + response = dbapi.get_usage_exist_stats(fake_request) + self.assertEqual(response.status_code, 200) + expected_response = json.dumps({'stats': result}) + self.assertEqual(expected_response, response.content) + self.mox.VerifyAll() + + def test_get_usage_exist_stats_nova_received_max(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + now = datetime.datetime.utcnow() + fake_request.GET = {'received_max': str(now)} + query = self.mox.CreateMockAnything() + filters = {'raw__when__lte': utils.decimal_utc(now)} + models.InstanceExists.objects.filter(**filters).AndReturn(query) + query.values('status', 'send_status').AndReturn(query) + result = [ + {'status': 'verified', 'send_status': 201L, 'event_count': 2}, + {'status': 'failed', 'send_status': 0L, 'event_count': 1} + ] + query.annotate(event_count=mox.IsA(Count)).AndReturn(result) + self.mox.ReplayAll() + response = dbapi.get_usage_exist_stats(fake_request) + self.assertEqual(response.status_code, 200) + expected_response = json.dumps({'stats': result}) + self.assertEqual(expected_response, response.content) + self.mox.VerifyAll() + + def test_get_usage_exist_stats_nova_class_field_filter(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + now = datetime.datetime.utcnow() + fake_request.GET = {'audit_period_ending_min': str(now)} + query = self.mox.CreateMockAnything() + models.InstanceExists._meta.get_field_by_name('audit_period_ending') + filters = {'audit_period_ending__gte': utils.decimal_utc(now)} + models.InstanceExists.objects.filter(**filters).AndReturn(query) + query.values('status', 'send_status').AndReturn(query) + result = [ + {'status': 'verified', 'send_status': 201L, 'event_count': 2}, + {'status': 'failed', 'send_status': 0L, 'event_count': 1} + ] + query.annotate(event_count=mox.IsA(Count)).AndReturn(result) + self.mox.ReplayAll() + response = dbapi.get_usage_exist_stats(fake_request) + self.assertEqual(response.status_code, 200) + expected_response = json.dumps({'stats': result}) + self.assertEqual(expected_response, response.content) + self.mox.VerifyAll() + + def test_get_usage_exist_stats_glance(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + fake_request.GET = {} + query = self.mox.CreateMockAnything() + models.ImageExists.objects.filter().AndReturn(query) + query.values('status', 'send_status').AndReturn(query) + result = [ + {'status': 'verified', 'send_status': 201L, 'event_count': 2}, + {'status': 'failed', 'send_status': 0L, 'event_count': 1} + ] + query.annotate(event_count=mox.IsA(Count)).AndReturn(result) + self.mox.ReplayAll() + response = dbapi.get_usage_exist_stats_glance(fake_request) + self.assertEqual(response.status_code, 200) + expected_response = json.dumps({'stats': result}) + self.assertEqual(expected_response, response.content) + self.mox.VerifyAll() + + def test_get_usage_exist_stats_glance_received_min(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + now = datetime.datetime.utcnow() + fake_request.GET = {'received_min': str(now)} + query = self.mox.CreateMockAnything() + filters = {'raw__when__gte': utils.decimal_utc(now)} + models.ImageExists.objects.filter(**filters).AndReturn(query) + query.values('status', 'send_status').AndReturn(query) + result = [ + {'status': 'verified', 'send_status': 201L, 'event_count': 2}, + {'status': 'failed', 'send_status': 0L, 'event_count': 1} + ] + query.annotate(event_count=mox.IsA(Count)).AndReturn(result) + self.mox.ReplayAll() + response = dbapi.get_usage_exist_stats_glance(fake_request) + self.assertEqual(response.status_code, 200) + expected_response = json.dumps({'stats': result}) + self.assertEqual(expected_response, response.content) + self.mox.VerifyAll() + + def test_get_usage_exist_stats_glance_received_max(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + now = datetime.datetime.utcnow() + fake_request.GET = {'received_max': str(now)} + query = self.mox.CreateMockAnything() + filters = {'raw__when__lte': utils.decimal_utc(now)} + models.ImageExists.objects.filter(**filters).AndReturn(query) + query.values('status', 'send_status').AndReturn(query) + result = [ + {'status': 'verified', 'send_status': 201L, 'event_count': 2}, + {'status': 'failed', 'send_status': 0L, 'event_count': 1} + ] + query.annotate(event_count=mox.IsA(Count)).AndReturn(result) + self.mox.ReplayAll() + response = dbapi.get_usage_exist_stats_glance(fake_request) + self.assertEqual(response.status_code, 200) + expected_response = json.dumps({'stats': result}) + self.assertEqual(expected_response, response.content) + self.mox.VerifyAll() + + def test_get_usage_exist_stats_glance_class_field_filter(self): + fake_request = self.mox.CreateMockAnything() + fake_request.method = 'GET' + now = datetime.datetime.utcnow() + fake_request.GET = {'audit_period_ending_min': str(now)} + query = self.mox.CreateMockAnything() + models.ImageExists._meta.get_field_by_name('audit_period_ending') + filters = {'audit_period_ending__gte': utils.decimal_utc(now)} + models.ImageExists.objects.filter(**filters).AndReturn(query) + query.values('status', 'send_status').AndReturn(query) + result = [ + {'status': 'verified', 'send_status': 201L, 'event_count': 2}, + {'status': 'failed', 'send_status': 0L, 'event_count': 1} + ] + query.annotate(event_count=mox.IsA(Count)).AndReturn(result) + self.mox.ReplayAll() + response = dbapi.get_usage_exist_stats_glance(fake_request) + self.assertEqual(response.status_code, 200) + expected_response = json.dumps({'stats': result}) + self.assertEqual(expected_response, response.content) + self.mox.VerifyAll()