Exists stats api

docs/dbapi.rst

@@ -41,7 +41,7 @@ Write APIs
 **********
 
 db/confirm/usage/exists/batch/
-=====================================
+==============================
 
 .. http:put:: http://example.com/db/confirm/usage/exists/batch/
 
@@ -100,6 +100,101 @@ Uses the provided message_id's and http status codes to update image and instanc
 Read APIs
 *********
 
+db/stats/nova/exists/
+=====================
+
+.. http:get:: http://example.com/db/stats/nova/exists
+
+Returns a list of status combinations and count of events with those status combinations.
+
+Note: Only status combinations with >0 count will show up.
+
+  **Query Parameters**
+
+  * ``audit_period_beginning_min``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``audit_period_beginning_max``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``audit_period_ending_min``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``audit_period_ending_max``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``launched_at_min``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``launched_at_max``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``deleted_at_min``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``deleted_at_max``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``received_min``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``received_max``: datetime (yyyy-mm-dd hh:mm:ss)
+
+  **Example request**:
+
+   .. sourcecode:: http
+
+      GET /db/stats/nova/exists/ HTTP/1.1
+      Host: example.com
+      Accept: application/json
+
+  **Example response**:
+
+   .. sourcecode:: http
+
+      HTTP/1.1 200 OK
+      Vary: Accept
+      Content-Type: application/json
+
+      {
+        "stats":
+        [
+          {"status": "pending", "send_status": 0, "event_count": 1},
+          {"status": "verified", "send_status": 200, "event_count": 100},
+          {"status": "reconciled", "send_status": 200, "event_count": 2},
+          {"status": "failed", "send_status": 0, "event_count": 1},
+        ]
+      }
+
+db/stats/glance/exists/
+=======================
+
+.. http:get:: http://example.com/db/status/usage/glance/exists
+
+Returns a list of status combinations and count of events with those status combinations.
+
+Note: Only status combinations with >0 count will show up.
+
+  **Query Parameters**
+
+  * ``audit_period_beginning_min``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``audit_period_beginning_max``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``audit_period_ending_min``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``audit_period_ending_max``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``created_at_min``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``created_at_max``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``deleted_at_min``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``deleted_at_max``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``received_min``: datetime (yyyy-mm-dd hh:mm:ss)
+  * ``received_max``: datetime (yyyy-mm-dd hh:mm:ss)
+
+  **Example request**:
+
+   .. sourcecode:: http
+
+      GET /db/stats/nova/exists/ HTTP/1.1
+      Host: example.com
+      Accept: application/json
+
+  **Example response**:
+
+   .. sourcecode:: http
+
+      HTTP/1.1 200 OK
+      Vary: Accept
+      Content-Type: application/json
+
+      {
+        "stats":
+        [
+          {"status": "verified", "send_status": 200, "event_count": 200},
+          {"status": "failed", "send_status": 0, "event_count": 2},
+        ]
+      }
+
+
 db/usage/launches/
 ==================
 

etc/pip-requires.txt

@@ -1,4 +1,4 @@
-Django>=1.4.2
+Django>=1.4.2, <1.6.0
 MySQL-python>=1.2.3
 eventlet>=0.9.17
 kombu>=2.4.7

stacktach/dbapi.py

@@ -23,6 +23,7 @@ import functools
 import json
 
 from django.db import transaction
+from django.db.models import Count
 from django.db.models import FieldDoesNotExist
 from django.forms.models import model_to_dict
 from django.http import HttpResponse
@@ -199,22 +200,7 @@ def list_usage_exists_glance(request):
 
 def list_usage_exists_with_service(request, service):
     model = _exists_model_factory(service)
-    try:
+    custom_filters = _get_exists_filter_args(request)
-        custom_filters = {}
-        if 'received_min' in request.GET:
-            received_min = request.GET['received_min']
-            custom_filters['received_min'] = {}
-            custom_filters['received_min']['raw__when__gte'] = \
-                utils.str_time_to_unix(received_min)
-        if 'received_max' in request.GET:
-            received_max = request.GET['received_max']
-            custom_filters['received_max'] = {}
-            custom_filters['received_max']['raw__when__lte'] = \
-                utils.str_time_to_unix(received_max)
-    except AttributeError:
-        msg = "Range filters must be dates."
-        raise BadRequestException(message=msg)
-
     objects = get_db_objects(model['klass'], request, 'id',
                              custom_filters=custom_filters)
     dicts = _convert_model_list(objects, _exists_extra_values)
@@ -232,6 +218,28 @@ def get_usage_exist_glance(request, exist_id):
                                       _exists_extra_values)}
 
 
+@api_call
+def get_usage_exist_stats(request):
+    return {'stats': _get_exist_stats(request, 'nova')}
+
+
+@api_call
+def get_usage_exist_stats_glance(request):
+    return {'stats': _get_exist_stats(request, 'glance')}
+
+
+def _get_exist_stats(request, service):
+    klass = _exists_model_factory(service)['klass']
+    exists_filters = _get_exists_filter_args(request)
+    filters = _get_filter_args(klass, request,
+                               custom_filters=exists_filters)
+    for value in exists_filters.values():
+        filters.update(value)
+    query = klass.objects.filter(**filters)
+    values = query.values('status', 'send_status')
+    stats = values.annotate(event_count=Count('send_status'))
+    return stats
+
 @api_call
 def exists_send_status(request, message_id):
     if request.method not in ['PUT', 'POST']:
@@ -330,6 +338,25 @@ def _check_has_field(klass, field_name):
         raise BadRequestException(msg)
 
 
+def _get_exists_filter_args(request):
+    try:
+        custom_filters = {}
+        if 'received_min' in request.GET:
+            received_min = request.GET['received_min']
+            custom_filters['received_min'] = {}
+            custom_filters['received_min']['raw__when__gte'] = \
+                utils.str_time_to_unix(received_min)
+        if 'received_max' in request.GET:
+            received_max = request.GET['received_max']
+            custom_filters['received_max'] = {}
+            custom_filters['received_max']['raw__when__lte'] = \
+                utils.str_time_to_unix(received_max)
+    except AttributeError:
+        msg = "Range filters must be dates."
+        raise BadRequestException(message=msg)
+    return custom_filters
+
+
 def _get_filter_args(klass, request, custom_filters=None):
     filter_args = {}
     if 'instance' in request.GET:

stacktach/urls.py

@@ -7,8 +7,22 @@ web_logger = stacklog.get_logger('stacktach-web')
 web_logger_listener = stacklog.LogListener(web_logger)
 web_logger_listener.start()
 
-urlpatterns = patterns('',
+web_urls = (
     url(r'^$', 'stacktach.views.welcome', name='welcome'),
+    url(r'^(?P<deployment_id>\d+)/$', 'stacktach.views.home', name='home'),
+    url(r'^(?P<deployment_id>\d+)/details/(?P<column>\w+)/(?P<row_id>\d+)/$',
+        'stacktach.views.details', name='details'),
+    url(r'^(?P<deployment_id>\d+)/search/$',
+        'stacktach.views.search', name='search'),
+    url(r'^(?P<deployment_id>\d+)/expand/(?P<row_id>\d+)/$',
+        'stacktach.views.expand', name='expand'),
+    url(r'^(?P<deployment_id>\d+)/latest_raw/$',
+        'stacktach.views.latest_raw', name='latest_raw'),
+    url(r'^(?P<deployment_id>\d+)/instance_status/$',
+        'stacktach.views.instance_status', name='instance_status'),
+)
+
+stacky_urls = (
     url(r'stacky/deployments/$', 'stacktach.stacky_server.do_deployments'),
     url(r'stacky/events/$', 'stacktach.stacky_server.do_events'),
     url(r'stacky/hosts/$', 'stacktach.stacky_server.do_hosts'),
@@ -35,7 +49,9 @@ urlpatterns = patterns('',
         'stacktach.stacky_server.do_list_usage_deletes'),
     url(r'stacky/usage/exists/$',
         'stacktach.stacky_server.do_list_usage_exists'),
+)
 
+dbapi_urls = (
     url(r'db/usage/launches/$',
         'stacktach.dbapi.list_usage_launches'),
     url(r'db/usage/nova/launches/$',
@@ -71,16 +87,10 @@ urlpatterns = patterns('',
         'stacktach.dbapi.get_usage_exist_glance'),
     url(r'db/confirm/usage/exists/(?P<message_id>[\w\-]+)/$',
         'stacktach.dbapi.exists_send_status'),
-
+    url(r'db/stats/nova/exists$',
-    url(r'^(?P<deployment_id>\d+)/$', 'stacktach.views.home', name='home'),
+        'stacktach.dbapi.get_usage_exist_stats'),
-    url(r'^(?P<deployment_id>\d+)/details/(?P<column>\w+)/(?P<row_id>\d+)/$',
+    url(r'db/stats/glance/exists$',
-        'stacktach.views.details', name='details'),
+        'stacktach.dbapi.get_usage_exist_stats_glance'),
-    url(r'^(?P<deployment_id>\d+)/search/$',
-        'stacktach.views.search', name='search'),
-    url(r'^(?P<deployment_id>\d+)/expand/(?P<row_id>\d+)/$',
-        'stacktach.views.expand', name='expand'),
-    url(r'^(?P<deployment_id>\d+)/latest_raw/$',
-        'stacktach.views.latest_raw', name='latest_raw'),
-    url(r'^(?P<deployment_id>\d+)/instance_status/$',
-        'stacktach.views.instance_status', name='instance_status'),
 )
+
+urlpatterns = patterns('', *(web_urls + stacky_urls + dbapi_urls))

tests/unit/test_dbapi.py

@@ -21,6 +21,7 @@
 import datetime
 import json
 
+from django.db.models import Count
 from django.db.models import FieldDoesNotExist
 from django.db import transaction
 import mox
@@ -43,6 +44,10 @@ class DBAPITestCase(StacktachBaseTestCase):
         mor_exception = models.InstanceExists.MultipleObjectsReturned
         self.mox.StubOutWithMock(models, 'InstanceExists',
                                  use_mock_anything=True)
+        self.mox.StubOutWithMock(models, 'ImageExists',
+                                 use_mock_anything=True)
+        models.InstanceExists._meta = self.mox.CreateMockAnything()
+        models.ImageExists._meta = self.mox.CreateMockAnything()
         models.InstanceExists.objects = self.mox.CreateMockAnything()
         models.ImageExists.objects = self.mox.CreateMockAnything()
         models.InstanceExists.DoesNotExist = dne_exception
@@ -921,3 +926,169 @@ class DBAPITestCase(StacktachBaseTestCase):
         self.assertEqual(resp.status_code, 200)
         self.assertEqual(json.loads(resp.content), {'deletes': deletes})
         self.mox.VerifyAll()
+
+    def test_get_usage_exist_stats_nova(self):
+        fake_request = self.mox.CreateMockAnything()
+        fake_request.method = 'GET'
+        fake_request.GET = {}
+        query = self.mox.CreateMockAnything()
+        models.InstanceExists.objects.filter().AndReturn(query)
+        query.values('status', 'send_status').AndReturn(query)
+        result = [
+            {'status': 'verified', 'send_status': 201L, 'event_count': 2},
+            {'status': 'failed', 'send_status': 0L, 'event_count': 1}
+        ]
+        query.annotate(event_count=mox.IsA(Count)).AndReturn(result)
+        self.mox.ReplayAll()
+        response = dbapi.get_usage_exist_stats(fake_request)
+        self.assertEqual(response.status_code, 200)
+        expected_response = json.dumps({'stats': result})
+        self.assertEqual(expected_response, response.content)
+        self.mox.VerifyAll()
+
+    def test_get_usage_exist_stats_nova_received_min(self):
+        fake_request = self.mox.CreateMockAnything()
+        fake_request.method = 'GET'
+        now = datetime.datetime.utcnow()
+        fake_request.GET = {'received_min': str(now)}
+        query = self.mox.CreateMockAnything()
+        filters = {'raw__when__gte': utils.decimal_utc(now)}
+        models.InstanceExists.objects.filter(**filters).AndReturn(query)
+        query.values('status', 'send_status').AndReturn(query)
+        result = [
+            {'status': 'verified', 'send_status': 201L, 'event_count': 2},
+            {'status': 'failed', 'send_status': 0L, 'event_count': 1}
+        ]
+        query.annotate(event_count=mox.IsA(Count)).AndReturn(result)
+        self.mox.ReplayAll()
+        response = dbapi.get_usage_exist_stats(fake_request)
+        self.assertEqual(response.status_code, 200)
+        expected_response = json.dumps({'stats': result})
+        self.assertEqual(expected_response, response.content)
+        self.mox.VerifyAll()
+
+    def test_get_usage_exist_stats_nova_received_max(self):
+        fake_request = self.mox.CreateMockAnything()
+        fake_request.method = 'GET'
+        now = datetime.datetime.utcnow()
+        fake_request.GET = {'received_max': str(now)}
+        query = self.mox.CreateMockAnything()
+        filters = {'raw__when__lte': utils.decimal_utc(now)}
+        models.InstanceExists.objects.filter(**filters).AndReturn(query)
+        query.values('status', 'send_status').AndReturn(query)
+        result = [
+            {'status': 'verified', 'send_status': 201L, 'event_count': 2},
+            {'status': 'failed', 'send_status': 0L, 'event_count': 1}
+        ]
+        query.annotate(event_count=mox.IsA(Count)).AndReturn(result)
+        self.mox.ReplayAll()
+        response = dbapi.get_usage_exist_stats(fake_request)
+        self.assertEqual(response.status_code, 200)
+        expected_response = json.dumps({'stats': result})
+        self.assertEqual(expected_response, response.content)
+        self.mox.VerifyAll()
+
+    def test_get_usage_exist_stats_nova_class_field_filter(self):
+        fake_request = self.mox.CreateMockAnything()
+        fake_request.method = 'GET'
+        now = datetime.datetime.utcnow()
+        fake_request.GET = {'audit_period_ending_min': str(now)}
+        query = self.mox.CreateMockAnything()
+        models.InstanceExists._meta.get_field_by_name('audit_period_ending')
+        filters = {'audit_period_ending__gte': utils.decimal_utc(now)}
+        models.InstanceExists.objects.filter(**filters).AndReturn(query)
+        query.values('status', 'send_status').AndReturn(query)
+        result = [
+            {'status': 'verified', 'send_status': 201L, 'event_count': 2},
+            {'status': 'failed', 'send_status': 0L, 'event_count': 1}
+        ]
+        query.annotate(event_count=mox.IsA(Count)).AndReturn(result)
+        self.mox.ReplayAll()
+        response = dbapi.get_usage_exist_stats(fake_request)
+        self.assertEqual(response.status_code, 200)
+        expected_response = json.dumps({'stats': result})
+        self.assertEqual(expected_response, response.content)
+        self.mox.VerifyAll()
+
+    def test_get_usage_exist_stats_glance(self):
+        fake_request = self.mox.CreateMockAnything()
+        fake_request.method = 'GET'
+        fake_request.GET = {}
+        query = self.mox.CreateMockAnything()
+        models.ImageExists.objects.filter().AndReturn(query)
+        query.values('status', 'send_status').AndReturn(query)
+        result = [
+            {'status': 'verified', 'send_status': 201L, 'event_count': 2},
+            {'status': 'failed', 'send_status': 0L, 'event_count': 1}
+        ]
+        query.annotate(event_count=mox.IsA(Count)).AndReturn(result)
+        self.mox.ReplayAll()
+        response = dbapi.get_usage_exist_stats_glance(fake_request)
+        self.assertEqual(response.status_code, 200)
+        expected_response = json.dumps({'stats': result})
+        self.assertEqual(expected_response, response.content)
+        self.mox.VerifyAll()
+
+    def test_get_usage_exist_stats_glance_received_min(self):
+        fake_request = self.mox.CreateMockAnything()
+        fake_request.method = 'GET'
+        now = datetime.datetime.utcnow()
+        fake_request.GET = {'received_min': str(now)}
+        query = self.mox.CreateMockAnything()
+        filters = {'raw__when__gte': utils.decimal_utc(now)}
+        models.ImageExists.objects.filter(**filters).AndReturn(query)
+        query.values('status', 'send_status').AndReturn(query)
+        result = [
+            {'status': 'verified', 'send_status': 201L, 'event_count': 2},
+            {'status': 'failed', 'send_status': 0L, 'event_count': 1}
+        ]
+        query.annotate(event_count=mox.IsA(Count)).AndReturn(result)
+        self.mox.ReplayAll()
+        response = dbapi.get_usage_exist_stats_glance(fake_request)
+        self.assertEqual(response.status_code, 200)
+        expected_response = json.dumps({'stats': result})
+        self.assertEqual(expected_response, response.content)
+        self.mox.VerifyAll()
+
+    def test_get_usage_exist_stats_glance_received_max(self):
+        fake_request = self.mox.CreateMockAnything()
+        fake_request.method = 'GET'
+        now = datetime.datetime.utcnow()
+        fake_request.GET = {'received_max': str(now)}
+        query = self.mox.CreateMockAnything()
+        filters = {'raw__when__lte': utils.decimal_utc(now)}
+        models.ImageExists.objects.filter(**filters).AndReturn(query)
+        query.values('status', 'send_status').AndReturn(query)
+        result = [
+            {'status': 'verified', 'send_status': 201L, 'event_count': 2},
+            {'status': 'failed', 'send_status': 0L, 'event_count': 1}
+        ]
+        query.annotate(event_count=mox.IsA(Count)).AndReturn(result)
+        self.mox.ReplayAll()
+        response = dbapi.get_usage_exist_stats_glance(fake_request)
+        self.assertEqual(response.status_code, 200)
+        expected_response = json.dumps({'stats': result})
+        self.assertEqual(expected_response, response.content)
+        self.mox.VerifyAll()
+
+    def test_get_usage_exist_stats_glance_class_field_filter(self):
+        fake_request = self.mox.CreateMockAnything()
+        fake_request.method = 'GET'
+        now = datetime.datetime.utcnow()
+        fake_request.GET = {'audit_period_ending_min': str(now)}
+        query = self.mox.CreateMockAnything()
+        models.ImageExists._meta.get_field_by_name('audit_period_ending')
+        filters = {'audit_period_ending__gte': utils.decimal_utc(now)}
+        models.ImageExists.objects.filter(**filters).AndReturn(query)
+        query.values('status', 'send_status').AndReturn(query)
+        result = [
+            {'status': 'verified', 'send_status': 201L, 'event_count': 2},
+            {'status': 'failed', 'send_status': 0L, 'event_count': 1}
+        ]
+        query.annotate(event_count=mox.IsA(Count)).AndReturn(result)
+        self.mox.ReplayAll()
+        response = dbapi.get_usage_exist_stats_glance(fake_request)
+        self.assertEqual(response.status_code, 200)
+        expected_response = json.dumps({'stats': result})
+        self.assertEqual(expected_response, response.content)
+        self.mox.VerifyAll()