add CLI command for executing elastic-recheck queries

This is an upstreaming of:

  https://github.com/dolph/spandex
  https://pypi.python.org/pypi/spandex

... which I'll nuke if this is merged.

I use this tool to write, test, and look for patterns in the results of
elastic recheck queries. I never use logstash.openstack.org anymore.

Change-Id: I864b22c05b398f6ad8ccb9009b5866f36b46789d

README.rst

@@ -172,6 +172,26 @@ Steps:
   then remove the related query.
 
 
+Running Queries Locally
+-----------------------
+
+You can execute an individual query locally and analyze the search results::
+
+    $ elastic-recheck-query queries/1331274.yaml
+    total hits: 133
+    build_status
+      100% FAILURE
+    build_name
+      48% check-grenade-dsvm
+      15% check-grenade-dsvm-partial-ncpu
+      13% gate-grenade-dsvm
+      9% check-grenade-dsvm-icehouse
+      9% check-grenade-dsvm-partial-ncpu-icehouse
+    build_branch
+      95% master
+      4% stable/icehouse
+
+
 Future Work
 -----------
 

elastic_recheck/cmd/query.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import argparse
+import base64
+import itertools
+import json
+import time
+
+import requests
+import yaml
+
+from elastic_recheck import log as logging
+
+
+LOG = logging.getLogger('erquery')
+
+ENDPOINT = 'http://logstash.openstack.org/api'
+DEFAULT_NUMBER_OF_DAYS = 10
+DEFAULT_MAX_QUANTITY = 5
+IGNORED_ATTRIBUTES = [
+    'build_master',
+    'build_patchset',
+    'build_ref',
+    'build_short_uuid',
+    'build_uuid',
+    'error_pr',
+    'host',
+    'received_at',
+    'type',
+]
+
+
+def _GET(path):
+    r = requests.get(ENDPOINT + path)
+
+    if r.status_code != requests.codes.ok:
+        LOG.info('Got HTTP %s, retrying...' % r.status_code)
+        # retry once
+        r = requests.get(ENDPOINT + path)
+
+    try:
+        return r.json()
+    except Exception:
+        raise SystemExit(r.text)
+
+
+def _encode(q):
+    """Encode a JSON dict for inclusion in a URL."""
+    return base64.b64encode(json.dumps(q))
+
+
+def _unix_time_in_microseconds():
+    return int(time.time() * 1000)
+
+
+def search(q, days):
+    search = {
+        'search': q,
+        'fields': [],
+        'offset': 0,
+        'timeframe': str(days * 86400),
+        'graphmode': 'count',
+        'time': {
+            'user_interval': 0},
+        'stamp': _unix_time_in_microseconds()}
+    return _GET('/search/%s' % _encode(search))
+
+
+def analyze_attributes(attributes):
+    analysis = {}
+    for attribute, values in attributes.iteritems():
+        if attribute[0] == '@' or attribute == 'message':
+            # skip meta attributes and raw messages
+            continue
+
+        analysis[attribute] = []
+
+        total_hits = sum(values.values())
+        for value_hash, hits in values.iteritems():
+            value = json.loads(value_hash)
+            analysis[attribute].append((100.0 * hits / total_hits, value))
+
+        # sort by hit percentage descending, and then by value ascending
+        analysis[attribute] = sorted(
+            analysis[attribute],
+            key=lambda x: (1 - x[0], x[1]))
+
+    return analysis
+
+
+def query(query_file_name, days=DEFAULT_NUMBER_OF_DAYS,
+          quantity=DEFAULT_MAX_QUANTITY, verbose=False):
+    with open(query_file_name) as f:
+        query_file = yaml.load(f.read())
+        query = query_file['query']
+
+    r = search(q=query, days=days)
+    print('total hits: %s' % r['hits']['total'])
+
+    attributes = {}
+    for hit in r['hits']['hits']:
+        for key, value in hit['_source'].iteritems():
+            value_hash = json.dumps(value)
+            attributes.setdefault(key, {}).setdefault(value_hash, 0)
+            attributes[key][value_hash] += 1
+
+    analysis = analyze_attributes(attributes)
+    for attribute, results in sorted(analysis.iteritems()):
+        if not verbose and attribute in IGNORED_ATTRIBUTES:
+            # skip less-than-useful attributes to reduce noise in the report
+            continue
+
+        print(attribute)
+        for percentage, value in itertools.islice(results, None, quantity):
+            if isinstance(value, list):
+                value = ' '.join(unicode(x) for x in value)
+            print('  %d%% %s' % (percentage, value))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Execute elastic-recheck query files and analyze the '
+                    'results.')
+    parser.add_argument(
+        'query_file', type=argparse.FileType('r'),
+        help='Path to an elastic-recheck YAML query file.')
+    parser.add_argument(
+        '--quantity', '-q', type=int, default=DEFAULT_MAX_QUANTITY,
+        help='Maximum quantity of values to show for each attribute.')
+    parser.add_argument(
+        '--days', '-d', type=float, default=DEFAULT_NUMBER_OF_DAYS,
+        help='Timespan to query, in days (may be a decimal).')
+    parser.add_argument(
+        '--verbose', '-v', action='store_true', default=False,
+        help='Report on additional query metadata.')
+    args = parser.parse_args()
+
+    query(args.query_file.name, args.days, args.quantity, args.verbose)
+
+
+if __name__ == "__main__":
+    main()

elastic_recheck/tests/unit/logstash/1284371.analysis

@@ -0,0 +1,48 @@
+total hits: 353
+build_branch
+  86% master
+  6% feature/lbaasv2
+  4% stable/icehouse
+  3% stable/juno
+build_change
+  5% 136792
+  4% 123485
+  3% 128258
+  2% 136511
+  1% 126244
+build_name
+  6% check-tempest-dsvm-full
+  6% check-tempest-dsvm-postgres-full
+  5% check-tempest-dsvm-neutron-full
+  5% check-grenade-dsvm
+  5% check-tempest-dsvm-neutron-heat-slow
+build_node
+  0% bare-centos6-rax-dfw-35738
+  0% bare-centos6-rax-dfw-35823
+  0% bare-centos6-rax-iad-36267
+  0% bare-precise-rax-iad-35805
+  0% bare-precise-rax-iad-36349
+build_queue
+  72% check
+  21% gate
+  4% check-tripleo
+  0% experimental
+  0% experimental-tripleo
+build_status
+  100% FAILURE
+filename
+  100% console.html
+log_url
+  0% http://logs.openstack.org/00/123000/8/check/check-neutron-dsvm-functional/da1210d/console.html
+  0% http://logs.openstack.org/00/123000/8/check/check-tempest-dsvm-neutron-full-2/78d5bf3/console.html
+  0% http://logs.openstack.org/00/123000/8/check/check-tempest-dsvm-neutron-heat-slow/caa6dc9/console.html
+  0% http://logs.openstack.org/00/123000/8/check/gate-rally-dsvm-neutron-neutron/58a4e58/console.html
+  0% http://logs.openstack.org/00/123000/8/check/gate-tempest-dsvm-neutron-large-ops/8faa759/console.html
+project
+  30% openstack/neutron
+  20% openstack/nova
+  6% openstack/cinder
+  6% openstack/tempest
+  5% openstack/swift
+tags
+  100% console.html console

elastic_recheck/tests/unit/queries/1284371.yaml

@@ -0,0 +1,4 @@
+query: >
+    message:"Looks like the node went offline during the build"
+    AND message:"slave.log \(No such file or directory\)"
+    AND filename:"console.html"

elastic_recheck/tests/unit/test_query.py

@@ -0,0 +1,51 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+import StringIO
+import sys
+
+import mock
+
+from elastic_recheck.cmd import query
+from elastic_recheck.tests import unit
+
+
+class FakeResponse(object):
+    def __init__(self, response_text):
+        super(FakeResponse, self).__init__()
+        self.text = response_text
+        self.status_code = 200
+
+    def json(self):
+        return json.loads(self.text)
+
+
+class TestQueryCmd(unit.UnitTestCase):
+    def setUp(self):
+        super(TestQueryCmd, self).setUp()
+        self._stdout = sys.stdout
+        sys.stdout = StringIO.StringIO()
+
+    def tearDown(self):
+        super(TestQueryCmd, self).tearDown()
+        sys.stdout = self._stdout
+
+    def test_query(self):
+        with open('elastic_recheck/tests/unit/logstash/1284371.analysis') as f:
+            expected_stdout = f.read()
+        with mock.patch('requests.get') as mock_get:
+            with open('elastic_recheck/tests/unit/logstash/1284371.json') as f:
+                mock_get.return_value = FakeResponse(f.read())
+            query.query('elastic_recheck/tests/unit/queries/1284371.yaml')
+            sys.stdout.seek(0)
+            self.assertEqual(expected_stdout, sys.stdout.read())

setup.cfg

@@ -31,6 +31,7 @@ console_scripts =
     elastic-recheck-graph = elastic_recheck.cmd.graph:main
     elastic-recheck-success = elastic_recheck.cmd.check_success:main
     elastic-recheck-uncategorized = elastic_recheck.cmd.uncategorized_fails:main
+    elastic-recheck-query = elastic_recheck.cmd.query:main
 
 [build_sphinx]
 source-dir = doc/source