From 8ab2a069b408284a5e6940876f6bb0af7ba2bfae Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Fri, 27 Jun 2014 22:25:19 +0400 Subject: [PATCH] Fix py33 compatibility errors Part of blueprint py33 Change-Id: Ib24bb7e5147a6d241d0392889832ed34e1c856eb --- stackalytics/dashboard/reports.py | 7 +-- stackalytics/processor/mls.py | 6 +-- stackalytics/processor/mps.py | 2 +- stackalytics/processor/normalizer.py | 4 +- stackalytics/processor/rcs.py | 2 +- stackalytics/processor/runtime_storage.py | 14 ++++-- stackalytics/processor/utils.py | 53 +++++++++++++++++++---- stackalytics/processor/vcs.py | 6 +-- tests/unit/test_config_files.py | 24 +++++++--- tests/unit/test_record_processor.py | 26 +++++++---- tests/unit/test_vcs.py | 13 +++--- tox.ini | 10 +++-- 12 files changed, 118 insertions(+), 49 deletions(-) diff --git a/stackalytics/dashboard/reports.py b/stackalytics/dashboard/reports.py index d9efc7290..b03105e11 100644 --- a/stackalytics/dashboard/reports.py +++ b/stackalytics/dashboard/reports.py @@ -20,6 +20,7 @@ import operator import time import flask +import six from stackalytics.dashboard import decorators from stackalytics.dashboard import helpers @@ -159,15 +160,15 @@ def members(): def _get_punch_card_data(records): punch_card_raw = [] # matrix days x hours - for wday in xrange(0, 7): + for wday in six.moves.range(0, 7): punch_card_raw.append([0] * 24) for record in records: tt = datetime.datetime.fromtimestamp(record['date']).timetuple() punch_card_raw[tt.tm_wday][tt.tm_hour] += 1 punch_card_data = [] # format for jqplot bubble renderer - for wday in xrange(0, 7): - for hour in xrange(0, 24): + for wday in six.moves.range(0, 7): + for hour in six.moves.range(0, 24): v = punch_card_raw[wday][hour] if v: punch_card_data.append([hour, wday, v, v]) diff --git a/stackalytics/processor/mls.py b/stackalytics/processor/mls.py index 18865a81a..35a2781ad 100644 --- a/stackalytics/processor/mls.py +++ b/stackalytics/processor/mls.py @@ -14,9 +14,7 @@ # limitations under the License. from email import utils as email_utils -import gzip import re -import StringIO import six from six.moves import http_client @@ -85,8 +83,8 @@ def _retrieve_mails(uri): if not content: LOG.error('Error reading mail archive from uri: %s', uri) return - gzip_fd = gzip.GzipFile(fileobj=StringIO.StringIO(content)) - content = gzip_fd.read() + + content = utils.gzip_decompress(content) LOG.debug('Mail archive is loaded, start processing') content += TRAILING_RECORD diff --git a/stackalytics/processor/mps.py b/stackalytics/processor/mps.py index 219b85840..f6ca05fde 100644 --- a/stackalytics/processor/mps.py +++ b/stackalytics/processor/mps.py @@ -33,7 +33,7 @@ CNT_EMPTY_MEMBERS = 50 def _convert_str_fields_to_unicode(result): - for field, value in result.iteritems(): + for field, value in six.iteritems(result): if type(value) is str: try: value = six.text_type(value, 'utf8') diff --git a/stackalytics/processor/normalizer.py b/stackalytics/processor/normalizer.py index 83d086807..eb0d6d6a5 100644 --- a/stackalytics/processor/normalizer.py +++ b/stackalytics/processor/normalizer.py @@ -33,9 +33,9 @@ def _normalize_user(user): elif y["end_date"] == 0: return -1 else: - return cmp(x["end_date"], y["end_date"]) + return x["end_date"] - y["end_date"] - user['companies'].sort(cmp=end_date_comparator) + user['companies'].sort(key=utils.cmp_to_key(end_date_comparator)) user['user_id'] = user['launchpad_id'] diff --git a/stackalytics/processor/rcs.py b/stackalytics/processor/rcs.py index 86b1aba93..c2409da53 100644 --- a/stackalytics/processor/rcs.py +++ b/stackalytics/processor/rcs.py @@ -108,7 +108,7 @@ class Gerrit(Rcs): return False def _poll_reviews(self, project_organization, module, branch, - start_id=None, last_id=None, is_open=False, + start_id=0, last_id=0, is_open=False, grab_comments=False): sort_key = start_id diff --git a/stackalytics/processor/runtime_storage.py b/stackalytics/processor/runtime_storage.py index ce48e52f8..496a10b57 100644 --- a/stackalytics/processor/runtime_storage.py +++ b/stackalytics/processor/runtime_storage.py @@ -127,23 +127,29 @@ class MemcachedStorage(RuntimeStorage): return self.memcached.incr('user:count') def get_all_users(self): - for n in xrange(0, self.get_by_key('user:count') + 1): + for n in six.moves.range(0, self.get_by_key('user:count') + 1): user = self.get_by_key('user:%s' % n) if user: yield user def get_by_key(self, key): - return self.memcached.get(key.encode('utf8')) + if six.PY2: + key = key.encode('utf8') + return self.memcached.get(key) def set_by_key(self, key, value): - if not self.memcached.set(key.encode('utf8'), value): + if six.PY2: + key = key.encode('utf8') + if not self.memcached.set(key, value): LOG.critical('Failed to store data in memcached: ' 'key %(key)s, value %(value)s', {'key': key, 'value': value}) raise Exception('Memcached set failed') def delete_by_key(self, key): - if not self.memcached.delete(key.encode('utf8')): + if six.PY2: + key = key.encode('utf8') + if not self.memcached.delete(key): LOG.critical('Failed to delete data from memcached: key %s', key) raise Exception('Memcached delete failed') diff --git a/stackalytics/processor/utils.py b/stackalytics/processor/utils.py index 3965e230f..dc66b2be5 100644 --- a/stackalytics/processor/utils.py +++ b/stackalytics/processor/utils.py @@ -15,14 +15,13 @@ import cgi import datetime +import gzip import json import re import time import iso8601 import six -from six.moves.urllib import parse -from six.moves.urllib import request from stackalytics.openstack.common import log as logging @@ -89,8 +88,8 @@ def check_email_validity(email): def read_uri(uri): try: - fd = request.urlopen(uri) - raw = fd.read() + fd = six.moves.urllib.request.urlopen(uri) + raw = fd.read().decode('utf8') fd.close() return raw except Exception as e: @@ -106,12 +105,50 @@ def read_json_from_uri(uri): {'error': e, 'uri': uri}) +def gzip_decompress(content): + if six.PY3: + return gzip.decompress(content) + else: + gzip_fd = gzip.GzipFile(fileobj=six.moves.StringIO.StringIO(content)) + return gzip_fd.read() + + +def cmp_to_key(mycmp): # ported from python 3 + """Convert a cmp= function into a key= function.""" + class K(object): + __slots__ = ['obj'] + + def __init__(self, obj): + self.obj = obj + + def __lt__(self, other): + return mycmp(self.obj, other.obj) < 0 + + def __gt__(self, other): + return mycmp(self.obj, other.obj) > 0 + + def __eq__(self, other): + return mycmp(self.obj, other.obj) == 0 + + def __le__(self, other): + return mycmp(self.obj, other.obj) <= 0 + + def __ge__(self, other): + return mycmp(self.obj, other.obj) >= 0 + + def __ne__(self, other): + return mycmp(self.obj, other.obj) != 0 + + __hash__ = None + return K + + def make_range(start, stop, step): last_full = stop - ((stop - start) % step) - for i in six.moves.xrange(start, last_full, step): - yield six.moves.xrange(i, i + step) + for i in six.moves.range(start, last_full, step): + yield six.moves.range(i, i + step) if stop > last_full: - yield six.moves.xrange(last_full, stop) + yield six.moves.range(last_full, stop) def store_user(runtime_storage_inst, user): @@ -200,7 +237,7 @@ def add_index(sequence, start=1, item_filter=lambda x: True): def safe_encode(s): - return parse.quote(s.encode('utf-8')) + return six.moves.urllib.parse.quote(s.encode('utf-8')) def make_module_group(module_group_id, name=None, modules=None, tag='module'): diff --git a/stackalytics/processor/vcs.py b/stackalytics/processor/vcs.py index 14e369747..b726f207a 100644 --- a/stackalytics/processor/vcs.py +++ b/stackalytics/processor/vcs.py @@ -190,18 +190,18 @@ class Git(Vcs): try: output = sh.git('log', '--pretty=' + GIT_LOG_FORMAT, '--shortstat', '-M', '--no-merges', commit_range, _tty_out=False, - _decode_errors='ignore') + _decode_errors='ignore', _encoding='utf8') except sh.ErrorReturnCode as e: LOG.error('Unable to get log of git repo %s. Ignore it', self.repo['uri']) LOG.exception(e) return - for rec in re.finditer(GIT_LOG_PATTERN, str(output)): + for rec in re.finditer(GIT_LOG_PATTERN, six.text_type(output)): i = 1 commit = {} for param in GIT_LOG_PARAMS: - commit[param[0]] = six.text_type(rec.group(i), 'utf8') + commit[param[0]] = rec.group(i) i += 1 if not utils.check_email_validity(commit['author_email']): diff --git a/tests/unit/test_config_files.py b/tests/unit/test_config_files.py index 3e1c6d9dc..427f5d1fd 100644 --- a/tests/unit/test_config_files.py +++ b/tests/unit/test_config_files.py @@ -13,9 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools import json import jsonschema +import six import testtools @@ -24,17 +26,25 @@ class TestConfigFiles(testtools.TestCase): super(TestConfigFiles, self).setUp() def _read_file(self, file_name): - with open(file_name, 'r') as content_file: + if six.PY3: + opener = functools.partial(open, encoding='utf8') + else: + opener = open + with opener(file_name, 'r') as content_file: content = content_file.read() return json.loads(content) def _verify_ordering(self, array, key, msg): - sorted_array = sorted(array, key=key) - diff_msg = None - for i in range(len(array)): - if array[i] != sorted_array[i]: - diff_msg = ('First differing element %s:\n%s\n%s' % - (i, array[i], sorted_array[i])) + comparator = lambda x, y: (x > y) - (x < y) + + diff_msg = '' + for i in range(len(array) - 1): + if comparator(key(array[i]), key(array[i + 1])) > 0: + diff_msg = ('Order fails at index %(index)s, ' + 'elements:\n%(first)s:\n%(second)s' % + {'index': i, 'first': array[i], + 'second': array[i + 1]}) + break if diff_msg: self.fail(msg + '\n' + diff_msg) diff --git a/tests/unit/test_record_processor.py b/tests/unit/test_record_processor.py index 8bd395c65..c8406dbe6 100644 --- a/tests/unit/test_record_processor.py +++ b/tests/unit/test_record_processor.py @@ -743,11 +743,11 @@ class TestRecordProcessor(testtools.TestCase): 'user_name': 'John Doe', 'emails': ['john_doe@ibm.com', 'john_doe@gmail.com'], 'companies': [{'company_name': 'IBM', 'end_date': 0}]} - self.assertEqual(user, utils.load_user( + self.assertUsersMatch(user, utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe')) - self.assertEqual(user, utils.load_user( + self.assertUsersMatch(user, utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe@gmail.com')) - self.assertEqual(user, utils.load_user( + self.assertUsersMatch(user, utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe@ibm.com')) def test_merge_users(self): @@ -866,10 +866,10 @@ class TestRecordProcessor(testtools.TestCase): 'companies': [{'company_name': '*independent', 'end_date': 0}]} runtime_storage_inst = record_processor_inst.runtime_storage_inst - self.assertEqual(user_1, utils.load_user(runtime_storage_inst, - 'john_doe')) - self.assertEqual(user_2, utils.load_user(runtime_storage_inst, - 'homer')) + self.assertUsersMatch(user_1, utils.load_user(runtime_storage_inst, + 'john_doe')) + self.assertUsersMatch(user_2, utils.load_user(runtime_storage_inst, + 'homer')) def test_process_commit_with_coauthors(self): record_processor_inst = self.make_record_processor( @@ -1337,6 +1337,16 @@ class TestRecordProcessor(testtools.TestCase): self.assertEqual(value, actual[key], 'Values for key %s do not match' % key) + def assertUsersMatch(self, expected, actual): + match = True + for key, value in six.iteritems(expected): + if key == 'emails': + match = (set(value) == set(actual[key])) + else: + match = (value == actual[key]) + + self.assertTrue(match, 'User %s should match %s' % (actual, expected)) + # Helpers def make_record_processor(self, users=None, companies=None, releases=None, @@ -1415,7 +1425,7 @@ def make_runtime_storage(users=None, companies=None, releases=None, return count def get_all_users(): - for n in six.moves.xrange( + for n in six.moves.range( 0, (runtime_storage_cache.get('user:count') or 0) + 1): u = runtime_storage_cache.get('user:%s' % n) if u: diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index de0d50531..d59a3e010 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -150,9 +150,12 @@ diff_stat: self.assertEqual(0, commits[4]['lines_deleted']) self.assertFalse('coauthor' in commits[4]) - self.assertEqual( - [{'author_name': 'Tupac Shakur', - 'author_email': 'tupac.shakur@openstack.com'}, - {'author_name': 'Bob Dylan', - 'author_email': 'bob.dylan@openstack.com'}], + self.assertIn( + {'author_name': 'Tupac Shakur', + 'author_email': 'tupac.shakur@openstack.com'}, + commits[5]['coauthor']) + + self.assertIn( + {'author_name': 'Bob Dylan', + 'author_email': 'bob.dylan@openstack.com'}, commits[5]['coauthor']) diff --git a/tox.ini b/tox.ini index 23d758858..782be7f48 100644 --- a/tox.ini +++ b/tox.ini @@ -21,13 +21,17 @@ deps = -r{toxinidir}/requirements-py3.txt -r{toxinidir}/test-requirements.txt # to be removed once all tests passed commands = python -m testtools.run \ - tests.unit.test_utils \ + tests.unit.test_config_files \ + tests.unit.test_default_data_processor \ tests.unit.test_mps \ - tests.unit.test_record_processor + tests.unit.test_mls \ + tests.unit.test_record_processor \ + tests.unit.test_utils \ + tests.unit.test_vcs [testenv:pep8] commands = flake8 - {toxinidir}/tools/requirements_style_check.sh requirements.txt test-requirements.txt + {toxinidir}/tools/requirements_style_check.sh requirements.txt requirements-py3.txt test-requirements.txt distribute = false [testenv:venv]