From 7ae701120a8940997bc4f04ea513f2b3c7e65f09 Mon Sep 17 00:00:00 2001 From: He Jie Xu Date: Fri, 30 Nov 2012 16:03:41 +0800 Subject: [PATCH] Add script for checking i18n message Part of bp make-string-localizable usage: tox -e i18n tools/check_i18n.py: used check i18n message for one file. tools/check_i18n_test_case.txt: test case of check_i18n.py. run test case with cmd: $ ./tools/check_i18n.py ./tools/check_i18n_test_case.txt -d Change-Id: I2c383b7bb11ab3bdb8e3bb3b887342b1225840ac --- tools/check_i18n.py | 154 +++++++++++++++++++++++++++++++++ tools/check_i18n_test_case.txt | 67 ++++++++++++++ tools/i18n_cfg.py | 98 +++++++++++++++++++++ tox.ini | 3 + 4 files changed, 322 insertions(+) create mode 100644 tools/check_i18n.py create mode 100644 tools/check_i18n_test_case.txt create mode 100644 tools/i18n_cfg.py diff --git a/tools/check_i18n.py b/tools/check_i18n.py new file mode 100644 index 0000000000..43a4f9bc3f --- /dev/null +++ b/tools/check_i18n.py @@ -0,0 +1,154 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 OpenStack LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import compiler +import imp +import os.path +import sys + + +def is_localized(node): + """ Check message wrapped by _() """ + if isinstance(node.parent, compiler.ast.CallFunc): + if isinstance(node.parent.node, compiler.ast.Name): + if node.parent.node.name == '_': + return True + return False + + +class ASTWalker(compiler.visitor.ASTVisitor): + + def default(self, node, *args): + for child in node.getChildNodes(): + child.parent = node + compiler.visitor.ASTVisitor.default(self, node, *args) + + +class Visitor(object): + + def __init__(self, filename, i18n_msg_predicates, + msg_format_checkers, debug): + self.filename = filename + self.debug = debug + self.error = 0 + self.i18n_msg_predicates = i18n_msg_predicates + self.msg_format_checkers = msg_format_checkers + with open(filename) as f: + self.lines = f.readlines() + + def visitConst(self, node): + if not isinstance(node.value, str): + return + + if is_localized(node): + for (checker, msg) in self.msg_format_checkers: + if checker(node): + print >> sys.stderr, ( + '%s:%d %s: %s' % + (self.filename, node.lineno, + self.lines[node.lineno - 1][:-1], + "Error: %s" % msg)) + self.error = 1 + return + if debug: + print ('%s:%d %s: %s' % + (self.filename, node.lineno, + self.lines[node.lineno - 1][:-1], + "Pass")) + else: + for (predicate, action, msg) in self.i18n_msg_predicates: + if predicate(node): + if action == 'skip': + if debug: + print ('%s:%d %s: %s' % + (self.filename, node.lineno, + self.lines[node.lineno - 1][:-1], + "Pass")) + return + elif action == 'error': + print >> sys.stderr, ( + '%s:%d %s: %s' % + (self.filename, node.lineno, + self.lines[node.lineno - 1][:-1], + "Error: %s" % msg)) + self.error = 1 + return + elif action == 'warn': + print ('%s:%d %s: %s' % + (self.filename, node.lineno, + self.lines[node.lineno - 1][:-1], + "Warn: %s" % msg)) + return + print >> sys.stderr, 'Predicate with wrong action!' + + +def is_file_in_black_list(black_list, f): + for f in black_list: + if os.path.abspath(input_file).startswith( + os.path.abspath(f)): + return True + return False + + +def check_i18n(input_file, i18n_msg_predicates, msg_format_checkers, debug): + input_mod = compiler.parseFile(input_file) + v = compiler.visitor.walk(input_mod, + Visitor(input_file, + i18n_msg_predicates, + msg_format_checkers, + debug), + ASTWalker()) + return v.error + + +if __name__ == '__main__': + input_path = sys.argv[1] + cfg_path = sys.argv[2] + try: + cfg_mod = imp.load_source('', cfg_path) + except: + print >> sys.stderr, "Load cfg module failed" + sys.exit(1) + + i18n_msg_predicates = cfg_mod.i18n_msg_predicates + msg_format_checkers = cfg_mod.msg_format_checkers + black_list = cfg_mod.file_black_list + + debug = False + if len(sys.argv) > 3: + if sys.argv[3] == '-d': + debug = True + + if os.path.isfile(input_path): + sys.exit(check_i18n(input_path, + i18n_msg_predicates, + msg_format_checkers, + debug)) + + error = 0 + for dirpath, dirs, files in os.walk(input_path): + for f in files: + if not f.endswith('.py'): + continue + input_file = os.path.join(dirpath, f) + if is_file_in_black_list(black_list, input_file): + continue + if check_i18n(input_file, + i18n_msg_predicates, + msg_format_checkers, + debug): + error = 1 + sys.exit(error) diff --git a/tools/check_i18n_test_case.txt b/tools/check_i18n_test_case.txt new file mode 100644 index 0000000000..3d1391d945 --- /dev/null +++ b/tools/check_i18n_test_case.txt @@ -0,0 +1,67 @@ +# test-case for check_i18n.py +# python check_i18n.py check_i18n.txt -d + +# message format checking +# capital checking +msg = _("hello world, error") +msg = _("hello world_var, error") +msg = _('file_list xyz, pass') +msg = _("Hello world, pass") + +# format specifier checking +msg = _("Hello %s world %d, error") +msg = _("Hello %s world, pass") +msg = _("Hello %(var1)s world %(var2)s, pass") + +# message has been localized +# is_localized +msg = _("Hello world, pass") +msg = _("Hello world, pass") % var +LOG.debug(_('Hello world, pass')) +LOG.info(_('Hello world, pass')) +raise x.y.Exception(_('Hello world, pass')) +raise Exception(_('Hello world, pass')) + +# message need be localized +# is_log_callfunc +LOG.debug('hello world, error') +LOG.debug('hello world, error' % xyz) +sys.append('hello world, warn') + +# is_log_i18n_msg_with_mod +LOG.debug(_('Hello world, error') % xyz) + +# default warn +msg = 'hello world, warn' +msg = 'hello world, warn' % var + +# message needn't be localized +# skip only one word +msg = '' +msg = "hello,pass" + +# skip dict +msg = {'hello world, pass': 1} + +# skip list +msg = ["hello world, pass"] + +# skip subscript +msg['hello world, pass'] + +# skip xml marker +msg = ", pass" + +# skip sql statement +msg = "SELECT * FROM xyz WHERE hello=1, pass" +msg = "select * from xyz, pass" + +# skip add statement +msg = 'hello world' + e + 'world hello, pass' + +# skip doc string +""" +Hello world, pass +""" +class Msg: + pass diff --git a/tools/i18n_cfg.py b/tools/i18n_cfg.py new file mode 100644 index 0000000000..23894a938a --- /dev/null +++ b/tools/i18n_cfg.py @@ -0,0 +1,98 @@ +import compiler +import re + + +def is_log_callfunc(n): + """ LOG.xxx('hello %s' % xyz) and LOG('hello') """ + if isinstance(n.parent, compiler.ast.Mod): + n = n.parent + if isinstance(n.parent, compiler.ast.CallFunc): + if isinstance(n.parent.node, compiler.ast.Getattr): + if isinstance(n.parent.node.getChildNodes()[0], + compiler.ast.Name): + if n.parent.node.getChildNodes()[0].name == 'LOG': + return True + return False + + +def is_log_i18n_msg_with_mod(n): + """ LOG.xxx("Hello %s" % xyz) should be LOG.xxx("Hello %s", xyz) """ + if not isinstance(n.parent.parent, compiler.ast.Mod): + return False + n = n.parent.parent + if isinstance(n.parent, compiler.ast.CallFunc): + if isinstance(n.parent.node, compiler.ast.Getattr): + if isinstance(n.parent.node.getChildNodes()[0], + compiler.ast.Name): + if n.parent.node.getChildNodes()[0].name == 'LOG': + return True + return False + + +def is_wrong_i18n_format(n): + """ Check _('hello %s' % xyz) """ + if isinstance(n.parent, compiler.ast.Mod): + n = n.parent + if isinstance(n.parent, compiler.ast.CallFunc): + if isinstance(n.parent.node, compiler.ast.Name): + if n.parent.node.name == '_': + return True + return False + + +""" +Used for check message need be localized or not. +(predicate_func, action, message) +""" +i18n_msg_predicates = [ + # Skip ['hello world', 1] + (lambda n: isinstance(n.parent, compiler.ast.List), 'skip', ''), + # Skip {'hellow world', 1} + (lambda n: isinstance(n.parent, compiler.ast.Dict), 'skip', ''), + # Skip msg['hello world'] + (lambda n: isinstance(n.parent, compiler.ast.Subscript), 'skip', ''), + # Skip doc string + (lambda n: isinstance(n.parent, compiler.ast.Discard), 'skip', ''), + # Skip msg = "hello", in normal, message should more than one word + (lambda n: len(n.value.strip().split(' ')) <= 1, 'skip', ''), + # Skip msg = 'hello world' + vars + 'world hello' + (lambda n: isinstance(n.parent, compiler.ast.Add), 'skip', ''), + # Skip xml markers msg = "" + (lambda n: len(re.compile("").findall(n.value)) > 0, 'skip', ''), + # Skip sql statement + (lambda n: len( + re.compile("^SELECT.*FROM", flags=re.I).findall(n.value)) > 0, + 'skip', ''), + # LOG.xxx() + (is_log_callfunc, 'error', 'Message must be localized'), + # _('hello %s' % xyz) should be _('hello %s') % xyz + (is_wrong_i18n_format, 'error', + ("Message format was wrong, _('hello %s' % xyz) " + "should be _('hello %s') % xyz")), + # default + (lambda n: True, 'warn', 'Message might need localized') +] + + +""" +Used for checking message format. (checker_func, message) +""" +msg_format_checkers = [ + # If message contain more than on format specifier, it should use + # mapping key + (lambda n: len(re.compile("%[bcdeEfFgGnosxX]").findall(n.value)) > 1, + "The message shouldn't contain more than one format specifier"), + # Check capital + (lambda n: n.value.split(' ')[0].count('_') == 0 and + n.value[0].isalpha() and + n.value[0].islower(), + "First letter must be capital"), + (is_log_i18n_msg_with_mod, + 'LOG.xxx("Hello %s" % xyz) should be LOG.xxx("Hello %s", xyz)') +] + + +file_black_list = ["./quantum/plugins/cisco/tests/unit", + "./quantum/tests/unit", + "./quantum/openstack", + "./quantum/plugins/bigswitch/tests"] diff --git a/tox.ini b/tox.ini index 937470e0ed..e3b17ecdb1 100644 --- a/tox.ini +++ b/tox.ini @@ -25,6 +25,9 @@ commands = pep8 --repeat --show-source --ignore=E125 --exclude=.venv,.tox,dist,doc,openstack,*egg . pep8 --repeat --show-source --ignore=E125 --filename=quantum* bin +[testenv:i18n] +commands = python ./tools/check_i18n.py ./quantum ./tools/i18n_cfg.py + [testenv:cover] setenv = NOSE_WITH_COVERAGE=1