From 744489121819393f2d5dd5b79793698fa029a7a6 Mon Sep 17 00:00:00 2001 From: Keiichi Hikita Date: Fri, 13 Apr 2018 11:35:05 +0900 Subject: [PATCH] Fix tox settings and relevant bugs I found problem as follows in current tox.ini - test-requirements can not be read correctly - some os level command is not in white_list - one rst file does not follow rst syntac - fix lack of gitreview file Purpose of this review request is fixing above problems. Change-Id: I1d2b29bbc7cdc506f5187c2ef6ee03fd2b908f58 --- .gitreview | 5 +++++ bindep.txt | 6 +++++ doc/requirements.txt | 8 +++---- doc/source/install/build_rpm_packages.rst | 21 +++++++++++++----- requirements.txt | 10 +++++---- setup.py | 13 ++++++----- slogging/access_processor.py | 11 +++++---- slogging/common.py | 12 ++++++++++ slogging/db_stats_collector.py | 5 ++--- slogging/log_processor.py | 5 ++--- test-requirements.txt | 7 +++--- test/speed/test_cidr_speed.py | 26 +++++++++++----------- tox.ini | 27 +++++------------------ 13 files changed, 87 insertions(+), 69 deletions(-) create mode 100644 .gitreview create mode 100644 bindep.txt diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..8ece3a8 --- /dev/null +++ b/.gitreview @@ -0,0 +1,5 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=openstack/slogging.git +defaultbranch=master diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000..30f7581 --- /dev/null +++ b/bindep.txt @@ -0,0 +1,6 @@ +# This is a cross-platform list tracking distribution packages needed by tests; +# see http://docs.openstack.org/infra/bindep/ for additional information. + +liberasurecode-dev [platform:dpkg] +# There's no library in CentOS 7 but Fedora and openSUSE have it. +liberasurecode-devel [platform:rpm !platform:centos] diff --git a/doc/requirements.txt b/doc/requirements.txt index c75634f..b2b61eb 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -2,7 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -openstackdocstheme>=1.18.1 # Apache-2.0 -sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD -reno>=2.5.0 # Apache-2.0 -sphinxcontrib-httpdomain>=1.3.0 # BSD +openstackdocstheme # Apache-2.0 +sphinx!=1.6.6,!=1.6.7 # BSD +reno # Apache-2.0 +sphinxcontrib-httpdomain # BSD diff --git a/doc/source/install/build_rpm_packages.rst b/doc/source/install/build_rpm_packages.rst index 7540bc5..9e1357e 100644 --- a/doc/source/install/build_rpm_packages.rst +++ b/doc/source/install/build_rpm_packages.rst @@ -4,18 +4,29 @@ How to Build to RPM Packages #. Make sure you have rpm-build installed:: - - on Fedora/CentOS/RedHat:: - sudo yum install rpm-build - - on OpenSUSE:: - sudo zypper install rpm-build +Fedora/CentOS/RedHat +~~~~~~~~~~~~~~~~~~~~ -#. Thsn type following command at the top of slogging directory:: + +#. Type following command at the top of slogging directory:: sudo python setup.py bdist_rpm #. Check if the RPM package has built:: ls dist/slogging-[slogging-version] + + +OpenSUSE +~~~~~~~~ + +#. Type following command at the top of slogging directory:: + + sudo zypper install rpm-build + +#. Check if the RPM package has built:: + + ls dist/slogging-[slogging-version] diff --git a/requirements.txt b/requirements.txt index cd90c7d..fe0e3b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ -pbr!=2.1.0,>=2.0.0 # Apache-2.0 -iptools>=0.4 -tzlocal -swift>=2.14.0 +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +pbr!=2.1.0 # Apache-2.0 +pytz +netaddr diff --git a/setup.py b/setup.py index 2536b50..566d844 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ -#!/usr/bin/python -# Copyright (c) 2010-2011 OpenStack, LLC. +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,15 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 try: import multiprocessing # noqa except ImportError: pass - setuptools.setup( - setup_requires=['pbr'], - pbr=True, -) + setup_requires=['pbr>=2.0.0'], + pbr=True) diff --git a/slogging/access_processor.py b/slogging/access_processor.py index efb6b95..8ec3ab8 100644 --- a/slogging/access_processor.py +++ b/slogging/access_processor.py @@ -16,20 +16,19 @@ from datetime import datetime import pytz from slogging import common -from tzlocal import get_localzone from urllib import unquote from urlparse import urlparse # conditionalize the return_ips method based on whether or not iptools # is present in the system. Without iptools, you will lack CIDR support. try: - from iptools import IpRangeList + from netaddr import IPSet CIDR_support = True def return_ips(conf, conf_tag): - return set(k for k in IpRangeList(*[ + return IPSet([ x.strip() for x in conf.get(conf_tag, '').split(',') - if x.strip()])) + if x.strip()]) def sanitize_ips(line_data): for x in ['lb_ip', 'client_ip', 'log_source']: @@ -48,7 +47,7 @@ from swift.common import utils month_map = '_ Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split() LISTING_PARAMS = set( 'path limit format delimiter marker end_marker prefix'.split()) -local_zone = get_localzone() +local_zone = common.get_localzone() class AccessLogProcessor(object): @@ -63,7 +62,7 @@ class AccessLogProcessor(object): self.warn_percent = float(conf.get('warn_percent', '0.8')) self.logger = utils.get_logger(conf, log_route='access-processor') self.time_zone = common.get_time_zone(conf, self.logger, 'time_zone', - str(local_zone)) + local_zone) def log_line_parser(self, raw_log): '''given a raw access log line, return a dict of the good parts''' diff --git a/slogging/common.py b/slogging/common.py index 65b78e8..bec060a 100644 --- a/slogging/common.py +++ b/slogging/common.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import pytz @@ -44,3 +45,14 @@ def get_format_type(conf, logger, key, default): logger.warning(msg) format_type = default return format_type + + +def get_localzone(): + """Get time_zone value.""" + try: + str_time_zone = '/'.join( + os.readlink('/etc/localtime').split('/')[-2:]) + except Exception: + str_time_zone = 'UTC' + + return str_time_zone diff --git a/slogging/db_stats_collector.py b/slogging/db_stats_collector.py index b2646f4..9cb2cb8 100644 --- a/slogging/db_stats_collector.py +++ b/slogging/db_stats_collector.py @@ -29,11 +29,10 @@ from swift.container.backend import ContainerBroker from swift.container.server import DATADIR as container_server_data_dir import time -from tzlocal import get_localzone import urllib -local_zone = get_localzone() +local_zone = common.get_localzone() class DatabaseStatsCollector(Daemon): @@ -57,7 +56,7 @@ class DatabaseStatsCollector(Daemon): self.logger = utils.get_logger(stats_conf, log_route='%s-stats' % stats_type) self.time_zone = common.get_time_zone(stats_conf, self.logger, - 'time_zone', str(local_zone)) + 'time_zone', local_zone) def run_once(self, *args, **kwargs): self.logger.info(_("Gathering %s stats") % self.stats_type) diff --git a/slogging/log_processor.py b/slogging/log_processor.py index c24cbd1..56fd488 100644 --- a/slogging/log_processor.py +++ b/slogging/log_processor.py @@ -25,11 +25,10 @@ from slogging import log_common from swift.common.daemon import Daemon from swift.common import utils import time -from tzlocal import get_localzone now = datetime.datetime.now -local_zone = get_localzone() +local_zone = common.get_localzone() class LogProcessor(log_common.LogProcessorCommon): @@ -137,7 +136,7 @@ class LogProcessorDaemon(Daemon): self._keylist_mapping = None self.processed_files_filename = 'processed_files.pickle.gz' self.time_zone = common.get_time_zone(c, self.logger, 'time_zone', - str(local_zone)) + local_zone) self.format_type = common.get_format_type(c, self.logger, 'format_type', 'csv') diff --git a/test-requirements.txt b/test-requirements.txt index fb90239..871904b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,11 +3,12 @@ # process, which may cause wedges in the gate later. # Hacking already pins down pep8, pyflakes and flake8 -hacking>=0.11.0,<0.12 # Apache-2.0 -coverage>=3.6 # Apache-2.0 +hacking<0.12,>=0.11.0 # Apache-2.0 +coverage!=4.4 # Apache-2.0 nose # LGPL nosexcover # BSD -nosehtmloutput>=0.0.3 # Apache-2.0 +nosehtmloutput # Apache-2.0 +flake8<=2.5.5 # Security checks bandit>=1.1.0 # Apache-2.0 diff --git a/test/speed/test_cidr_speed.py b/test/speed/test_cidr_speed.py index 50e36a1..40e1c0d 100644 --- a/test/speed/test_cidr_speed.py +++ b/test/speed/test_cidr_speed.py @@ -15,7 +15,7 @@ import datetime -import iptools +import netaddr import unittest @@ -25,15 +25,15 @@ class TestAccessProcessorSpeed(unittest.TestCase): line = 'Sep 16 20:00:02 srv testsrv 192.%s.119.%s - ' \ '16/Sep/2012/20/00/02 GET /v1/a/c/o HTTP/1.0 ' \ '200 - StaticWeb - - 17005 - txn - 0.0095 -' - ips1 = iptools.IpRangeList(*[x.strip() for x in - '127.0.0.1,192.168/16,10/24'.split(',') - if x.strip()]) - ips2 = iptools.IpRangeList(*[x.strip() for x in - '172.168/16,10/30'.split(',') - if x.strip()]) - ips3 = iptools.IpRangeList(*[x.strip() for x in - '127.0.0.1,11/24'.split(',') - if x.strip()]) + ips1 = netaddr.IPSet([x.strip() for x in + '127.0.0.1,192.168/16,10/24'.split(',') + if x.strip()]) + ips2 = netaddr.IPSet([x.strip() for x in + '172.168/16,10/30'.split(',') + if x.strip()]) + ips3 = netaddr.IPSet([x.strip() for x in + '127.0.0.1,11/24'.split(',') + if x.strip()]) orig_start = datetime.datetime.utcnow() hit = 0 @@ -49,9 +49,9 @@ class TestAccessProcessorSpeed(unittest.TestCase): self.assertEqual(hit, 255) # now, let's check the speed with sets - set1 = set(k for k in ips1) - set2 = set(k for k in ips2) - set3 = set(k for k in ips3) + set1 = ips1 + set2 = ips2 + set3 = ips3 new_start = datetime.datetime.utcnow() hit = 0 diff --git a/tox.ini b/tox.ini index 5b2d40c..7b9af2c 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,7 @@ envlist = py27,docs,pep8 [testenv] +usedevelop = True setenv = VIRTUAL_ENV={envdir} NOSE_WITH_OPENSTACK=1 NOSE_OPENSTACK_COLOR=1 @@ -13,55 +14,37 @@ deps = -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/re commands= py27: {[unit_tests]commands} py35: {[unit_tests]commands} - +whitelist_externals = find + rm [testenv:pep8] deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz + -r{toxinidir}/test-requirements.txt commands = flake8 {posargs} - -[testenv:cover] -setenv = VIRTUAL_ENV={envdir} - NOSE_WITH_COVERAGE=1 - NOSE_COVER_HTML_DIR={toxinidir}/cover - NOSE_COVER_HTML=1 - NOSE_COVER_ERASE=1 -commands = - {[unit_tests]commands} - - [testenv:py27] basepython = python2.7 deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz + -r{toxinidir}/test-requirements.txt setenv = SWIFT_TEST_CONFIG_FILE={toxinidir}/test/sample.conf SWIFT_PROXY_TEST_CONFIG_FILE={toxinidir}/test/sample.proxy-server.conf commands = {[unit_tests]commands} - [unit_tests] -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt commands = find . -type f -name "*.py[c|o]" -delete find . -type d -name "__pycache__" -delete nosetests {posargs:test/unit} - -[testenv:venv] -commands = {posargs} - [flake8] exclude = .venv,.git,.tox,dist,*lib/python*,*egg,build,dash_template max-complexity = 20 -import-order-style = pep8 - [testenv:docs] deps = http://tarballs.openstack.org/swift/swift-stable-queens.tar.gz -r{toxinidir}/doc/requirements.txt commands = python setup.py build_sphinx - [doc8] # File extensions to check extensions = .rst, .yaml