Add tox configuration and fixe flake8 errors

This change adds tests structure.

Change-Id: I62f6c238de5c55b7673ae8b38a6ededdd77f4d4b

.stestr.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+test_path=tests/
+top_dir=./

.zuul.yaml

@@ -56,9 +56,15 @@
     check:
       jobs:
         - zuul-registry-build-image
+        - tox-pep8
+        - tox-py37:
+            nodeset: fedora-latest
     gate:
       jobs:
         - zuul-registry-upload-image
+        - tox-pep8
+        - tox-py37:
+            nodeset: fedora-latest
     promote:
       jobs:
         - zuul-registry-promote-image

test-requirements.txt

@@ -0,0 +1,2 @@
+flake8
+stestr

tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.

tests/test_filesystem.py

@@ -0,0 +1,24 @@
+# Copyright 2019 Red Hat, Inc.
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software.  If not, see <http://www.gnu.org/licenses/>.
+
+import testtools
+
+from zuul_registry.filesystem import FilesystemDriver
+
+
+class TestFilesystemDriver(testtools.TestCase):
+    def test_list_objects(self):
+        driver = FilesystemDriver({'root': '.'})
+        print(driver.list_objects("."))

tox.ini

@@ -0,0 +1,29 @@
+[tox]
+minversion = 3.2
+skipsdist = True
+envlist = pep8,py37
+
+[testenv]
+basepython = python3
+usedevelop = True
+deps =
+  -r{toxinidir}/requirements.txt
+  -r{toxinidir}/test-requirements.txt
+commands =
+  stestr run {posargs}
+  stestr slowest
+
+[testenv:pep8]
+install_command = pip install {opts} {packages}
+commands =
+  flake8 {posargs}
+
+[testenv:venv]
+commands = {posargs}
+
+[flake8]
+# These are ignored intentionally in zuul projects;
+# please don't submit patches that solely correct them or enable them.
+ignore = E124,E125,E129,E252,E402,E741,H,W503,W504
+show-source = True
+exclude = .venv,.tox,dist,doc,build,*.egg,node_modules

zuul_registry/filesystem.py

@@ -14,16 +14,15 @@
 # along with this software.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import time
 
 from . import storageutils
 
+
 class FilesystemDriver(storageutils.StorageDriver):
     def __init__(self, conf):
         self.root = conf['root']
 
     def list_objects(self, path):
-        now = time.time()
         path = os.path.join(self.root, path)
         if not os.path.isdir(path):
             return []
@@ -88,4 +87,5 @@ class FilesystemDriver(storageutils.StorageDriver):
             chunk_path = os.path.join(self.root, chunk_path)
             os.unlink(chunk_path)
 
+
 Driver = FilesystemDriver

zuul_registry/main.py

@@ -31,6 +31,7 @@ DRIVERS = {
     'swift': swift.Driver,
 }
 
+
 class Authorization:
     def __init__(self, users):
         self.ro = {}
@@ -52,6 +53,7 @@ class Authorization:
             return False
         return store[user] == password
 
+
 class RegistryAPI:
     """Registry API server.
 
@@ -125,9 +127,10 @@ class RegistryAPI:
         method = cherrypy.request.method
         uuid = self.storage.start_upload(namespace)
         self.log.info('Start upload %s %s uuid %s digest %s',
-                       method, repository, uuid, digest)
+                      method, repository, uuid, digest)
         res = cherrypy.response
-        res.headers['Location'] = '/v2/%s/blobs/uploads/%s' % (repository, uuid)
+        res.headers['Location'] = '/v2/%s/blobs/uploads/%s' % (
+            repository, uuid)
         res.headers['Docker-Upload-UUID'] = uuid
         res.headers['Range'] = '0-0'
         res.status = '202 Accepted'
@@ -140,11 +143,13 @@ class RegistryAPI:
         old_length, new_length = self.storage.upload_chunk(
             namespace, uuid, cherrypy.request.body)
         res = cherrypy.response
-        res.headers['Location'] = '/v2/%s/blobs/uploads/%s' % (repository, uuid)
+        res.headers['Location'] = '/v2/%s/blobs/uploads/%s' % (
+            repository, uuid)
         res.headers['Docker-Upload-UUID'] = uuid
         res.headers['Range'] = '0-%s' % (new_length,)
         res.status = '204 No Content'
-        self.log.info('Finish Upload chunk %s %s %s', repository, uuid, new_length)
+        self.log.info(
+            'Finish Upload chunk %s %s %s', repository, uuid, new_length)
 
     @cherrypy.expose
     @cherrypy.config(**{'tools.auth_basic.checkpassword': require_write})
@@ -217,8 +222,10 @@ class RegistryAPI:
         self.log.error('Manifest %s %s not found', repository, ref)
         return self.not_found()
 
+
 class RegistryServer:
     log = logging.getLogger("registry.server")
+
     def __init__(self, config_path):
         self.log.info("Loading config from %s", config_path)
         self._load_config(config_path)
@@ -297,6 +304,7 @@ class RegistryServer:
     def prune(self):
         self.store.prune()
 
+
 def main():
     parser = argparse.ArgumentParser(
         description='Zuul registry server')
@@ -321,7 +329,7 @@ def main():
     logging.getLogger("urllib3").setLevel(logging.DEBUG)
     logging.getLogger("stevedore").setLevel(logging.INFO)
     logging.getLogger("openstack").setLevel(logging.DEBUG)
-    #cherrypy.log.error_log.propagate = False
+    # cherrypy.log.error_log.propagate = False
 
     s = RegistryServer(args.config)
     if args.command == 'serve':

zuul_registry/storage.py

@@ -23,6 +23,7 @@ import threading
 import time
 from uuid import uuid4
 
+
 class UploadRecord:
     """Information about an upload.
 
@@ -65,16 +66,19 @@ class UploadRecord:
         data = json.loads(data.decode('utf8'))
         self.chunks = data['chunks']
         hash_state = data['hash_state']
-        hash_state['md_data'] = base64.decodebytes(hash_state['md_data'].encode('ascii'))
+        hash_state['md_data'] = base64.decodebytes(
+            hash_state['md_data'].encode('ascii'))
         self.hasher.__setstate__(hash_state)
 
     def dump(self):
         hash_state = self.hasher.__getstate__()
-        hash_state['md_data'] = base64.encodebytes(hash_state['md_data']).decode('ascii')
+        hash_state['md_data'] = base64.encodebytes(
-        data = dict(chunks = self.chunks,
+            hash_state['md_data']).decode('ascii')
-                    hash_state = hash_state)
+        data = dict(chunks=self.chunks,
+                    hash_state=hash_state)
         return json.dumps(data).encode('utf8')
 
+
 class UploadStreamer:
     """Stream an upload to the object storage.
 
@@ -96,6 +100,7 @@ class UploadStreamer:
                 break
             yield d
 
+
 class Storage:
     """Storage abstraction layer.
 
@@ -145,7 +150,6 @@ class Storage:
 
         """
         uuid = uuid4().hex
-        path = os.path.join(namespace, 'uploads', uuid, 'metadata')
         upload = UploadRecord()
         self._update_upload(namespace, uuid, upload)
         return uuid
@@ -198,7 +202,7 @@ class Storage:
         t.join()
         upload.chunks.append(dict(size=size))
         self._update_upload(namespace, uuid, upload)
-        return upload.size-size, upload.size
+        return upload.size - size, upload.size
 
     def store_upload(self, namespace, uuid, digest):
         """Complete an upload.
@@ -215,7 +219,7 @@ class Storage:
         # Move the chunks into the blob dir to get them out of the
         # uploads dir.
         chunks = []
-        for i in range(1, upload.count+1):
+        for i in range(1, upload.count + 1):
             src_path = os.path.join(namespace, 'uploads', uuid, str(i))
             dst_path = os.path.join(namespace, 'blobs', digest, str(i))
             chunks.append(dst_path)
@@ -271,7 +275,8 @@ class Storage:
         self.log.debug('Get layers %s', path)
         data = self.backend.get_object(path)
         manifest = json.loads(data)
-        target = manifest.get('application/vnd.docker.distribution.manifest.v2+json')
+        target = manifest.get(
+            'application/vnd.docker.distribution.manifest.v2+json')
         layers = []
         if not target:
             self.log.debug('Unknown manifest %s', path)

zuul_registry/storageutils.py

@@ -15,6 +15,7 @@
 
 from abc import ABCMeta, abstractmethod
 
+
 class ObjectInfo:
     def __init__(self, path, name, ctime, isdir):
         self.path = path
@@ -22,6 +23,7 @@ class ObjectInfo:
         self.ctime = ctime
         self.isdir = isdir
 
+
 class StorageDriver(metaclass=ABCMeta):
     """Base class for storage drivers.
 

zuul_registry/swift.py

@@ -27,6 +27,7 @@ from . import storageutils
 
 POST_ATTEMPTS = 3
 
+
 def retry_function(func):
     for attempt in range(1, POST_ATTEMPTS + 1):
         try:
@@ -40,6 +41,7 @@ def retry_function(func):
                 logging.exception("Error on attempt %d" % attempt)
                 time.sleep(attempt * 10)
 
+
 class SwiftDriver(storageutils.StorageDriver):
     log = logging.getLogger('registry.swift')
 
@@ -76,7 +78,8 @@ class SwiftDriver(storageutils.StorageDriver):
             else:
                 objpath = obj['name']
                 name = obj['name'].split('/')[-1]
-                ctime = dateutil.parser.parse(obj['last_modified']+'Z').timestamp()
+                ctime = dateutil.parser.parse(
+                    obj['last_modified'] + 'Z').timestamp()
                 isdir = False
             ret.append(storageutils.ObjectInfo(
                 objpath, name, ctime, isdir))
@@ -126,7 +129,7 @@ class SwiftDriver(storageutils.StorageDriver):
         dst = os.path.join(self.container_name, dst_path)
         retry_function(
             lambda: self.conn.session.request(
-                self.get_url(src_path)+"?multipart-manfest=get",
+                self.get_url(src_path) + "?multipart-manfest=get",
                 'COPY',
                 headers={'Destination': dst}
             ))
@@ -136,7 +139,7 @@ class SwiftDriver(storageutils.StorageDriver):
 
     def cat_objects(self, path, chunks):
         manifest = []
-        #TODO: Would it be better to move 1-chunk objects?
+        # TODO: Would it be better to move 1-chunk objects?
         for chunk_path in chunks:
             ret = retry_function(
                 lambda: self.conn.session.head(self.get_url(chunk_path)))
@@ -148,7 +151,8 @@ class SwiftDriver(storageutils.StorageDriver):
                              'size_bytes': ret.headers['Content-Length']})
         retry_function(lambda:
                        self.conn.session.put(
-                           self.get_url(path)+"?multipart-manifest=put",
+                           self.get_url(path) + "?multipart-manifest=put",
                            data=json.dumps(manifest)))
 
+
 Driver = SwiftDriver