From 3c7bcc0c7a7e58da5efa7cbd59cde3f56fe143fd Mon Sep 17 00:00:00 2001 From: Phil Bridges Date: Wed, 8 Jun 2016 15:53:51 -0500 Subject: [PATCH] Fix Gerrit gates and other build issues There are a few problems with the configuration in our source tree that causes the Jenkins gates to always fail, and things that might cause a problem later. This patch set will fix those and other problems, to bring us into a known valid state for future commits. Change-Id: Idf7a0ce5902c40985caa78390b01f9fc2dfbfcf4 --- .gitreview | 14 +++--- bin/swiftonhpss-nstool | 63 ++++++++++-------------- setup.py | 4 +- swiftonhpss/swift/common/hpssfs_ioctl.py | 40 +++++++++++++++ swiftonhpss/swift/common/utils.py | 1 - swiftonhpss/swift/obj/diskfile.py | 32 +++++++----- swiftonhpss/swift/obj/server.py | 50 +++++++++---------- tox.ini | 10 ++-- 8 files changed, 123 insertions(+), 91 deletions(-) create mode 100644 swiftonhpss/swift/common/hpssfs_ioctl.py diff --git a/.gitreview b/.gitreview index 6e4fd6e..d89c6df 100644 --- a/.gitreview +++ b/.gitreview @@ -1,8 +1,6 @@ -# TODO: get ourselves a nice and shiny CI system like this - -#[gerrit] -#host=review.openstack.org -#port=29418 -#project=openstack/swiftonfile.git -#defaultbranch=master -#defaultremote=gerrit +[gerrit] +host=review.openstack.org +port=29418 +project=openstack/swiftonhpss.git +defaultbranch=master +defaultremote=gerrit diff --git a/bin/swiftonhpss-nstool b/bin/swiftonhpss-nstool index c6525a5..3996f69 100755 --- a/bin/swiftonhpss-nstool +++ b/bin/swiftonhpss-nstool @@ -18,7 +18,6 @@ import sys import stat import os -import multiprocessing from pwd import getpwuid import logging import argparse @@ -87,11 +86,11 @@ def main(program_args): del password # Figure out what we're doing. - target_account, target_container = program_args.account,\ - program_args.container + target_account = program_args.account + target_container = program_args.container # Start doing it. - #pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()-1) + # pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()-1) # Multiprocessing does not play nicely with the lazy loading that # keystoneclient does, so let's not mess with it for now. @@ -227,7 +226,7 @@ def check_usage(): return parser.parse_args() -class Reconciler: +class Reconciler(object): def __init__(self, args): self._args = args @@ -436,7 +435,8 @@ class Reconciler: logging.exception("Putting container %s went wrong" % target_container) raise e - print "Reconciling container %s/%s" % (target_account, target_container) + print "Reconciling container %s/%s" % \ + (target_account, target_container) # Make sure those objects get added into the Swift metadata DBs self.add_objects_from_hpss(swift_api, target_container, container_dir) @@ -458,7 +458,8 @@ class Reconciler: return objects @trace_function - def add_objects_from_hpss(self, swift_api, target_container, container_dir): + def add_objects_from_hpss(self, swift_api, target_container, + container_dir): """ Update object metadata on object creates, and returns a list of all the objects existing in the container from Swift. @@ -512,7 +513,8 @@ class Reconciler: try: hpss_containers = os.listdir(account_directory) except OSError as err: - print "Unable to list files under directory: %s" % account_directory + print "Unable to list files under directory: %s" % \ + account_directory raise err # Delete containers that only exist in Swift, but not HPSS @@ -552,8 +554,8 @@ class Reconciler: known_good_objects = [] swift_only_objects = list(set(swift_objects) - set(hpss_objects)) - # If we have objects that only exist in the Swift metadata, delete those - # objects. + # If we have objects that only exist in the Swift metadata, + # delete those objects. for target_obj in swift_only_objects: try: swift_api.delete_object(target_container, target_obj) @@ -614,7 +616,8 @@ class Reconciler: if file_user not in keystone_users: fail_reason = \ "Cannot configure proper permissions for this path %s\ - because user %s does not exist in keystone" % (path, file_user) + because user %s does not exist in keystone" % \ + (path, file_user) print fail_reason logging.error(fail_reason) raise IOError(fail_reason) @@ -639,7 +642,7 @@ class Reconciler: # This only exists because the keystoneclient library is so massive that it has # to have a lazy-loading mechanism that ensures only one of it can be active, # so we can't have handles to multiple different Keystone scopes simultaneously -class LightweightKeystoneAPI: +class LightweightKeystoneAPI(object): MEMBER_ROLE_ID = '9fe2ff9ee4384b1894a90878d3e92bab' @@ -677,33 +680,19 @@ class LightweightKeystoneAPI: if self.version == 'v2': url = '%s/tokens' % self.url + creds = {'username': self.username, 'password': self.password} token_req = {'auth': {'tenantName': self.tenant_name, - 'passwordCredentials': { - 'username': self.username, - 'password': self.password - }}} + 'passwordCredentials': creds}} else: url = '%s/auth/tokens' % self.url - token_req = {'auth': {'identity': - {'methods': ['password'], - 'password': { - 'user': { - 'name': self.username, - 'password': self.password, - 'domain': {'id': 'default'} - } - } - }, - 'scope': { - 'project': { - 'name': self.tenant_name, - 'domain': { - 'id': 'default' - } - } - } - } - } + domain = {'id': 'default'} + creds = {'user': {'name': self.username, + 'password': self.password, + 'domain': domain}} + scope = {'project': {'name': self.tenant_name, 'domain': domain}} + token_req = {'auth': {'identity': {'methods': ['password'], + 'password': creds}, + 'scope': scope}} try: resp_headers, resp_json =\ self._get_keystone_response(requests.post, @@ -852,4 +841,4 @@ if __name__ == "__main__": log_level = logging.ERROR logging.basicConfig(filename=_args.logfile, level=log_level) - main(_args) \ No newline at end of file + main(_args) diff --git a/setup.py b/setup.py index ec139f9..0e9f132 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ if 'install' in sys.argv: # Install man pages the crappy hacky way, because setuptools doesn't # have any facility to do it. man_path = '/usr/local/share/man/1' - man_pages = filter(lambda x: os.path.isfile('./doc/troff/'+x), + man_pages = filter(lambda x: os.path.isfile('./doc/troff/%s' % x), os.listdir('./doc/troff')) for page in man_pages: - shutil.copyfile('./doc/troff/'+page, man_path) + shutil.copyfile('./doc/troff/%s' % page, man_path) diff --git a/swiftonhpss/swift/common/hpssfs_ioctl.py b/swiftonhpss/swift/common/hpssfs_ioctl.py new file mode 100644 index 0000000..fbdeb1e --- /dev/null +++ b/swiftonhpss/swift/common/hpssfs_ioctl.py @@ -0,0 +1,40 @@ +# Copyright (c) 2016 IBM Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import array +import fcntl + +HPSSFS_GET_COS = 0x80046c01 +HPSSFS_SET_COS_HINT = 0x40046c02 + +HPSSFS_SET_FSIZE_HINT = 0x40086c03 +HPSSFS_SET_MAXSEGSZ_HINT = 0x40046c04 + +HPSSFS_PURGE_CACHE = 0x00006c05 +HPSSFS_PURGE_LOCK = 0x40046c06 + +HPSSFS_UNDELETE = 0x40046c07 +HPSSFS_UNDELETE_NONE = 0x00000000 +HPSSFS_UNDELETE_RESTORE_TIME = 0x00000001 +HPSSFS_UNDELETE_OVERWRITE = 0x00000002 +HPSSFS_UNDELETE_OVERWRITE_AND_RESTORE = 0x00000003 + + +def ioctl(fd, cmd, val=None): + if val is not None: + valbuf = array.array("i", val) + fcntl.ioctl(fd, cmd, valbuf) + else: + fcntl.ioctl(fd, cmd) diff --git a/swiftonhpss/swift/common/utils.py b/swiftonhpss/swift/common/utils.py index 94660ec..236026b 100644 --- a/swiftonhpss/swift/common/utils.py +++ b/swiftonhpss/swift/common/utils.py @@ -24,7 +24,6 @@ from eventlet import sleep import cPickle as pickle from cStringIO import StringIO import pickletools -import xattr from swiftonhpss.swift.common.exceptions import SwiftOnFileSystemIOError from swift.common.exceptions import DiskFileNoSpace from swift.common.db import utf8encodekeys diff --git a/swiftonhpss/swift/obj/diskfile.py b/swiftonhpss/swift/obj/diskfile.py index ff72c7a..76031bf 100644 --- a/swiftonhpss/swift/obj/diskfile.py +++ b/swiftonhpss/swift/obj/diskfile.py @@ -23,7 +23,10 @@ except ImportError: import random import logging import time -import hpssfs +try: + import hpssfs +except ImportError: + import swiftonhpss.swift.common.hpssfs_ioctl as hpssfs import xattr from uuid import uuid4 from hashlib import md5 @@ -55,7 +58,7 @@ from swift.obj.diskfile import get_async_dir # FIXME: Hopefully we'll be able to move to Python 2.7+ where O_CLOEXEC will # be back ported. See http://www.python.org/dev/peps/pep-0433/ -O_CLOEXEC = 0o20000000 +O_CLOEXEC = 0o02000000 MAX_RENAME_ATTEMPTS = 10 MAX_OPEN_ATTEMPTS = 10 @@ -313,11 +316,12 @@ class DiskFileWriter(object): # (HPSS) Purge lock the file now if we're asked to. if purgelock: try: - hpssfs.ioctl(self._fd, hpssfs.HPSSFS_PURGE_LOCK, int(purgelock)) + hpssfs.ioctl(self._fd, hpssfs.HPSSFS_PURGE_LOCK, + int(purgelock)) except IOError as err: - raise SwiftOnFileSystemIOError(err.errno, - '%s, hpssfs.ioctl("%s", ...)' % ( - err.strerror, self._fd)) + raise SwiftOnFileSystemIOError( + err.errno, + '%s, hpssfs.ioctl("%s", ...)' % (err.strerror, self._fd)) # From the Department of the Redundancy Department, make sure # we call drop_cache() after fsync() to avoid redundant work @@ -811,7 +815,7 @@ class DiskFile(object): raise SwiftOnFileSystemIOError( err.errno, '%s, xattr.getxattr("system.hpss.level", ...)' % err.strerror - ) + ) try: file_levels = raw_file_levels.split(";") top_level = file_levels[0].split(':') @@ -1051,17 +1055,19 @@ class DiskFile(object): hpssfs.ioctl(fd, hpssfs.HPSSFS_SET_FSIZE_HINT, long(size)) except IOError as err: - raise SwiftOnFileSystemIOError(err.errno, - '%s, hpssfs.ioctl("%s", SET_FSIZE)' % ( - err.strerror, fd)) + message = '%s, hpssfs.ioctl("%s", SET_FSIZE)' + raise SwiftOnFileSystemIOError( + err.errno, + message % (err.strerror, fd)) if cos: try: hpssfs.ioctl(fd, hpssfs.HPSSFS_SET_COS_HINT, int(cos)) except IOError as err: - raise SwiftOnFileSystemIOError(err.errno, - '%s, hpssfs.ioctl("%s", SET_COS)' % ( - err.strerror, fd)) + message = '%s, hpssfs.ioctl("%s", SET_COS)' + raise SwiftOnFileSystemIOError( + err.errno, + message % (err.strerror, fd)) except SwiftOnFileSystemOSError as gerr: if gerr.errno in (errno.ENOSPC, errno.EDQUOT): diff --git a/swiftonhpss/swift/obj/server.py b/swiftonhpss/swift/obj/server.py index 1c73754..6e860c9 100644 --- a/swiftonhpss/swift/obj/server.py +++ b/swiftonhpss/swift/obj/server.py @@ -19,7 +19,10 @@ import math import logging import xattr import os -import hpssfs +try: + import hpssfs +except ImportError: + import swiftonhpss.swift.common.hpssfs_ioctl as hpssfs import time import eventlet @@ -29,7 +32,7 @@ from swift.common.swob import HTTPConflict, HTTPBadRequest, HeaderKeyDict, \ HTTPInsufficientStorage, HTTPPreconditionFailed, HTTPRequestTimeout, \ HTTPClientDisconnect, HTTPUnprocessableEntity, HTTPNotImplemented, \ HTTPServiceUnavailable, HTTPCreated, HTTPNotFound, HTTPAccepted, \ - HTTPNoContent, Request, Response + HTTPNoContent, Response from swift.common.utils import public, timing_stats, replication, \ config_true_value, Timestamp, csv_append from swift.common.request_helpers import get_name_and_placement, \ @@ -40,7 +43,7 @@ from swiftonhpss.swift.common.exceptions import AlreadyExistsAsFile, \ from swift.common.exceptions import DiskFileDeviceUnavailable, \ DiskFileNotExist, DiskFileQuarantined, ChunkReadTimeout, DiskFileNoSpace, \ DiskFileXattrNotSupported, DiskFileExpired, DiskFileDeleted -from swift.common.constraints import valid_timestamp, check_account_format +from swift.common.constraints import valid_timestamp from swift.obj import server from swift.common.ring import Ring @@ -64,8 +67,9 @@ class SwiftOnFileDiskFileRouter(object): class ObjectController(server.ObjectController): """ - Subclass of the object server's ObjectController that supports HPSS-specific - metadata headers and operations (such as COS assignment and purge locking). + Subclass of the object server's ObjectController that supports + HPSS-specific metadata headers and operations (such as COS assignment + and purge locking). """ def setup(self, conf): @@ -91,7 +95,6 @@ class ObjectController(server.ObjectController): self.container_ring = Ring(self.swift_dir, ring_name='container') return self.container_ring - @public @timing_stats() def PUT(self, request): @@ -195,8 +198,9 @@ class ObjectController(server.ObjectController): return HTTPUnprocessableEntity(request=request) # Update object metadata + content_type = request.headers['content-type'] metadata = {'X-Timestamp': request.timestamp.internal, - 'Content-Type': request.headers['content-type'], + 'Content-Type': content_type, 'ETag': etag, 'Content-Length': str(upload_size), } @@ -206,7 +210,8 @@ class ObjectController(server.ObjectController): metadata.update(meta_headers) backend_headers = \ request.headers.get('X-Backend-Replication-Headers') - for header_key in (backend_headers or self.allowed_headers): + for header_key in (backend_headers or + self.allowed_headers): if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] @@ -259,16 +264,12 @@ class ObjectController(server.ObjectController): self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy) + container_headers = {'x-size': metadata['Content-Length'], + 'x-content-type': metadata['Content-Type'], + 'x-timestamp': metadata['X-Timestamp'], + 'x-etag': metadata['ETag']} self.container_update('PUT', account, container, obj, request, - HeaderKeyDict( - {'x-size': - metadata['Content-Length'], - 'x-content-type': - metadata['Content-Type'], - 'x-timestamp': - metadata['X-Timestamp'], - 'x-etag': - metadata['ETag']}), + HeaderKeyDict(container_headers), device, policy) # Create convenience symlink try: @@ -346,8 +347,8 @@ class ObjectController(server.ObjectController): # Get DiskFile try: - disk_file = self.get_diskfile(device, partition, account, container, - obj, policy=policy) + disk_file = self.get_diskfile(device, partition, account, + container, obj, policy=policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) @@ -417,8 +418,8 @@ class ObjectController(server.ObjectController): # Get Diskfile try: - disk_file = self.get_diskfile(device, partition, account, container, - obj, policy) + disk_file = self.get_diskfile(device, partition, account, + container, obj, policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) @@ -446,7 +447,7 @@ class ObjectController(server.ObjectController): ) for key, value in metadata.iteritems(): if is_sys_or_user_meta('object', key) or \ - key.lower() in self.allowed_headers: + key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(float(file_x_ts)) @@ -524,10 +525,9 @@ class ObjectController(server.ObjectController): return HTTPNotFound(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: + backend_headers = {'X-Backend-Timestamp': orig_timestamp.internal} return HTTPConflict(request=request, - headers={ - 'X-Backend-Timestamp': orig_timestamp.internal - }) + headers=backend_headers) metadata = {'X-Timestamp': req_timestamp.internal} metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) diff --git a/tox.ini b/tox.ini index 2deccdf..2afc14e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,6 @@ [tox] -envlist = py27,pep8,functest +#envlist = py27,pep8,functest +envlist = py27,pep8 minversion = 1.6 skipsdist = True @@ -10,7 +11,7 @@ whitelist_externals=bash setenv = VIRTUAL_ENV={envdir} NOSE_WITH_COVERAGE=1 NOSE_COVER_BRANCHES=1 - NOSE_COVER_PACKAGE=swiftonfile + NOSE_COVER_PACKAGE=swiftonhpss deps = # Note: pip supports installing from git repos. # https://pip.pypa.io/en/latest/reference/pip_install.html#git @@ -35,9 +36,8 @@ commands = bash ./.functests -q [testenv:pep8] changedir = {toxinidir} -commands = -flake8 swiftonhpss test setup.py -flake8 --filename=swiftonhpss* bin +commands = flake8 swiftonhpss test setup.py + flake8 --filename=swiftonhpss* bin [testenv:venv] changedir = {toxinidir}