x/swiftonhpss
Code Issues Proposed changes

Fix Gerrit gates and other build issues

Browse Source 
There are a few problems with the configuration in our source tree that
causes the Jenkins gates to always fail, and things that might cause
a problem later. This patch set will fix those and other problems, to bring
us into a known valid state for future commits.

Change-Id: Idf7a0ce5902c40985caa78390b01f9fc2dfbfcf4
This commit is contained in:
Phil Bridges 2016-06-08 15:53:51 -05:00
parent 8127749f17
commit 3c7bcc0c7a
8 changed files with 123 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.gitreview
View File

@ -1,8 +1,6 @@
# TODO: get ourselves a nice and shiny CI system like this
#[gerrit]
#host=review.openstack.org
#port=29418
#project=openstack/swiftonfile.git
#defaultbranch=master
#defaultremote=gerrit
[gerrit]
host=review.openstack.org
port=29418
project=openstack/swiftonhpss.git
defaultbranch=master
defaultremote=gerrit

63
bin/swiftonhpss-nstool
View File

@ -18,7 +18,6 @@
import sys
import stat
import os
import multiprocessing
from pwd import getpwuid
import logging
import argparse
@ -87,11 +86,11 @@ def main(program_args):
del password
# Figure out what we're doing.
target_account, target_container = program_args.account,\
program_args.container
target_account = program_args.account
target_container = program_args.container
# Start doing it.
#pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()-1)
# pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()-1)
# Multiprocessing does not play nicely with the lazy loading that
# keystoneclient does, so let's not mess with it for now.
@ -227,7 +226,7 @@ def check_usage():
return parser.parse_args()
class Reconciler:
class Reconciler(object):
def __init__(self, args):
self._args = args
@ -436,7 +435,8 @@ class Reconciler:
logging.exception("Putting container %s went wrong" %
target_container)
raise e
print "Reconciling container %s/%s" % (target_account, target_container)
print "Reconciling container %s/%s" % \
(target_account, target_container)
# Make sure those objects get added into the Swift metadata DBs
self.add_objects_from_hpss(swift_api, target_container, container_dir)
@ -458,7 +458,8 @@ class Reconciler:
return objects
@trace_function
def add_objects_from_hpss(self, swift_api, target_container, container_dir):
def add_objects_from_hpss(self, swift_api, target_container,
container_dir):
"""
Update object metadata on object creates, and returns a list of all the
objects existing in the container from Swift.
@ -512,7 +513,8 @@ class Reconciler:
try:
hpss_containers = os.listdir(account_directory)
except OSError as err:
print "Unable to list files under directory: %s" % account_directory
print "Unable to list files under directory: %s" % \
account_directory
raise err
# Delete containers that only exist in Swift, but not HPSS
@ -552,8 +554,8 @@ class Reconciler:
known_good_objects = []
swift_only_objects = list(set(swift_objects) - set(hpss_objects))
# If we have objects that only exist in the Swift metadata, delete those
# objects.
# If we have objects that only exist in the Swift metadata,
# delete those objects.
for target_obj in swift_only_objects:
try:
swift_api.delete_object(target_container, target_obj)
@ -614,7 +616,8 @@ class Reconciler:
if file_user not in keystone_users:
fail_reason = \
"Cannot configure proper permissions for this path %s\
because user %s does not exist in keystone" % (path, file_user)
because user %s does not exist in keystone" % \
(path, file_user)
print fail_reason
logging.error(fail_reason)
raise IOError(fail_reason)
@ -639,7 +642,7 @@ class Reconciler:
# This only exists because the keystoneclient library is so massive that it has
# to have a lazy-loading mechanism that ensures only one of it can be active,
# so we can't have handles to multiple different Keystone scopes simultaneously
class LightweightKeystoneAPI:
class LightweightKeystoneAPI(object):
MEMBER_ROLE_ID = '9fe2ff9ee4384b1894a90878d3e92bab'
@ -677,33 +680,19 @@ class LightweightKeystoneAPI:
if self.version == 'v2':
url = '%s/tokens' % self.url
creds = {'username': self.username, 'password': self.password}
token_req = {'auth': {'tenantName': self.tenant_name,
'passwordCredentials': {
'username': self.username,
'password': self.password
}}}
'passwordCredentials': creds}}
else:
url = '%s/auth/tokens' % self.url
token_req = {'auth': {'identity':
{'methods': ['password'],
'password': {
'user': {
'name': self.username,
'password': self.password,
'domain': {'id': 'default'}
}
}
},
'scope': {
'project': {
'name': self.tenant_name,
'domain': {
'id': 'default'
}
}
}
}
}
domain = {'id': 'default'}
creds = {'user': {'name': self.username,
'password': self.password,
'domain': domain}}
scope = {'project': {'name': self.tenant_name, 'domain': domain}}
token_req = {'auth': {'identity': {'methods': ['password'],
'password': creds},
'scope': scope}}
try:
resp_headers, resp_json =\
self._get_keystone_response(requests.post,
@ -852,4 +841,4 @@ if __name__ == "__main__":
log_level = logging.ERROR
logging.basicConfig(filename=_args.logfile,
level=log_level)
main(_args)
main(_args)

4
setup.py
View File

@ -66,7 +66,7 @@ if 'install' in sys.argv:
# Install man pages the crappy hacky way, because setuptools doesn't
# have any facility to do it.
man_path = '/usr/local/share/man/1'
man_pages = filter(lambda x: os.path.isfile('./doc/troff/'+x),
man_pages = filter(lambda x: os.path.isfile('./doc/troff/%s' % x),
os.listdir('./doc/troff'))
for page in man_pages:
shutil.copyfile('./doc/troff/'+page, man_path)
shutil.copyfile('./doc/troff/%s' % page, man_path)

40
swiftonhpss/swift/common/hpssfs_ioctl.py Normal file
View File

@ -0,0 +1,40 @@
# Copyright (c) 2016 IBM Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import array
import fcntl
HPSSFS_GET_COS = 0x80046c01
HPSSFS_SET_COS_HINT = 0x40046c02
HPSSFS_SET_FSIZE_HINT = 0x40086c03
HPSSFS_SET_MAXSEGSZ_HINT = 0x40046c04
HPSSFS_PURGE_CACHE = 0x00006c05
HPSSFS_PURGE_LOCK = 0x40046c06
HPSSFS_UNDELETE = 0x40046c07
HPSSFS_UNDELETE_NONE = 0x00000000
HPSSFS_UNDELETE_RESTORE_TIME = 0x00000001
HPSSFS_UNDELETE_OVERWRITE = 0x00000002
HPSSFS_UNDELETE_OVERWRITE_AND_RESTORE = 0x00000003
def ioctl(fd, cmd, val=None):
if val is not None:
valbuf = array.array("i", val)
fcntl.ioctl(fd, cmd, valbuf)
else:
fcntl.ioctl(fd, cmd)

1
swiftonhpss/swift/common/utils.py
View File

@ -24,7 +24,6 @@ from eventlet import sleep
import cPickle as pickle
from cStringIO import StringIO
import pickletools
import xattr
from swiftonhpss.swift.common.exceptions import SwiftOnFileSystemIOError
from swift.common.exceptions import DiskFileNoSpace
from swift.common.db import utf8encodekeys

32
swiftonhpss/swift/obj/diskfile.py
View File

@ -23,7 +23,10 @@ except ImportError:
import random
import logging
import time
import hpssfs
try:
import hpssfs
except ImportError:
import swiftonhpss.swift.common.hpssfs_ioctl as hpssfs
import xattr
from uuid import uuid4
from hashlib import md5
@ -55,7 +58,7 @@ from swift.obj.diskfile import get_async_dir
# FIXME: Hopefully we'll be able to move to Python 2.7+ where O_CLOEXEC will
# be back ported. See http://www.python.org/dev/peps/pep-0433/
O_CLOEXEC = 0o20000000
O_CLOEXEC = 0o02000000
MAX_RENAME_ATTEMPTS = 10
MAX_OPEN_ATTEMPTS = 10
@ -313,11 +316,12 @@ class DiskFileWriter(object):
# (HPSS) Purge lock the file now if we're asked to.
if purgelock:
try:
hpssfs.ioctl(self._fd, hpssfs.HPSSFS_PURGE_LOCK, int(purgelock))
hpssfs.ioctl(self._fd, hpssfs.HPSSFS_PURGE_LOCK,
int(purgelock))
except IOError as err:
raise SwiftOnFileSystemIOError(err.errno,
'%s, hpssfs.ioctl("%s", ...)' % (
err.strerror, self._fd))
raise SwiftOnFileSystemIOError(
err.errno,
'%s, hpssfs.ioctl("%s", ...)' % (err.strerror, self._fd))
# From the Department of the Redundancy Department, make sure
# we call drop_cache() after fsync() to avoid redundant work
@ -811,7 +815,7 @@ class DiskFile(object):
raise SwiftOnFileSystemIOError(
err.errno,
'%s, xattr.getxattr("system.hpss.level", ...)' % err.strerror
)
)
try:
file_levels = raw_file_levels.split(";")
top_level = file_levels[0].split(':')
@ -1051,17 +1055,19 @@ class DiskFile(object):
hpssfs.ioctl(fd, hpssfs.HPSSFS_SET_FSIZE_HINT,
long(size))
except IOError as err:
raise SwiftOnFileSystemIOError(err.errno,
'%s, hpssfs.ioctl("%s", SET_FSIZE)' % (
err.strerror, fd))
message = '%s, hpssfs.ioctl("%s", SET_FSIZE)'
raise SwiftOnFileSystemIOError(
err.errno,
message % (err.strerror, fd))
if cos:
try:
hpssfs.ioctl(fd, hpssfs.HPSSFS_SET_COS_HINT, int(cos))
except IOError as err:
raise SwiftOnFileSystemIOError(err.errno,
'%s, hpssfs.ioctl("%s", SET_COS)' % (
err.strerror, fd))
message = '%s, hpssfs.ioctl("%s", SET_COS)'
raise SwiftOnFileSystemIOError(
err.errno,
message % (err.strerror, fd))
except SwiftOnFileSystemOSError as gerr:
if gerr.errno in (errno.ENOSPC, errno.EDQUOT):

50
swiftonhpss/swift/obj/server.py
View File

@ -19,7 +19,10 @@ import math
import logging
import xattr
import os
import hpssfs
try:
import hpssfs
except ImportError:
import swiftonhpss.swift.common.hpssfs_ioctl as hpssfs
import time
import eventlet
@ -29,7 +32,7 @@ from swift.common.swob import HTTPConflict, HTTPBadRequest, HeaderKeyDict, \
HTTPInsufficientStorage, HTTPPreconditionFailed, HTTPRequestTimeout, \
HTTPClientDisconnect, HTTPUnprocessableEntity, HTTPNotImplemented, \
HTTPServiceUnavailable, HTTPCreated, HTTPNotFound, HTTPAccepted, \
HTTPNoContent, Request, Response
HTTPNoContent, Response
from swift.common.utils import public, timing_stats, replication, \
config_true_value, Timestamp, csv_append
from swift.common.request_helpers import get_name_and_placement, \
@ -40,7 +43,7 @@ from swiftonhpss.swift.common.exceptions import AlreadyExistsAsFile, \
from swift.common.exceptions import DiskFileDeviceUnavailable, \
DiskFileNotExist, DiskFileQuarantined, ChunkReadTimeout, DiskFileNoSpace, \
DiskFileXattrNotSupported, DiskFileExpired, DiskFileDeleted
from swift.common.constraints import valid_timestamp, check_account_format
from swift.common.constraints import valid_timestamp
from swift.obj import server
from swift.common.ring import Ring
@ -64,8 +67,9 @@ class SwiftOnFileDiskFileRouter(object):
class ObjectController(server.ObjectController):
"""
Subclass of the object server's ObjectController that supports HPSS-specific
metadata headers and operations (such as COS assignment and purge locking).
Subclass of the object server's ObjectController that supports
HPSS-specific metadata headers and operations (such as COS assignment
and purge locking).
"""
def setup(self, conf):
@ -91,7 +95,6 @@ class ObjectController(server.ObjectController):
self.container_ring = Ring(self.swift_dir, ring_name='container')
return self.container_ring
@public
@timing_stats()
def PUT(self, request):
@ -195,8 +198,9 @@ class ObjectController(server.ObjectController):
return HTTPUnprocessableEntity(request=request)
# Update object metadata
content_type = request.headers['content-type']
metadata = {'X-Timestamp': request.timestamp.internal,
'Content-Type': request.headers['content-type'],
'Content-Type': content_type,
'ETag': etag,
'Content-Length': str(upload_size),
}
@ -206,7 +210,8 @@ class ObjectController(server.ObjectController):
metadata.update(meta_headers)
backend_headers = \
request.headers.get('X-Backend-Replication-Headers')
for header_key in (backend_headers or self.allowed_headers):
for header_key in (backend_headers or
self.allowed_headers):
if header_key in request.headers:
header_caps = header_key.title()
metadata[header_caps] = request.headers[header_key]
@ -259,16 +264,12 @@ class ObjectController(server.ObjectController):
self.delete_at_update('DELETE', orig_delete_at, account,
container, obj, request, device,
policy)
container_headers = {'x-size': metadata['Content-Length'],
'x-content-type': metadata['Content-Type'],
'x-timestamp': metadata['X-Timestamp'],
'x-etag': metadata['ETag']}
self.container_update('PUT', account, container, obj, request,
HeaderKeyDict(
{'x-size':
metadata['Content-Length'],
'x-content-type':
metadata['Content-Type'],
'x-timestamp':
metadata['X-Timestamp'],
'x-etag':
metadata['ETag']}),
HeaderKeyDict(container_headers),
device, policy)
# Create convenience symlink
try:
@ -346,8 +347,8 @@ class ObjectController(server.ObjectController):
# Get DiskFile
try:
disk_file = self.get_diskfile(device, partition, account, container,
obj, policy=policy)
disk_file = self.get_diskfile(device, partition, account,
container, obj, policy=policy)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
@ -417,8 +418,8 @@ class ObjectController(server.ObjectController):
# Get Diskfile
try:
disk_file = self.get_diskfile(device, partition, account, container,
obj, policy)
disk_file = self.get_diskfile(device, partition, account,
container, obj, policy)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
@ -446,7 +447,7 @@ class ObjectController(server.ObjectController):
)
for key, value in metadata.iteritems():
if is_sys_or_user_meta('object', key) or \
key.lower() in self.allowed_headers:
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = math.ceil(float(file_x_ts))
@ -524,10 +525,9 @@ class ObjectController(server.ObjectController):
return HTTPNotFound(request=request)
orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))
if orig_timestamp >= req_timestamp:
backend_headers = {'X-Backend-Timestamp': orig_timestamp.internal}
return HTTPConflict(request=request,
headers={
'X-Backend-Timestamp': orig_timestamp.internal
})
headers=backend_headers)
metadata = {'X-Timestamp': req_timestamp.internal}
metadata.update(val for val in request.headers.iteritems()
if is_user_meta('object', val[0]))

10
tox.ini
View File

@ -1,5 +1,6 @@
[tox]
envlist = py27,pep8,functest
#envlist = py27,pep8,functest
envlist = py27,pep8
minversion = 1.6
skipsdist = True
@ -10,7 +11,7 @@ whitelist_externals=bash
setenv = VIRTUAL_ENV={envdir}
NOSE_WITH_COVERAGE=1
NOSE_COVER_BRANCHES=1
NOSE_COVER_PACKAGE=swiftonfile
NOSE_COVER_PACKAGE=swiftonhpss
deps =
# Note: pip supports installing from git repos.
# https://pip.pypa.io/en/latest/reference/pip_install.html#git
@ -35,9 +36,8 @@ commands = bash ./.functests -q
[testenv:pep8]
changedir = {toxinidir}
commands =
flake8 swiftonhpss test setup.py
flake8 --filename=swiftonhpss* bin
commands = flake8 swiftonhpss test setup.py
flake8 --filename=swiftonhpss* bin
[testenv:venv]
changedir = {toxinidir}