b13b49a27c
Instead of using a separate .durable file to indicate the durable status of a .data file, rename the .data to include a durable marker in the filename. This saves one inode for every EC fragment archive. An EC policy PUT will, as before, first rename a temp file to: <timestamp>#<frag_index>.data but now, when the object is committed, that file will be renamed: <timestamp>#<frag_index>#d.data with the '#d' suffix marking the data file as durable. Diskfile suffix hashing returns the same result when the new durable-data filename or the legacy durable file is found in an object directory. A fragment archive that has been created on an upgraded object server will therefore appear to be in the same state, as far as the consistency engine is concerned, as the same fragment archive created on an older object server. Since legacy .durable files will still exist in deployed clusters, many of the unit tests scenarios have been duplicated for both new durable-data filenames and legacy durable files. Change-Id: I6f1f62d47be0b0ac7919888c77480a636f11f607
110 lines
4.1 KiB
Python
110 lines
4.1 KiB
Python
# Copyright (c) 2013 - 2015 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
import hashlib
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
import unittest
|
|
import time
|
|
|
|
from swift.common.storage_policy import POLICIES
|
|
from swift.common.utils import Timestamp
|
|
from swift.obj import diskfile
|
|
|
|
from test.unit import debug_logger
|
|
|
|
|
|
class FakeReplicator(object):
|
|
def __init__(self, testdir, policy=None):
|
|
self.logger = debug_logger('test-ssync-sender')
|
|
self.conn_timeout = 1
|
|
self.node_timeout = 2
|
|
self.http_timeout = 3
|
|
self.network_chunk_size = 65536
|
|
self.disk_chunk_size = 4096
|
|
conf = {
|
|
'devices': testdir,
|
|
'mount_check': 'false',
|
|
}
|
|
policy = POLICIES.default if policy is None else policy
|
|
self._diskfile_router = diskfile.DiskFileRouter(conf, self.logger)
|
|
self._diskfile_mgr = self._diskfile_router[policy]
|
|
|
|
|
|
def write_diskfile(df, timestamp, data='test data', frag_index=None,
|
|
commit=True, legacy_durable=False, extra_metadata=None):
|
|
# Helper method to write some data and metadata to a diskfile.
|
|
# Optionally do not commit the diskfile, or commit but using a legacy
|
|
# durable file
|
|
with df.create() as writer:
|
|
writer.write(data)
|
|
metadata = {
|
|
'ETag': hashlib.md5(data).hexdigest(),
|
|
'X-Timestamp': timestamp.internal,
|
|
'Content-Length': str(len(data)),
|
|
}
|
|
if extra_metadata:
|
|
metadata.update(extra_metadata)
|
|
if frag_index is not None:
|
|
metadata['X-Object-Sysmeta-Ec-Frag-Index'] = str(frag_index)
|
|
writer.put(metadata)
|
|
if commit and legacy_durable:
|
|
# simulate legacy .durable file creation
|
|
durable_file = os.path.join(df._datadir,
|
|
timestamp.internal + '.durable')
|
|
with open(durable_file, 'wb'):
|
|
pass
|
|
elif commit:
|
|
writer.commit(timestamp)
|
|
# else: don't make it durable
|
|
return metadata
|
|
|
|
|
|
class BaseTest(unittest.TestCase):
|
|
def setUp(self):
|
|
# daemon will be set in subclass setUp
|
|
self.daemon = None
|
|
self.tmpdir = tempfile.mkdtemp()
|
|
|
|
def tearDown(self):
|
|
shutil.rmtree(self.tmpdir, ignore_errors=True)
|
|
|
|
def _make_diskfile(self, device='dev', partition='9',
|
|
account='a', container='c', obj='o', body='test',
|
|
extra_metadata=None, policy=None,
|
|
frag_index=None, timestamp=None, df_mgr=None,
|
|
commit=True):
|
|
policy = policy or POLICIES.legacy
|
|
object_parts = account, container, obj
|
|
timestamp = Timestamp(time.time()) if timestamp is None else timestamp
|
|
if df_mgr is None:
|
|
df_mgr = self.daemon._diskfile_router[policy]
|
|
df = df_mgr.get_diskfile(
|
|
device, partition, *object_parts, policy=policy,
|
|
frag_index=frag_index)
|
|
write_diskfile(df, timestamp, data=body, extra_metadata=extra_metadata,
|
|
commit=commit)
|
|
return df
|
|
|
|
def _make_open_diskfile(self, device='dev', partition='9',
|
|
account='a', container='c', obj='o', body='test',
|
|
extra_metadata=None, policy=None,
|
|
frag_index=None, timestamp=None, df_mgr=None):
|
|
df = self._make_diskfile(device, partition, account, container, obj,
|
|
body, extra_metadata, policy, frag_index,
|
|
timestamp, df_mgr)
|
|
df.open()
|
|
return df
|