x/anvil
Code Issues Proposed changes

Swift cleanups.

Browse Source
This commit is contained in:
Joshua Harlow 2012-02-13 22:25:37 -08:00
parent fe94293714
commit 97699f1b15
3 changed files with 55 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
devstack/components/nova.py
View File

@ -16,7 +16,6 @@
import io import io
import os import os
import stat
from devstack import cfg from devstack import cfg
from devstack import component as comp from devstack import component as comp

84
devstack/components/swift.py
View File

@ -14,6 +14,8 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import re
from devstack import component as comp from devstack import component as comp
from devstack import log as logging from devstack import log as logging
from devstack import settings from devstack import settings
@ -39,10 +41,19 @@ DEVICE_PATH = 'drives/sdb1'
CONFIGS = [SWIFT_CONF, PROXY_SERVER_CONF, ACCOUNT_SERVER_CONF, CONFIGS = [SWIFT_CONF, PROXY_SERVER_CONF, ACCOUNT_SERVER_CONF,
CONTAINER_SERVER_CONF, OBJECT_SERVER_CONF, RSYNC_CONF, CONTAINER_SERVER_CONF, OBJECT_SERVER_CONF, RSYNC_CONF,
SYSLOG_CONF, SWIFT_MAKERINGS, SWIFT_STARTMAIN] SYSLOG_CONF, SWIFT_MAKERINGS, SWIFT_STARTMAIN]
SWIFT_RSYNC_LOC = '/etc/rsyslog.d/10-swift.conf'
RSYNC_CONF_LOC = '/etc/default/rsync'
RSYNCD_CONF_LOC = '/etc/rsyncd.conf'
RSYNC_SERVICE_RESTART = ['service', 'rsync', 'restart']
RSYSLOG_SERVICE_RESTART = ['service', 'rsyslog', 'restart']
RSYNC_ON_OFF_RE = re.compile(r'^\s*RSYNC_ENABLE\s*=\s*(.*)$', re.I)
AUTH_SERVICE = 'keystone'
# subdirs of the git checkout # subdirs of the git checkout
BIN_DIR = 'bin' BIN_DIR = 'bin'
CONFIG_DIR = 'etc' CONFIG_DIR = 'etc'
LOG_DIR = 'logs'
#the pkg json files swift requires for installation #the pkg json files swift requires for installation
REQ_PKGS = ['general.json', 'swift.json'] REQ_PKGS = ['general.json', 'swift.json']
@ -52,17 +63,17 @@ class SwiftUninstaller(comp.PythonUninstallComponent):
def __init__(self, *args, **kargs): def __init__(self, *args, **kargs):
comp.PythonUninstallComponent.__init__(self, TYPE, *args, **kargs) comp.PythonUninstallComponent.__init__(self, TYPE, *args, **kargs)
self.datadir = sh.joinpths(self.appdir, self.cfg.get('swift', 'data_location')) self.datadir = sh.joinpths(self.appdir, self.cfg.get('swift', 'data_location'))
self.cfgdir = sh.joinpths(self.appdir, CONFIG_DIR)
self.bindir = sh.joinpths(self.appdir, BIN_DIR)
self.logdir = sh.joinpths(self.datadir, LOG_DIR)
def pre_uninstall(self): def pre_uninstall(self):
sh.umount(sh.joinpths(self.datadir, DEVICE_PATH)) sh.umount(sh.joinpths(self.datadir, DEVICE_PATH))
sh.replace_in_file('/etc/default/rsync', sh.replace_in(RSYNC_CONF_LOC, RSYNC_ON_OFF_RE, 'RSYNC_ENABLE=false', True)
'RSYNC_ENABLE=true',
'RSYNC_ENABLE=false',
run_as_root=True)
def post_uninstall(self): def post_uninstall(self):
sh.execute('restart', 'rsyslog', run_as_root=True) sh.execute(*RSYSLOG_SERVICE_RESTART, run_as_root=True)
sh.execute('/etc/init.d/rsync', 'restart', run_as_root=True) sh.execute(*RSYNC_SERVICE_RESTART, run_as_root=True)
class SwiftInstaller(comp.PythonInstallComponent): class SwiftInstaller(comp.PythonInstallComponent):
@ -71,12 +82,12 @@ class SwiftInstaller(comp.PythonInstallComponent):
self.cfgdir = sh.joinpths(self.appdir, CONFIG_DIR) self.cfgdir = sh.joinpths(self.appdir, CONFIG_DIR)
self.bindir = sh.joinpths(self.appdir, BIN_DIR) self.bindir = sh.joinpths(self.appdir, BIN_DIR)
self.datadir = sh.joinpths(self.appdir, self.cfg.get('swift', 'data_location')) self.datadir = sh.joinpths(self.appdir, self.cfg.get('swift', 'data_location'))
self.logdir = sh.joinpths(self.datadir, 'logs') self.logdir = sh.joinpths(self.datadir, LOG_DIR)
self.startmain_file = sh.joinpths(self.bindir, SWIFT_STARTMAIN) self.startmain_file = sh.joinpths(self.bindir, SWIFT_STARTMAIN)
self.makerings_file = sh.joinpths(self.bindir, SWIFT_MAKERINGS) self.makerings_file = sh.joinpths(self.bindir, SWIFT_MAKERINGS)
self.fs_dev = sh.joinpths(self.datadir, DEVICE_PATH) self.fs_dev = sh.joinpths(self.datadir, DEVICE_PATH)
self.fs_image = sh.joinpths(self.datadir, SWIFT_IMG) self.fs_image = sh.joinpths(self.datadir, SWIFT_IMG)
self.auth_server = 'keystone' self.auth_server = AUTH_SERVICE
def _get_download_locations(self): def _get_download_locations(self):
places = list() places = list()
@ -114,8 +125,7 @@ class SwiftInstaller(comp.PythonInstallComponent):
'AUTH_SERVER': self.auth_server, 'AUTH_SERVER': self.auth_server,
'SWIFT_HASH': self.cfg.get('passwords', 'swift_hash'), 'SWIFT_HASH': self.cfg.get('passwords', 'swift_hash'),
'SWIFT_LOGDIR': self.logdir, 'SWIFT_LOGDIR': self.logdir,
'SWIFT_PARTITION_POWER_SIZE': self.cfg.get('swift', 'SWIFT_PARTITION_POWER_SIZE': self.cfg.get('swift', 'partition_power_size'),
'partition_power_size'),
'NODE_PATH': '%NODE_PATH%', 'NODE_PATH': '%NODE_PATH%',
'BIND_PORT': '%BIND_PORT%', 'BIND_PORT': '%BIND_PORT%',
'LOG_FACILITY': '%LOG_FACILITY%' 'LOG_FACILITY': '%LOG_FACILITY%'
@ -123,18 +133,16 @@ class SwiftInstaller(comp.PythonInstallComponent):
def __create_data_location(self): def __create_data_location(self):
sh.create_loopback_file(fname=self.fs_image, sh.create_loopback_file(fname=self.fs_image,
size=int(self.cfg.get('swift', size=int(self.cfg.get('swift', 'loopback_disk_size')),
'loopback_disk_size')),
fs_type='xfs') fs_type='xfs')
self.tracewriter.file_touched(self.fs_image) self.tracewriter.file_touched(self.fs_image)
sh.mount_loopback_file(self.fs_image, self.fs_dev, 'xfs') sh.mount_loopback_file(self.fs_image, self.fs_dev, 'xfs')
sh.chown_r(self.fs_dev, sh.geteuid(), sh.getegid()) sh.chown_r(self.fs_dev, sh.geteuid(), sh.getegid())
def __create_node_config(self, node_number, port): def __create_node_config(self, node_number, port):
for type_ in ['object', 'container', 'account']: for t in ['object', 'container', 'account']:
sh.copy_replace_file(sh.joinpths(self.cfgdir, '%s-server.conf' % type_), sh.copy_replace_file(sh.joinpths(self.cfgdir, '%s-server.conf' % t),
sh.joinpths(self.cfgdir, '%s-server/%d.conf' \ sh.joinpths(self.cfgdir, '%s-server/%d.conf' % (t, node_number)),
% (type_, node_number)),
{ {
'%NODE_PATH%': sh.joinpths(self.datadir, str(node_number)), '%NODE_PATH%': sh.joinpths(self.datadir, str(node_number)),
'%BIND_PORT%': str(port), '%BIND_PORT%': str(port),
@ -143,39 +151,30 @@ class SwiftInstaller(comp.PythonInstallComponent):
port += 1 port += 1
def __delete_templates(self): def __delete_templates(self):
for type_ in ['object', 'container', 'account']: for t in ['object', 'container', 'account']:
sh.unlink(sh.joinpths(self.cfgdir, '%s-server.conf' % type_)) sh.unlink(sh.joinpths(self.cfgdir, '%s-server.conf' % t))
def __create_nodes(self): def __create_nodes(self):
for i in range(1, 5): for i in range(1, 5):
self.tracewriter.make_dir(sh.joinpths(self.fs_dev, self.tracewriter.make_dir(sh.joinpths(self.fs_dev, '%d/node' % i))
'%d/node' % i))
self.tracewriter.symlink(sh.joinpths(self.fs_dev, str(i)), self.tracewriter.symlink(sh.joinpths(self.fs_dev, str(i)),
sh.joinpths(self.datadir, str(i))) sh.joinpths(self.datadir, str(i)))
self.__create_node_config(i, 6010 + (i - 1) * 5) self.__create_node_config(i, 6010 + (i - 1) * 5)
self.__delete_templates() self.__delete_templates()
def __turn_on_rsync(self): def __turn_on_rsync(self):
self.tracewriter.symlink(sh.joinpths(self.cfgdir, RSYNC_CONF), self.tracewriter.symlink(sh.joinpths(self.cfgdir, RSYNC_CONF), RSYNCD_CONF_LOC)
'/etc/rsyncd.conf') sh.replace_in(RSYNC_CONF_LOC, RSYNC_ON_OFF_RE, 'RSYNC_ENABLE=true', True)
sh.replace_in_file('/etc/default/rsync',
'RSYNC_ENABLE=false',
'RSYNC_ENABLE=true',
run_as_root=True)
def __create_log_dirs(self): def __create_log_dirs(self):
self.tracewriter.make_dir(sh.joinpths(self.logdir, 'hourly')) self.tracewriter.make_dir(sh.joinpths(self.logdir, 'hourly'))
self.tracewriter.symlink(sh.joinpths(self.cfgdir, SYSLOG_CONF), self.tracewriter.symlink(sh.joinpths(self.cfgdir, SYSLOG_CONF), SWIFT_RSYNC_LOC)
'/etc/rsyslog.d/10-swift.conf')
def __setup_binaries(self): def __setup_binaries(self):
sh.move(sh.joinpths(self.cfgdir, SWIFT_MAKERINGS), sh.move(sh.joinpths(self.cfgdir, SWIFT_MAKERINGS), self.makerings_file)
self.makerings_file)
sh.chmod(self.makerings_file, 0777) sh.chmod(self.makerings_file, 0777)
self.tracewriter.file_touched(self.makerings_file) self.tracewriter.file_touched(self.makerings_file)
sh.move(sh.joinpths(self.cfgdir, SWIFT_STARTMAIN), self.startmain_file)
sh.move(sh.joinpths(self.cfgdir, SWIFT_STARTMAIN),
self.startmain_file)
sh.chmod(self.startmain_file, 0777) sh.chmod(self.startmain_file, 0777)
self.tracewriter.file_touched(self.startmain_file) self.tracewriter.file_touched(self.startmain_file)
@ -194,18 +193,21 @@ class SwiftInstaller(comp.PythonInstallComponent):
class SwiftRuntime(comp.PythonRuntime): class SwiftRuntime(comp.PythonRuntime):
def __init__(self, *args, **kargs): def __init__(self, *args, **kargs):
comp.PythonRuntime.__init__(self, TYPE, *args, **kargs) comp.PythonRuntime.__init__(self, TYPE, *args, **kargs)
self.datadir = sh.joinpths(self.appdir, self.cfg.get('swift', 'data_location'))
self.cfgdir = sh.joinpths(self.appdir, CONFIG_DIR)
self.bindir = sh.joinpths(self.appdir, BIN_DIR) self.bindir = sh.joinpths(self.appdir, BIN_DIR)
self.logdir = sh.joinpths(self.datadir, LOG_DIR)
def start(self): def start(self):
sh.execute('restart', 'rsyslog', run_as_root=True) sh.execute(*RSYSLOG_SERVICE_RESTART, run_as_root=True)
sh.execute('/etc/init.d/rsync', 'restart', run_as_root=True) sh.execute(*RSYNC_SERVICE_RESTART, run_as_root=True)
sh.execute(sh.joinpths(self.bindir, SWIFT_INIT), 'all', 'start', swift_start_cmd = [sh.joinpths(self.bindir, SWIFT_INIT)] + ['all', 'start']
run_as_root=True) sh.execute(*swift_start_cmd, run_as_root=True)
def stop(self): def stop(self):
sh.execute(sh.joinpths(self.bindir, SWIFT_INIT), 'all', 'stop', swift_stop_cmd = [sh.joinpths(self.bindir, SWIFT_INIT)] + ['all', 'stop']
run_as_root=True) sh.execute(*swift_stop_cmd, run_as_root=True)
def restart(self): def restart(self):
sh.execute(sh.joinpths(self.bindir, SWIFT_INIT), 'all', 'restart', swift_restart_cmd = [sh.joinpths(self.bindir, SWIFT_INIT)] + ['all', 'restart']
run_as_root=True) sh.execute(*swift_restart_cmd, run_as_root=True)

21
devstack/shell.py
View File

@ -17,7 +17,7 @@
import fileinput import fileinput
import getpass import getpass
import grp import grp
import os.path import os
import pwd import pwd
import shutil import shutil
import subprocess import subprocess
@ -460,20 +460,23 @@ def chmod(fname, mode):
os.chmod(fname, mode) os.chmod(fname, mode)
def replace_in_file(fname, search, replace, run_as_root=False): def replace_in(fn, search, replace, run_as_root=False):
with Rooted(run_as_root): with Rooted(run_as_root):
# fileinput with inplace=1 moves file to tmp and redirects stdio to file contents = load_file(fn)
for line in fileinput.input(fname, inplace=1):
if search in line: def replacer(match):
line = line.replace(search, replace) return replace
print line
(contents, num_changed) = search.subn(replacer, contents)
if num_changed:
write_file(fn, contents)
def copy_replace_file(fsrc, fdst, map_): def copy_replace_file(fsrc, fdst, linemap):
files = mkdirslist(dirname(fdst)) files = mkdirslist(dirname(fdst))
with open(fdst, 'w') as fh: with open(fdst, 'w') as fh:
for line in fileinput.input(fsrc): for line in fileinput.input(fsrc):
for (k, v) in map_.items(): for (k, v) in linemap.items():
line = line.replace(k, v) line = line.replace(k, v)
fh.write(line) fh.write(line)
return files return files