x/anvil
Code Issues Proposed changes

Merge pull request #42 from hagleitn/master

Browse Source 
swift stuff
This commit is contained in:
Gunther Hagleitner 2012-02-04 04:04:51 -08:00
commit 5cde9b6012
5 changed files with 155 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
conf/pkgs/swift.json
View File

@ -41,6 +41,10 @@
"sqlite3": {
"version": "3.7*",
"removable": true
},
"xfsprogs": {
"version": "3.1*",
"removable": true
}
},
"rhel-6": {

2
devstack/components/keystone.py
View File

@ -119,7 +119,7 @@ class KeystoneInstaller(comp.PythonInstallComponent):
def _setup_data(self):
LOG.info("Configuring data setup template %s.", MANAGE_DATA_CONF)
(src_fn, contents) = utils.load_template(self.component_name, MANAGE_DATA_CONF)
(_, contents) = utils.load_template(self.component_name, MANAGE_DATA_CONF)
params = self._get_param_map(MANAGE_DATA_CONF)
contents = utils.param_replace(contents, params, True)
tgt_fn = sh.joinpths(self.bindir, MANAGE_DATA_CONF)

3
devstack/components/nova.py
View File

@ -545,7 +545,8 @@ class NovaConfigurator(object):
if img_service.lower().find("glance") != -1:
glance_api_server = self._getstr('glance_server')
if not glance_api_server:
glance_api_server = "%s:%d" % (hostip, DEF_GLANCE_PORT)
glance_api_server = "%s:%d" % (self.cfg.get('host', 'ip'),
DEF_GLANCE_PORT)
nova_conf.add('glance_api_servers', glance_api_server)
def _configure_vnc(self, nova_conf):

93
devstack/components/swift.py
View File

@ -14,14 +14,10 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
import os.path
from devstack import component as comp
from devstack import log as logging
from devstack import settings
from devstack import shell as sh
from devstack import utils
LOG = logging.getLogger("devstack.components.swift")
@ -58,6 +54,10 @@ REQ_PKGS = ['general.json', 'swift.json']
class SwiftUninstaller(comp.PythonUninstallComponent):
def __init__(self, *args, **kargs):
comp.PythonUninstallComponent.__init__(self, TYPE, *args, **kargs)
self.datadir = sh.joinpths(self.appdir, self.cfg.get('swift', 'data_location'))
def pre_uninstall(self):
sh.umount(sh.joinpths(self.datadir, 'drives/sdb1'))
class SwiftInstaller(comp.PythonInstallComponent):
@ -82,6 +82,9 @@ class SwiftInstaller(comp.PythonInstallComponent):
def _get_pkgs(self):
return list(REQ_PKGS)
def _get_symlinks(self):
return {self.cfgdir: '/etc/swift'}
def warm_configs(self):
pws = ['service_token', 'swift_hash']
for pw_key in pws:
@ -91,25 +94,93 @@ class SwiftInstaller(comp.PythonInstallComponent):
return {
'USER': self.cfg.get('swift', 'swift_user'),
'GROUP': self.cfg.get('swift', 'swift_group'),
'SWIFT_DATA_LOCATION': self.cfg.get('swift', 'data_location'),
'SWIFT_DATA_LOCATION': self.datadir,
'SWIFT_CONFIG_LOCATION': self.cfgdir,
'SERVICE_TOKEN': self.cfg.get('passwords', 'service_token'),
'AUTH_SERVER': self.auth_server,
'SWIFT_HASH': self.cfg.get('passwords', 'swift_hash'),
'NODE_PATH': '',
'BIND_PORT': '',
'LOG_FACILITY': '',
'SWIFT_LOGDIR': self.logdir,
'SWIFT_PARTITION_POWER_SIZE': self.cfg.get('swift', 'partition_power_size')
'SWIFT_PARTITION_POWER_SIZE': self.cfg.get('swift',
'partition_power_size'),
'NODE_PATH': '%NODE_PATH%',
'BIND_PORT': '%BIND_PORT%',
'LOG_FACILITY': '%LOG_FACILITY%'
}
def _post_install(self):
pass
def __create_data_location(self):
self.fs_image = sh.joinpths(self.datadir, 'drives/images/swift.img')
sh.create_loopback_file(fname=self.fs_image,
size=int(self.cfg.get('swift',
'loopback_disk_size')),
fs_type='xfs')
self.fs_dev = sh.joinpths(self.datadir, 'drives/sdb1/')
sh.mount_loopback_file(self.fs_image, self.fs_dev, 'xfs',
run_as_root=False)
def __create_node_config(self, node_number, port):
for type_ in ['object', 'container', 'account']:
sh.copy_replace_file(sh.joinpths(self.cfgdir, '%s-server.conf' % type_),
sh.joinpths(self.cfgdir, '%s-server/%d' % (type_, node_number)),
{
'%NODE_PATH%': sh.joinpths(self.datadir, str(node_number)),
'%BIND_PORT%': str(port),
'%LOG_FACILITY%': str(2 + node_number)
})
port += 1
def __create_nodes(self):
for i in range(1, 5):
sh.mkdirslist(sh.joinpths(self.fs_dev, '%d/node' % i))
sh.symlink(sh.joinpths(self.fs_dev, str(i)),
sh.joinpths(self.datadir, str(i)))
self.__create_node_config(i, 6010 + (i - 1) * 5)
def __turn_on_rsync(self):
sh.symlink(sh.joinpths(self.cfgdir, RSYNC_CONF),
'/etc/rsyncd.conf')
sh.replace_in_file('/etc/default/rsync',
'RSYNC_ENABLE=false',
'RSYNC_ENABLE=true')
def __create_log_dirs(self):
sh.mkdirslist(sh.joinpths(self.logdir, 'hourly'))
sh.symlink(sh.joinpths(self.cfgdir, SYSLOG_CONF),
'/etc/rsyslog.d/10-swift.conf')
def __setup_binaries(self):
self.makerings_file = sh.joinpths(self.bindir, SWIFT_MAKERINGS)
sh.move(sh.joinpths(self.cfgdir, SWIFT_MAKERINGS),
self.makerings_file)
sh.chmod(self.makerings_file, 777)
self.startmain_file = sh.joinpths(self.bindir, SWIFT_STARTMAIN)
sh.move(sh.joinpths(self.cfgdir, SWIFT_STARTMAIN),
self.startmain_file)
sh.chmod(self.startmain_file, 777)
def __make_rings(self):
sh.execute(self.makerings_file)
def post_install(self):
self.__create_data_location()
self.__create_nodes()
self.__turn_on_rsync()
self.__create_log_dirs()
self.__setup_binaries()
self.__make_rings()
class SwiftRuntime(comp.PythonRuntime):
def __init__(self, *args, **kargs):
comp.PythonRuntime.__init__(self, TYPE, *args, **kargs)
self.bindir = sh.joinpths(self.appdir, BIN_DIR)
def pre_start(self):
sh.execute('restart', 'rsyslog')
sh.execute('/etc/init.d/rsync', 'restart')
def post_start(self):
sh.execute(sh.joinpths(self.bindir, SWIFT_STARTMAIN))
def describe(opts=None):

70
devstack/shell.py
View File

@ -16,12 +16,12 @@
import getpass
import grp
import os
import os.path
import pwd
import shutil
import subprocess
import tempfile
import fileinput
from devstack import env
from devstack import exceptions as excp
@ -263,7 +263,6 @@ def touch_file(fn, die_if_there=True, quiet=False, file_size=0):
def load_file(fn, quiet=False):
if not quiet:
LOG.debug("Loading data from file %s", fn)
data = ""
with open(fn, "r") as f:
data = f.read()
if not quiet:
@ -301,11 +300,12 @@ def rmdir(path, quiet=True):
pass
def symlink(source, link):
def symlink(source, link, force=True):
path = dirname(link)
file_ = basename(link)
mkdirslist(path)
LOG.debug("Creating symlink from %s => %s" % (link, source))
if force and exists(link):
unlink(link, True)
os.symlink(source, link)
@ -350,6 +350,42 @@ def getgroupname(gid=None):
return gid_info.gr_name
def create_loopback_file(fname, size, bsize=1024, fs_type='ext3', run_as_root=False):
dd_cmd = ['dd', 'if=/dev/zero', 'of=%s' % fname, 'bs=%d' % bsize,
'count=0', 'seek=%d' % size]
mkfs_cmd = ['mkfs.%s' % fs_type, '-f', '-i', 'size=%d' % bsize, fname]
# make sure folder exists
files = mkdirslist(dirname(fname))
# create file
touch_file(fname)
# fill with zeroes
execute(*dd_cmd, run_as_root=run_as_root)
# create fs on the file
execute(*mkfs_cmd, run_as_root=run_as_root)
return files
def mount_loopback_file(fname, device_name, fs_type='ext3', run_as_root=True):
mount_cmd = ['mount', '-t', fs_type, '-o',
'loop,noatime,nodiratime,nobarrier,logbufs=8', fname,
device_name]
files = mkdirslist(dirname(device_name))
execute(*mount_cmd, run_as_root=run_as_root)
return files
def umount(dev_name, run_as_root=True):
execute('umount', dev_name, run_as_root=run_as_root)
def unlink(path, ignore_errors=True):
try:
LOG.debug("Unlinking (removing) %s" % (path))
@ -359,3 +395,29 @@ def unlink(path, ignore_errors=True):
raise
else:
pass
def move(src, dst):
shutil.move(src, dst)
def chmod(fname, mode):
os.chmod(fname, mode)
def replace_in_file(fname, search, replace):
# fileinput with inplace=1 moves file to tmp and redirects stdio to file
for line in fileinput.input(fname, inplace=1):
if search in line:
line = line.replace(search, replace)
print line,
def copy_replace_file(fsrc, fdst, map_):
files = mkdirslist(dirname(fdst))
with open(fdst, 'w') as fh:
for line in fileinput.input(fsrc):
for (k, v) in map_.items():
line = line.replace(k, v)
fh.write(line)
return files