openstack/openstack-ansible
Code Issues Proposed changes

Clean up some unrequired swift scripts

Browse Source 
* Remove the swift_inventory script
* Remove the swift_rings.py (its moved to the swift_ring_builder role)
* Remove the swift_setup.yml (We will use rpc_user_config.yml for these)
This commit is contained in:
Andy McCrae 2014-09-26 14:42:41 +00:00 committed by Andy McCrae
parent 0d2f69bc03
commit e787a0de43
3 changed files with 0 additions and 630 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
etc/rpc_deploy/swift_setup.yml
View File

@ -1,111 +0,0 @@
---
# This swift setup file, is used as a simple way of managing your swift
# ring configuration. The swift_inventory binary will generate the ansible
# iventory file for you based of this file. Giving you one place to manage your
# cluster.
#
# NOTE: The swift ansible configuration has other variables that can be tweeked,
# see group_vars/ and roles/*/vars/ for other areas to tweek.
local:
ansible_connection=local
# Swift settings are global through out the cluster.
swift:
part_power: 8
output_directory: /tmp/swift
output_filename: hosts
user: swift
hash_path_suffix: changeme
hash_path_prefix: changeme
syslog_host: 10.1.1.1:514
# Global hosts
# Hosts can be defined in the sections below or here, the advantage of
# defining them here is that they can be reused in multiple sections below.
#
# When need to reference a global host use the form:
# - name: <host>/<drive>
hosts:
- host: 10.0.0.4
drive: sda
drives: /srv/disk
region: 0
zone: 0
weight: 100
repl_ip: 192.168.0.4
repl_port: 55555
- host: 10.0.0.5
drive: sda
drives: /srv/disk
region: 0
zone: 1
weight: 100
proxy:
memcache_servers:
- 192.168.100.7:11211
authtoken:
active: true
delay_auth_decision: true
auth_version: v2.0
auth_host: 192.168.100.7
auth_port: 35357
auth_protocol: https
auth_uri: http://192.168.100.7:5000/
admin_tenant_name: service
admin_user: swift
admin_password: ADMIN
hosts:
- host:
account:
repl_number: 3
min_part_hours: 1
port: 6002
hosts:
- host: 10.0.0.2
drive: sdba
region: 0
zone: 0
weight: 100
container:
repl_number: 3
port: 6001
hosts:
- host: 10.0.0.3
drive: sda
region: 0
zone: 0
weight: 100
storage_policies:
default: gold
port: 6000
policies:
- name: gold
index: 0
type: replication
repl_number: 3
min_part_hours: 1
hosts:
- name: 10.0.0.4/sda
- name: 10.0.0.5/sda
- host: 10.0.0.6
drive: sdb
region: 1
zone: 0
weight: 50
- host: 10.0.0.7
drive: sdb
region: 1
zone: 1
weight: 50
- name: silver
index: 1
type: replication
repl_number: 2
depricated: True
hosts:
- name: 10.0.0.4/sda
- name: 10.0.0.5/sda

361
scripts/swift_inventory.py
View File

@ -1,361 +0,0 @@
#!/usr/bin/env python
from __future__ import print_function
import datetime
import sys
import yaml
from optparse import OptionParser
from os.path import exists, isdir, join
VERSION = '0.1'
USAGE = "usage: %prog [options] -s <swift setup yaml>"
DEFAULT_PART_POWER = 8
DEFAULT_REPL_NUM = 3
DEFAULT_REGION = 0
DEFAULT_ZONE = 0
DEFAULT_WEIGHT = 100
DEFAULT_DRIVE = "sda"
DEFAULT_DRIVES = "/srv/disk"
DEFAULT_OUTPUT_DIR = "/etc/ansible"
DEFAULT_OUTPUT_FILENAME = "hosts"
RETURN_NOT_DEFINED = 3
# FILE formatted strings
HEADER = """# This file was generated using the %s version %s at %s
[local]
localhost ansible_connection=local
[proxy]"""
CATCH_ALL_GROUPS = """
[object:children]
storagepolicy
[swift:children]
proxy
account
container
object
local
[swift:vars]"""
DRIVE_FORMAT = "%(host)s drive=%(drive)s region=%(region)s zone=%(zone)s "
DRIVE_FORMAT += "weight=%(weight)s drives=%(drives)s extra_data=%(extra_data)s"
DEFAULT_AUTHTOKEN_SETTINGS = {
'auth_version': 'v2.0',
'auth_host': 'keystone',
'auth_port': '35357',
'auth_protocol': 'https',
'admin_tenant_name': 'service',
'admin_user': 'swift',
'admin_password': 'ADMIN',
}
DATA_ORDER = [
NAME,
INDEX,
DEPRICATED,
TYPE,
] = [
'name',
'index',
'depricated',
'type',
]
class DriveException(Exception):
pass
def main(setup, verbose=False, dry_run=False, overwrite=True):
# Parse the setup file, which should be yaml
_swift = {}
_drives = {}
try:
with open(setup) as yaml_stream:
_swift = yaml.load(yaml_stream)
except Exception as err:
print("ERROR: Failed to yaml failure: %s", err)
return 2
def _section_defined(section):
if section not in _swift:
print("ERROR: no swift section defined")
return False
return True
def _get_output_fd(filename):
if dry_run:
return None
elif not overwrite and exists(filename):
i = 1
while exists("%s_%d" % (filename, i)):
i += 1
return open("%s_%d" % (filename, i), 'w')
else:
return open(filename, 'w')
def _write_to_file(fd, data):
if not fd or verbose:
print(data)
if fd:
if not data.endswith('\n'):
data += "\n"
fd.write(data)
fd.flush()
def _get_drive(drive, extra_data={}):
if 'name' in drive:
# We are dealing with a global host reference
key = drive['name']
if key not in _drives:
raise DriveException("Drive referenced doesn't exist")
else:
# Not a referenced host.
_drive = {
'drive': DEFAULT_DRIVE,
'region': DEFAULT_REGION,
'zone': DEFAULT_ZONE,
'weight': DEFAULT_WEIGHT,
'drives': DEFAULT_DRIVES,
'extra_data': ''}
if "drive" not in drive:
drive["drive"] = DEFAULT_DRIVE
key = "%(host)s/%(drive)s" % drive
if extra_data:
data_str = ":".join([str(extra_data.get(i, '')) for i in
DATA_ORDER])
else:
data_str = ""
if key in _drives:
if not _drives[key]['extra_data'] and extra_data:
_drives[key]['extra_data'] = data_str
elif _drives[key]['extra_data'] and extra_data:
_drives[key]['extra_data'] += ";%s" % (data_str)
return _drives[key]
else:
_drive.update(drive)
_drive['extra_data'] = data_str
_drives[key] = _drive
return _drive
# First attempt to get swift settings
if not _section_defined("swift"):
return RETURN_NOT_DEFINED
swift_options = [
"part_power=%s" % (_swift['swift'].get('part_power',
DEFAULT_PART_POWER)),
"user=%s" % (_swift['swift'].get('user', 'swift')),
"hash_path_suffix=%s" % (_swift['swift'].get("hash_path_suffix")),
"hash_path_prefix=%s" % (_swift['swift'].get("hash_path_prefix")),
"syslog_host=%s" % (_swift['swift'].get('syslog_host',
'localhost:514')),
]
output_path = _swift['swift'].get("output_directory", DEFAULT_OUTPUT_DIR)
output_file = _swift['swift'].get("output_filename",
DEFAULT_OUTPUT_FILENAME)
if not isdir(output_path):
print("Outdir path '%s' doesn't exist", output_path)
return 4
output_file = join(output_path, output_file)
output_fd = _get_output_fd(output_file)
n = datetime.datetime.now()
_write_to_file(output_fd, HEADER % (__file__, VERSION, n.ctime()))
# Global hosts
if _section_defined("hosts"):
for host in _swift["hosts"]:
_get_drive(host)
if not _section_defined("proxy"):
return RETURN_NOT_DEFINED
# Parse proxies
# TODO: Add read anfinity and pipeline here?
for proxy in _swift["proxy"]["hosts"]:
_write_to_file(output_fd, "%s" % (proxy["host"]))
_write_to_file(output_fd, "\n[proxy:vars]")
_mc_servers = _swift["proxy"].get('memcache_servers')
memcache_servers = ",".join(_mc_servers) if _mc_servers else \
'127.0.0.1:11211'
_write_to_file(output_fd, "memcache_servers=%s" % (memcache_servers))
_at = _swift["proxy"].get('authtoken')
if _at:
authtoken = DEFAULT_AUTHTOKEN_SETTINGS
authtoken.update(_at)
at_active = authtoken.get("active", False)
if at_active:
_write_to_file(output_fd, "authtoken_active=true")
_write_to_file(output_fd, "delay_auth_decision="
"%(delay_auth_decision)s" % authtoken)
_write_to_file(output_fd, "auth_version="
"%(auth_version)s" % authtoken)
_write_to_file(output_fd, "auth_host="
"%(auth_host)s" % authtoken)
_write_to_file(output_fd, "auth_port="
"%(auth_port)s" % authtoken)
_write_to_file(output_fd, "auth_protocol="
"%(auth_protocol)s" % authtoken)
_write_to_file(output_fd, "auth_uri="
"%(auth_uri)s" % authtoken)
_write_to_file(output_fd, "admin_tenant_name="
"%(admin_tenant_name)s" % authtoken)
_write_to_file(output_fd, "admin_user="
"%(admin_user)s" % authtoken)
_write_to_file(output_fd, "admin_password="
"%(admin_password)s" % authtoken)
else:
_write_to_file(output_fd, "authtoken_active=false")
_write_to_file(output_fd, "\n[account]")
if not _section_defined("account"):
return RETURN_NOT_DEFINED
for account in _swift["account"]["hosts"]:
data = _get_drive(account)
_write_to_file(output_fd, DRIVE_FORMAT % data)
_write_to_file(output_fd, "\n[account:vars]")
repl_num = _swift["account"].get("repl_number", DEFAULT_REPL_NUM)
_write_to_file(output_fd, "repl_number=%d" % (repl_num))
port = _swift["account"].get("port", 6002)
_write_to_file(output_fd, "account_server_port=%d" % (port))
# Container section
_write_to_file(output_fd, "\n[container]")
if not _section_defined("container"):
return RETURN_NOT_DEFINED
for container in _swift["container"]["hosts"]:
data = _get_drive(container)
_write_to_file(output_fd, DRIVE_FORMAT % data)
_write_to_file(output_fd, "\n[container:vars]")
repl_num = _swift["container"].get("repl_number", DEFAULT_REPL_NUM)
_write_to_file(output_fd, "repl_number=%d" % (repl_num))
port = _swift["container"].get("port", 6001)
_write_to_file(output_fd, "container_server_port=%d" % (port))
# Objects / Storage polices
_storage_policies = {}
_storage_policies_idx = {}
if not _section_defined("storage_policies"):
return RETURN_NOT_DEFINED
if "policies" not in _swift["storage_policies"]:
print("ERROR: No storage policies defined")
return 4
policy_hosts = {}
for policy in _swift["storage_policies"]["policies"]:
if policy["name"] in _storage_policies:
print("ERROR: Storage policy '%s' already defined" % policy["name"])
return 5
if policy["index"] in _storage_policies_idx:
print("ERROR: Storage policy index '%s' already defined" %
policy["index"])
return 5
_storage_policies[policy['name']] = "storagepolicy_%(name)s" % policy
_storage_policies_idx[policy['index']] = policy["name"]
policy_type = policy.get("type", 'replication')
_host_data = {
NAME: policy['name'],
INDEX: policy['index'],
TYPE: policy_type,
DEPRICATED: policy.get("depricated", False),
}
# print the storage policy hosts.
for drive in policy.get("hosts", []):
_get_drive(drive, _host_data)
policy_hosts[policy['name']] = policy.get("hosts")
_write_to_file(output_fd,
"\n[%s:vars]" % (_storage_policies[policy['name']]))
default = policy.get("default", False)
if default:
_write_to_file(output_fd, "default=%s" % (policy['name']))
if policy_type == 'replication':
repl_num = policy.get("repl_number", DEFAULT_REPL_NUM)
_write_to_file(output_fd, "repl_num=%d" % (repl_num))
# now write out the drives.
for policy in _swift["storage_policies"]["policies"]:
_write_to_file(output_fd,
"\n[%s]" % (_storage_policies[policy['name']]))
for drive in policy_hosts[policy['name']]:
data = _get_drive(drive)
_write_to_file(output_fd, DRIVE_FORMAT % data)
# Write out the storage policy catch all group
_write_to_file(output_fd, "\n[storagepolicy:children]")
for name, longname in _storage_policies.items():
_write_to_file(output_fd, "%s" % (longname))
_write_to_file(output_fd, "\n[storagepolicy:vars]")
if 'default' in _swift["storage_policies"]:
default_sp = _swift["storage_policies"]["default"]
if default_sp in _storage_policies:
_write_to_file(output_fd, "default=%s" % (default_sp))
elif default_sp in _storage_policies_idx:
_write_to_file(output_fd,
"default=%s" % (_storage_policies_idx[default_sp]))
else:
print("ERROR: Default storage policy '%s' doesn't exist",
default_sp)
port = _swift["storage_policies"].get("port", 6000)
_write_to_file(output_fd, "object_server_port=%d" % (port))
# Write out the object and swift catchall groups
_write_to_file(output_fd, CATCH_ALL_GROUPS)
# Now write out the global swift options that is gathered in the file
for option in swift_options:
_write_to_file(output_fd, option)
# Done
if output_fd:
output_fd.flush()
output_fd.close()
return 0
if __name__ == "__main__":
parser = OptionParser(USAGE)
parser.add_option("-s", "--setup", dest="setup",
help="Specify the swift setup file.", metavar="FILE")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
default=False, help="Be more verbose")
parser.add_option("-d", "--dryrun", action="store_true", dest="dry_run",
default=False, help="Print result out to stdout.")
parser.add_option("-C", "--copy", action="store_false", dest="overwrite",
default=True, help="Make a copy if inventory file exists")
parser.add_option("-i", "--import", dest="ring_folder", metavar="FILE",
help="Attempt to build a swift setup file"
" from the Swift builder files. Pass directory here")
options, args = parser.parse_args(sys.argv[1:])
if not options.setup or not exists(options.setup):
print("Swift setup file not found or doesn't exist")
parser.print_help()
sys.exit(1)
sys.exit(main(options.setup, options.verbose, options.dry_run,
options.overwrite))

158
scripts/swift_rings.py
View File

@ -1,158 +0,0 @@
#!/usr/bin/env python
from __future__ import print_function
from optparse import OptionParser
from os.path import exists
from swift.cli.ringbuilder import main as rb_main
import sys
import threading
import yaml
USAGE = "usage: %prog -s <swift setup yaml>"
DEFAULT_PART_POWER = 10
DEFAULT_REPL = 3
DEFAULT_MIN_PART_HOURS = 1
DEFAULT_HOST_DRIVES = '/srv/drive/'
DEFAULT_HOST_DRIVE = '/sdb'
DEFAULT_HOST_ZONE = 0
DEFAULT_HOST_WEIGHT = 1
DEFAULT_ACCOUNT_PORT = 6002
DEFAULT_CONTAINER_PORT = 6001
DEFAULT_OBJECT_PORT = 6000
DEFAULT_SECTION_PORT = {
'account': DEFAULT_ACCOUNT_PORT,
'container': DEFAULT_CONTAINER_PORT,
'object': DEFAULT_OBJECT_PORT,
}
def create_buildfile(build_file, part_power, repl, min_part_hours):
run_and_wait(rb_main, ["swift-ring-builder", build_file, "create",
part_power, repl, min_part_hours])
def add_host_to_ring(build_file, host):
host_str = ""
if host.get('region') is not None:
host_str += 'r%(region)d' % host
host_str += "z%d" % (host.get('zone', DEFAULT_HOST_ZONE))
host_str += "-%(host)s:%(port)d" % host
if host.get('repl_port'):
r_ip = host.get('repl_ip', host['host'])
host_str += "R%s:%d" % (r_ip, host['repl_port'])
host_str += "/%(drive)s" % host
weight = host.get('weight', DEFAULT_HOST_WEIGHT)
run_and_wait(rb_main, ["swift-ring-builder", build_file, 'add',
host_str, str(weight)])
def run_and_wait(func, *args):
t = threading.Thread(target=func, args=args)
t.start()
return t.join()
def has_section(conf, section):
return True if conf.get(section) else False
def check_section(conf, section):
if not has_section(conf, section):
print("Section %s doesn't exist" % (section))
sys.exit(2)
def build_ring(section, conf, part_power, hosts):
# Create the build file
build_file = "%s.builder" % (section)
repl = conf.get('repl_number', DEFAULT_REPL)
min_part_hours = conf.get('min_part_hours',
DEFAULT_MIN_PART_HOURS)
create_buildfile(build_file, part_power, repl, min_part_hours)
# Add the hosts
if not has_section(conf, 'hosts') or len(conf.get('hosts')) == 0:
print("No hosts/drives assigned to the %s ring" % section)
sys.exit(3)
section_key = section.split('-')[0]
service_port = conf.get('port', DEFAULT_SECTION_PORT[section_key])
for host in conf['hosts']:
if 'name' in host:
if host['name'] not in hosts:
print("Host %(name) reference not found." % host)
sys.exit(3)
host = hosts[host['name']]
else:
if 'drive' not in host:
host['drive'] = DEFAULT_HOST_DRIVE
host['port'] = service_port
add_host_to_ring(build_file, host)
# Rebalance ring
run_and_wait(rb_main, ["swift-ring-builder", build_file, "rebalance"])
# rb_main(("swift-ring-builder", build_file, "rebalance"))
def main(setup):
# load the yaml file
try:
with open(setup) as yaml_stream:
_swift = yaml.load(yaml_stream)
except Exception as ex:
print("Failed to load yaml string %s" % (ex))
return 1
_hosts = {}
if _swift.get("hosts"):
for host in _swift['hosts']:
if not host.get('drive'):
host['drive'] = DEFAULT_HOST_DRIVE
key = "%(host)s/%(drive)s" % host
if key in _hosts:
print("%(host)s already definined" % host)
return 1
_hosts[key] = host
check_section(_swift, 'swift')
part_power = _swift['swift'].get('part_power', DEFAULT_PART_POWER)
# Create account ring
check_section(_swift, 'account')
build_ring('account', _swift['account'], part_power, _hosts)
# Create container ring
check_section(_swift, 'container')
build_ring('container', _swift['container'], part_power, _hosts)
# Create object rings (storage policies)
check_section(_swift, 'storage_policies')
check_section(_swift['storage_policies'], 'policies')
indexes = set()
for sp in _swift['storage_policies']['policies']:
if sp['index'] in indexes:
print("Storage Policy index %d already in use" % (sp['index']))
return 4
buildfilename = 'object-%d' % (sp['index'])
indexes.add(sp['index'])
if 'port' not in sp:
sp['port'] = _swift['storage_policies'].get('port',
DEFAULT_OBJECT_PORT)
build_ring(buildfilename, sp, part_power, _hosts)
if __name__ == "__main__":
parser = OptionParser(USAGE)
parser.add_option("-s", "--setup", dest="setup",
help="Specify the swift setup file.", metavar="FILE",
default="/etc/swift/swift_inventory.yml")
options, args = parser.parse_args(sys.argv[1:])
if options.setup and not exists(options.setup):
print("Swift setup file not found or doesn't exist")
parser.print_help()
sys.exit(1)
sys.exit(main(options.setup))