From e787a0de43919143b61701ff799fb399a0c8dc93 Mon Sep 17 00:00:00 2001 From: Andy McCrae Date: Fri, 26 Sep 2014 14:42:41 +0000 Subject: [PATCH] Clean up some unrequired swift scripts * Remove the swift_inventory script * Remove the swift_rings.py (its moved to the swift_ring_builder role) * Remove the swift_setup.yml (We will use rpc_user_config.yml for these) --- etc/rpc_deploy/swift_setup.yml | 111 ---------- scripts/swift_inventory.py | 361 --------------------------------- scripts/swift_rings.py | 158 --------------- 3 files changed, 630 deletions(-) delete mode 100644 etc/rpc_deploy/swift_setup.yml delete mode 100644 scripts/swift_inventory.py delete mode 100644 scripts/swift_rings.py diff --git a/etc/rpc_deploy/swift_setup.yml b/etc/rpc_deploy/swift_setup.yml deleted file mode 100644 index 4ac4ead48c..0000000000 --- a/etc/rpc_deploy/swift_setup.yml +++ /dev/null @@ -1,111 +0,0 @@ ---- -# This swift setup file, is used as a simple way of managing your swift -# ring configuration. The swift_inventory binary will generate the ansible -# iventory file for you based of this file. Giving you one place to manage your -# cluster. -# -# NOTE: The swift ansible configuration has other variables that can be tweeked, -# see group_vars/ and roles/*/vars/ for other areas to tweek. -local: - ansible_connection=local - -# Swift settings are global through out the cluster. -swift: - part_power: 8 - output_directory: /tmp/swift - output_filename: hosts - user: swift - hash_path_suffix: changeme - hash_path_prefix: changeme - syslog_host: 10.1.1.1:514 - -# Global hosts -# Hosts can be defined in the sections below or here, the advantage of -# defining them here is that they can be reused in multiple sections below. -# -# When need to reference a global host use the form: -# - name: / -hosts: - - host: 10.0.0.4 - drive: sda - drives: /srv/disk - region: 0 - zone: 0 - weight: 100 - repl_ip: 192.168.0.4 - repl_port: 55555 - - host: 10.0.0.5 - drive: sda - drives: /srv/disk - region: 0 - zone: 1 - weight: 100 - -proxy: - memcache_servers: - - 192.168.100.7:11211 - authtoken: - active: true - delay_auth_decision: true - auth_version: v2.0 - auth_host: 192.168.100.7 - auth_port: 35357 - auth_protocol: https - auth_uri: http://192.168.100.7:5000/ - admin_tenant_name: service - admin_user: swift - admin_password: ADMIN - hosts: - - host: - -account: - repl_number: 3 - min_part_hours: 1 - port: 6002 - hosts: - - host: 10.0.0.2 - drive: sdba - region: 0 - zone: 0 - weight: 100 - -container: - repl_number: 3 - port: 6001 - hosts: - - host: 10.0.0.3 - drive: sda - region: 0 - zone: 0 - weight: 100 - -storage_policies: - default: gold - port: 6000 - policies: - - name: gold - index: 0 - type: replication - repl_number: 3 - min_part_hours: 1 - hosts: - - name: 10.0.0.4/sda - - name: 10.0.0.5/sda - - host: 10.0.0.6 - drive: sdb - region: 1 - zone: 0 - weight: 50 - - host: 10.0.0.7 - drive: sdb - region: 1 - zone: 1 - weight: 50 - - name: silver - index: 1 - type: replication - repl_number: 2 - depricated: True - hosts: - - name: 10.0.0.4/sda - - name: 10.0.0.5/sda diff --git a/scripts/swift_inventory.py b/scripts/swift_inventory.py deleted file mode 100644 index aefbaea0d7..0000000000 --- a/scripts/swift_inventory.py +++ /dev/null @@ -1,361 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function - -import datetime -import sys -import yaml - -from optparse import OptionParser -from os.path import exists, isdir, join - -VERSION = '0.1' -USAGE = "usage: %prog [options] -s " - -DEFAULT_PART_POWER = 8 -DEFAULT_REPL_NUM = 3 -DEFAULT_REGION = 0 -DEFAULT_ZONE = 0 -DEFAULT_WEIGHT = 100 -DEFAULT_DRIVE = "sda" -DEFAULT_DRIVES = "/srv/disk" -DEFAULT_OUTPUT_DIR = "/etc/ansible" -DEFAULT_OUTPUT_FILENAME = "hosts" - -RETURN_NOT_DEFINED = 3 - -# FILE formatted strings -HEADER = """# This file was generated using the %s version %s at %s -[local] -localhost ansible_connection=local - -[proxy]""" - -CATCH_ALL_GROUPS = """ -[object:children] -storagepolicy - -[swift:children] -proxy -account -container -object -local - -[swift:vars]""" - -DRIVE_FORMAT = "%(host)s drive=%(drive)s region=%(region)s zone=%(zone)s " -DRIVE_FORMAT += "weight=%(weight)s drives=%(drives)s extra_data=%(extra_data)s" - -DEFAULT_AUTHTOKEN_SETTINGS = { - 'auth_version': 'v2.0', - 'auth_host': 'keystone', - 'auth_port': '35357', - 'auth_protocol': 'https', - 'admin_tenant_name': 'service', - 'admin_user': 'swift', - 'admin_password': 'ADMIN', -} - -DATA_ORDER = [ - NAME, - INDEX, - DEPRICATED, - TYPE, -] = [ - 'name', - 'index', - 'depricated', - 'type', -] - - -class DriveException(Exception): - pass - - -def main(setup, verbose=False, dry_run=False, overwrite=True): - # Parse the setup file, which should be yaml - _swift = {} - _drives = {} - try: - with open(setup) as yaml_stream: - _swift = yaml.load(yaml_stream) - except Exception as err: - print("ERROR: Failed to yaml failure: %s", err) - return 2 - - def _section_defined(section): - if section not in _swift: - print("ERROR: no swift section defined") - return False - return True - - def _get_output_fd(filename): - if dry_run: - return None - elif not overwrite and exists(filename): - i = 1 - while exists("%s_%d" % (filename, i)): - i += 1 - return open("%s_%d" % (filename, i), 'w') - else: - return open(filename, 'w') - - def _write_to_file(fd, data): - if not fd or verbose: - print(data) - - if fd: - if not data.endswith('\n'): - data += "\n" - fd.write(data) - fd.flush() - - def _get_drive(drive, extra_data={}): - if 'name' in drive: - # We are dealing with a global host reference - key = drive['name'] - if key not in _drives: - raise DriveException("Drive referenced doesn't exist") - else: - # Not a referenced host. - _drive = { - 'drive': DEFAULT_DRIVE, - 'region': DEFAULT_REGION, - 'zone': DEFAULT_ZONE, - 'weight': DEFAULT_WEIGHT, - 'drives': DEFAULT_DRIVES, - 'extra_data': ''} - - if "drive" not in drive: - drive["drive"] = DEFAULT_DRIVE - - key = "%(host)s/%(drive)s" % drive - - if extra_data: - data_str = ":".join([str(extra_data.get(i, '')) for i in - DATA_ORDER]) - else: - data_str = "" - if key in _drives: - if not _drives[key]['extra_data'] and extra_data: - _drives[key]['extra_data'] = data_str - elif _drives[key]['extra_data'] and extra_data: - _drives[key]['extra_data'] += ";%s" % (data_str) - return _drives[key] - else: - _drive.update(drive) - _drive['extra_data'] = data_str - _drives[key] = _drive - return _drive - - # First attempt to get swift settings - if not _section_defined("swift"): - return RETURN_NOT_DEFINED - - swift_options = [ - "part_power=%s" % (_swift['swift'].get('part_power', - DEFAULT_PART_POWER)), - "user=%s" % (_swift['swift'].get('user', 'swift')), - "hash_path_suffix=%s" % (_swift['swift'].get("hash_path_suffix")), - "hash_path_prefix=%s" % (_swift['swift'].get("hash_path_prefix")), - "syslog_host=%s" % (_swift['swift'].get('syslog_host', - 'localhost:514')), - ] - output_path = _swift['swift'].get("output_directory", DEFAULT_OUTPUT_DIR) - output_file = _swift['swift'].get("output_filename", - DEFAULT_OUTPUT_FILENAME) - if not isdir(output_path): - print("Outdir path '%s' doesn't exist", output_path) - return 4 - - output_file = join(output_path, output_file) - output_fd = _get_output_fd(output_file) - - n = datetime.datetime.now() - _write_to_file(output_fd, HEADER % (__file__, VERSION, n.ctime())) - - # Global hosts - if _section_defined("hosts"): - for host in _swift["hosts"]: - _get_drive(host) - - if not _section_defined("proxy"): - return RETURN_NOT_DEFINED - - # Parse proxies - # TODO: Add read anfinity and pipeline here? - for proxy in _swift["proxy"]["hosts"]: - _write_to_file(output_fd, "%s" % (proxy["host"])) - _write_to_file(output_fd, "\n[proxy:vars]") - _mc_servers = _swift["proxy"].get('memcache_servers') - memcache_servers = ",".join(_mc_servers) if _mc_servers else \ - '127.0.0.1:11211' - _write_to_file(output_fd, "memcache_servers=%s" % (memcache_servers)) - _at = _swift["proxy"].get('authtoken') - if _at: - authtoken = DEFAULT_AUTHTOKEN_SETTINGS - authtoken.update(_at) - at_active = authtoken.get("active", False) - if at_active: - _write_to_file(output_fd, "authtoken_active=true") - _write_to_file(output_fd, "delay_auth_decision=" - "%(delay_auth_decision)s" % authtoken) - _write_to_file(output_fd, "auth_version=" - "%(auth_version)s" % authtoken) - _write_to_file(output_fd, "auth_host=" - "%(auth_host)s" % authtoken) - _write_to_file(output_fd, "auth_port=" - "%(auth_port)s" % authtoken) - _write_to_file(output_fd, "auth_protocol=" - "%(auth_protocol)s" % authtoken) - _write_to_file(output_fd, "auth_uri=" - "%(auth_uri)s" % authtoken) - _write_to_file(output_fd, "admin_tenant_name=" - "%(admin_tenant_name)s" % authtoken) - _write_to_file(output_fd, "admin_user=" - "%(admin_user)s" % authtoken) - _write_to_file(output_fd, "admin_password=" - "%(admin_password)s" % authtoken) - else: - _write_to_file(output_fd, "authtoken_active=false") - - _write_to_file(output_fd, "\n[account]") - - if not _section_defined("account"): - return RETURN_NOT_DEFINED - - for account in _swift["account"]["hosts"]: - data = _get_drive(account) - _write_to_file(output_fd, DRIVE_FORMAT % data) - - _write_to_file(output_fd, "\n[account:vars]") - repl_num = _swift["account"].get("repl_number", DEFAULT_REPL_NUM) - _write_to_file(output_fd, "repl_number=%d" % (repl_num)) - port = _swift["account"].get("port", 6002) - _write_to_file(output_fd, "account_server_port=%d" % (port)) - - # Container section - _write_to_file(output_fd, "\n[container]") - - if not _section_defined("container"): - return RETURN_NOT_DEFINED - - for container in _swift["container"]["hosts"]: - data = _get_drive(container) - _write_to_file(output_fd, DRIVE_FORMAT % data) - - _write_to_file(output_fd, "\n[container:vars]") - repl_num = _swift["container"].get("repl_number", DEFAULT_REPL_NUM) - _write_to_file(output_fd, "repl_number=%d" % (repl_num)) - port = _swift["container"].get("port", 6001) - _write_to_file(output_fd, "container_server_port=%d" % (port)) - - # Objects / Storage polices - _storage_policies = {} - _storage_policies_idx = {} - if not _section_defined("storage_policies"): - return RETURN_NOT_DEFINED - - if "policies" not in _swift["storage_policies"]: - print("ERROR: No storage policies defined") - return 4 - - policy_hosts = {} - for policy in _swift["storage_policies"]["policies"]: - if policy["name"] in _storage_policies: - print("ERROR: Storage policy '%s' already defined" % policy["name"]) - return 5 - - if policy["index"] in _storage_policies_idx: - print("ERROR: Storage policy index '%s' already defined" % - policy["index"]) - return 5 - - _storage_policies[policy['name']] = "storagepolicy_%(name)s" % policy - _storage_policies_idx[policy['index']] = policy["name"] - - policy_type = policy.get("type", 'replication') - _host_data = { - NAME: policy['name'], - INDEX: policy['index'], - TYPE: policy_type, - DEPRICATED: policy.get("depricated", False), - } - # print the storage policy hosts. - for drive in policy.get("hosts", []): - _get_drive(drive, _host_data) - - policy_hosts[policy['name']] = policy.get("hosts") - _write_to_file(output_fd, - "\n[%s:vars]" % (_storage_policies[policy['name']])) - default = policy.get("default", False) - if default: - _write_to_file(output_fd, "default=%s" % (policy['name'])) - - if policy_type == 'replication': - repl_num = policy.get("repl_number", DEFAULT_REPL_NUM) - _write_to_file(output_fd, "repl_num=%d" % (repl_num)) - - # now write out the drives. - for policy in _swift["storage_policies"]["policies"]: - _write_to_file(output_fd, - "\n[%s]" % (_storage_policies[policy['name']])) - for drive in policy_hosts[policy['name']]: - data = _get_drive(drive) - _write_to_file(output_fd, DRIVE_FORMAT % data) - - # Write out the storage policy catch all group - _write_to_file(output_fd, "\n[storagepolicy:children]") - for name, longname in _storage_policies.items(): - _write_to_file(output_fd, "%s" % (longname)) - - _write_to_file(output_fd, "\n[storagepolicy:vars]") - if 'default' in _swift["storage_policies"]: - default_sp = _swift["storage_policies"]["default"] - if default_sp in _storage_policies: - _write_to_file(output_fd, "default=%s" % (default_sp)) - elif default_sp in _storage_policies_idx: - _write_to_file(output_fd, - "default=%s" % (_storage_policies_idx[default_sp])) - else: - print("ERROR: Default storage policy '%s' doesn't exist", - default_sp) - port = _swift["storage_policies"].get("port", 6000) - _write_to_file(output_fd, "object_server_port=%d" % (port)) - - # Write out the object and swift catchall groups - _write_to_file(output_fd, CATCH_ALL_GROUPS) - - # Now write out the global swift options that is gathered in the file - for option in swift_options: - _write_to_file(output_fd, option) - - # Done - if output_fd: - output_fd.flush() - output_fd.close() - return 0 - -if __name__ == "__main__": - parser = OptionParser(USAGE) - parser.add_option("-s", "--setup", dest="setup", - help="Specify the swift setup file.", metavar="FILE") - parser.add_option("-v", "--verbose", action="store_true", dest="verbose", - default=False, help="Be more verbose") - parser.add_option("-d", "--dryrun", action="store_true", dest="dry_run", - default=False, help="Print result out to stdout.") - parser.add_option("-C", "--copy", action="store_false", dest="overwrite", - default=True, help="Make a copy if inventory file exists") - parser.add_option("-i", "--import", dest="ring_folder", metavar="FILE", - help="Attempt to build a swift setup file" - " from the Swift builder files. Pass directory here") - - options, args = parser.parse_args(sys.argv[1:]) - if not options.setup or not exists(options.setup): - print("Swift setup file not found or doesn't exist") - parser.print_help() - sys.exit(1) - - sys.exit(main(options.setup, options.verbose, options.dry_run, - options.overwrite)) diff --git a/scripts/swift_rings.py b/scripts/swift_rings.py deleted file mode 100644 index 3c32845ae1..0000000000 --- a/scripts/swift_rings.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function -from optparse import OptionParser -from os.path import exists -from swift.cli.ringbuilder import main as rb_main - -import sys -import threading -import yaml - -USAGE = "usage: %prog -s " - -DEFAULT_PART_POWER = 10 -DEFAULT_REPL = 3 -DEFAULT_MIN_PART_HOURS = 1 -DEFAULT_HOST_DRIVES = '/srv/drive/' -DEFAULT_HOST_DRIVE = '/sdb' -DEFAULT_HOST_ZONE = 0 -DEFAULT_HOST_WEIGHT = 1 -DEFAULT_ACCOUNT_PORT = 6002 -DEFAULT_CONTAINER_PORT = 6001 -DEFAULT_OBJECT_PORT = 6000 -DEFAULT_SECTION_PORT = { - 'account': DEFAULT_ACCOUNT_PORT, - 'container': DEFAULT_CONTAINER_PORT, - 'object': DEFAULT_OBJECT_PORT, -} - - -def create_buildfile(build_file, part_power, repl, min_part_hours): - run_and_wait(rb_main, ["swift-ring-builder", build_file, "create", - part_power, repl, min_part_hours]) - - -def add_host_to_ring(build_file, host): - host_str = "" - if host.get('region') is not None: - host_str += 'r%(region)d' % host - host_str += "z%d" % (host.get('zone', DEFAULT_HOST_ZONE)) - host_str += "-%(host)s:%(port)d" % host - if host.get('repl_port'): - r_ip = host.get('repl_ip', host['host']) - host_str += "R%s:%d" % (r_ip, host['repl_port']) - host_str += "/%(drive)s" % host - - weight = host.get('weight', DEFAULT_HOST_WEIGHT) - run_and_wait(rb_main, ["swift-ring-builder", build_file, 'add', - host_str, str(weight)]) - - -def run_and_wait(func, *args): - t = threading.Thread(target=func, args=args) - t.start() - return t.join() - - -def has_section(conf, section): - return True if conf.get(section) else False - - -def check_section(conf, section): - if not has_section(conf, section): - print("Section %s doesn't exist" % (section)) - sys.exit(2) - - -def build_ring(section, conf, part_power, hosts): - # Create the build file - build_file = "%s.builder" % (section) - repl = conf.get('repl_number', DEFAULT_REPL) - min_part_hours = conf.get('min_part_hours', - DEFAULT_MIN_PART_HOURS) - create_buildfile(build_file, part_power, repl, min_part_hours) - - # Add the hosts - if not has_section(conf, 'hosts') or len(conf.get('hosts')) == 0: - print("No hosts/drives assigned to the %s ring" % section) - sys.exit(3) - - section_key = section.split('-')[0] - service_port = conf.get('port', DEFAULT_SECTION_PORT[section_key]) - for host in conf['hosts']: - if 'name' in host: - if host['name'] not in hosts: - print("Host %(name) reference not found." % host) - sys.exit(3) - host = hosts[host['name']] - else: - if 'drive' not in host: - host['drive'] = DEFAULT_HOST_DRIVE - host['port'] = service_port - add_host_to_ring(build_file, host) - - # Rebalance ring - run_and_wait(rb_main, ["swift-ring-builder", build_file, "rebalance"]) - # rb_main(("swift-ring-builder", build_file, "rebalance")) - - -def main(setup): - # load the yaml file - try: - with open(setup) as yaml_stream: - _swift = yaml.load(yaml_stream) - except Exception as ex: - print("Failed to load yaml string %s" % (ex)) - return 1 - - _hosts = {} - - if _swift.get("hosts"): - for host in _swift['hosts']: - if not host.get('drive'): - host['drive'] = DEFAULT_HOST_DRIVE - key = "%(host)s/%(drive)s" % host - if key in _hosts: - print("%(host)s already definined" % host) - return 1 - _hosts[key] = host - - check_section(_swift, 'swift') - part_power = _swift['swift'].get('part_power', DEFAULT_PART_POWER) - - # Create account ring - check_section(_swift, 'account') - build_ring('account', _swift['account'], part_power, _hosts) - - # Create container ring - check_section(_swift, 'container') - build_ring('container', _swift['container'], part_power, _hosts) - - # Create object rings (storage policies) - check_section(_swift, 'storage_policies') - check_section(_swift['storage_policies'], 'policies') - indexes = set() - for sp in _swift['storage_policies']['policies']: - if sp['index'] in indexes: - print("Storage Policy index %d already in use" % (sp['index'])) - return 4 - buildfilename = 'object-%d' % (sp['index']) - indexes.add(sp['index']) - if 'port' not in sp: - sp['port'] = _swift['storage_policies'].get('port', - DEFAULT_OBJECT_PORT) - build_ring(buildfilename, sp, part_power, _hosts) - -if __name__ == "__main__": - parser = OptionParser(USAGE) - parser.add_option("-s", "--setup", dest="setup", - help="Specify the swift setup file.", metavar="FILE", - default="/etc/swift/swift_inventory.yml") - - options, args = parser.parse_args(sys.argv[1:]) - if options.setup and not exists(options.setup): - print("Swift setup file not found or doesn't exist") - parser.print_help() - sys.exit(1) - - sys.exit(main(options.setup))