commit 0ccf3e144306c85a86f8f09a08d7ccc1213780bf Author: Joshua Harlow Date: Wed Jan 11 12:47:33 2012 -0800 Initial public commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..05bdf812 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +*.pyc +.settings/ +.venv +build/ +dist/ +doc/source/sourcecode +*.db +.*.swp +*.log +*.pid +pidfile +*.komodoproject +.coverage +.DS_Store diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 00000000..2bcdf119 --- /dev/null +++ b/AUTHORS @@ -0,0 +1,26 @@ +V2 +-- +Joshua Harlow +Ken Thomas + +V1 +-- +Andy Smith +Anthony Young +Brad Hall +Chmouel Boudjnah +Dean Troyer +Devin Carlen +Eddie Hebert +Jake Dahn +James E. Blair +Jason Cannavale +Jay Pipes +Jesse Andrews +Justin Shepherd +Kiall Mac Innes +Scott Moser +Todd Willey +Tres Henry +Vishvananda Ishaya +Yun Mao diff --git a/README.md b/README.md new file mode 100644 index 00000000..a695c077 --- /dev/null +++ b/README.md @@ -0,0 +1,59 @@ +Devstack v2 is a set of python scripts and utilities to quickly deploy an OpenStack cloud. + +# Goals + +* To quickly build dev OpenStack environments in a clean environment (as well as start, stop, and uninstall those environments) +* To describe working configurations of OpenStack (which code branches work together? what do config files look like for those branches? what packages are needed for installation?) +* To make it easier for developers to dive into OpenStack so that they can productively contribute without having to understand every part of the system at once +* To make it easy to prototype cross-project features + +Read more at + +IMPORTANT: Be sure to carefully read *stack* and any other scripts you execute before you run them, as they install software and may alter your networking configuration. We strongly recommend that you run stack in a clean and disposable vm when you are first getting started. + +# Help + +In order to determine what *stack* can do run the following. + + ./stack --help + +This will typically produce: + + $ ./stack --help + Usage: stack [options] + + Options: + -h, --help show this help message and exit + -a ACTION, --action=ACTION + action to perform, ie (start, stop, install, + uninstall) + -d DIR, --directory=DIR + root DIR for new components or DIR with existing + components (ACTION dependent) + -c COMPONENT, --component=COMPONENT + stack component, ie (rabbit, db, nova, keystone, + horizon, quantum, glance, swift) + +# Actions + +You will note that *stack* can uninstall, install, start and stop openstack components. Typically the interaction would be that you install a set of components and then start them. + +# Config + +If you want to change which devstack branches or other various devstack configurations. +Check out *conf/stack.ini* for various configuration settings applied (branches, git repositories...). +When you see a configuration in *stack.ini* with the format *${NAME:-DEFAULT}* this means that the environment the *stack* script is running in while be referred to and if that value exists it will be used (otherwise the *DEFAULT* will be used). +Also check out *conf/* for various component specific settings and *conf/pkgs* for package listings (with versions) for various distributions. + +# To start a dev cloud (Installing in a dedicated, disposable vm is safer than installing on your dev machine!): + + ./stack -a install -d $HOME/openstack && ./stack -a start -d $HOME/openstack + +When the script finishes executing, you should be able to access OpenStack endpoints, like so: + +* Horizon: http://myhost/ +* Keystone: http://myhost:5000/v2.0/ + +# Customizing + +You can override environment variables used in *stack* by editing *stack.ini* or by sourcing a file that contains your overrides before your run *stack*. diff --git a/conf/apache/000-default.template b/conf/apache/000-default.template new file mode 100644 index 00000000..1d7380d9 --- /dev/null +++ b/conf/apache/000-default.template @@ -0,0 +1,28 @@ + + WSGIScriptAlias / %HORIZON_DIR%/openstack-dashboard/dashboard/wsgi/django.wsgi + WSGIDaemonProcess horizon user=%USER% group=%USER% processes=3 threads=10 + SetEnv APACHE_RUN_USER %USER% + SetEnv APACHE_RUN_GROUP %USER% + WSGIProcessGroup horizon + + DocumentRoot %HORIZON_DIR%/.blackhole/ + Alias /media %HORIZON_DIR%/openstack-dashboard/dashboard/static + Alias /vpn /opt/stack/vpn + + + Options FollowSymLinks + AllowOverride None + + + + Options Indexes FollowSymLinks MultiViews + AllowOverride None + Order allow,deny + allow from all + + + ErrorLog /var/log/apache2/error.log + LogLevel warn + CustomLog /var/log/apache2/access.log combined + + diff --git a/conf/apt/sources.list b/conf/apt/sources.list new file mode 100644 index 00000000..77a1bfb5 --- /dev/null +++ b/conf/apt/sources.list @@ -0,0 +1,9 @@ +deb http://mirror.rackspace.com/ubuntu/ %DIST% main restricted +deb http://mirror.rackspace.com/ubuntu/ %DIST%-updates main restricted +deb http://mirror.rackspace.com/ubuntu/ %DIST% universe +deb http://mirror.rackspace.com/ubuntu/ %DIST%-updates universe +deb http://mirror.rackspace.com/ubuntu/ %DIST% multiverse +deb http://mirror.rackspace.com/ubuntu/ %DIST%-updates multiverse +deb http://security.ubuntu.com/ubuntu %DIST%-security main restricted +deb http://security.ubuntu.com/ubuntu %DIST%-security universe +deb http://security.ubuntu.com/ubuntu %DIST%-security multiverse diff --git a/conf/glance/glance-api.conf b/conf/glance/glance-api.conf new file mode 100644 index 00000000..6c670b56 --- /dev/null +++ b/conf/glance/glance-api.conf @@ -0,0 +1,184 @@ +[DEFAULT] +# Show more verbose log output (sets INFO log level output) +verbose = True + +# Show debugging output in logs (sets DEBUG log level output) +debug = True + +# Which backend store should Glance use by default is not specified +# in a request to add a new image to Glance? Default: 'file' +# Available choices are 'file', 'swift', and 's3' +default_store = file + +# Address to bind the API server +bind_host = 0.0.0.0 + +# Port the bind the API server to +bind_port = 9292 + +# Address to find the registry server +registry_host = 0.0.0.0 + +# Port the registry server is listening on +registry_port = 9191 + +# Log to this file. Make sure you do not set the same log +# file for both the API and registry servers! +#log_file = %DEST%/glance/api.log + +# Send logs to syslog (/dev/log) instead of to file specified by `log_file` +use_syslog = %SYSLOG% + +# ============ Notification System Options ===================== + +# Notifications can be sent when images are create, updated or deleted. +# There are three methods of sending notifications, logging (via the +# log_file directive), rabbit (via a rabbitmq queue) or noop (no +# notifications sent, the default) +notifier_strategy = noop + +# Configuration options if sending notifications via rabbitmq (these are +# the defaults) +rabbit_host = localhost +rabbit_port = 5672 +rabbit_use_ssl = false +rabbit_userid = guest +rabbit_password = guest +rabbit_virtual_host = / +rabbit_notification_topic = glance_notifications + +# ============ Filesystem Store Options ======================== + +# Directory that the Filesystem backend store +# writes image data to +filesystem_store_datadir = %DEST%/glance/images/ + +# ============ Swift Store Options ============================= + +# Address where the Swift authentication service lives +swift_store_auth_address = 127.0.0.1:8080/v1.0/ + +# User to authenticate against the Swift authentication service +swift_store_user = jdoe + +# Auth key for the user authenticating against the +# Swift authentication service +swift_store_key = a86850deb2742ec3cb41518e26aa2d89 + +# Container within the account that the account should use +# for storing images in Swift +swift_store_container = glance + +# Do we create the container if it does not exist? +swift_store_create_container_on_put = False + +# What size, in MB, should Glance start chunking image files +# and do a large object manifest in Swift? By default, this is +# the maximum object size in Swift, which is 5GB +swift_store_large_object_size = 5120 + +# When doing a large object manifest, what size, in MB, should +# Glance write chunks to Swift? This amount of data is written +# to a temporary disk buffer during the process of chunking +# the image file, and the default is 200MB +swift_store_large_object_chunk_size = 200 + +# Whether to use ServiceNET to communicate with the Swift storage servers. +# (If you aren't RACKSPACE, leave this False!) +# +# To use ServiceNET for authentication, prefix hostname of +# `swift_store_auth_address` with 'snet-'. +# Ex. https://example.com/v1.0/ -> https://snet-example.com/v1.0/ +swift_enable_snet = False + +# ============ S3 Store Options ============================= + +# Address where the S3 authentication service lives +s3_store_host = 127.0.0.1:8080/v1.0/ + +# User to authenticate against the S3 authentication service +s3_store_access_key = <20-char AWS access key> + +# Auth key for the user authenticating against the +# S3 authentication service +s3_store_secret_key = <40-char AWS secret key> + +# Container within the account that the account should use +# for storing images in S3. Note that S3 has a flat namespace, +# so you need a unique bucket name for your glance images. An +# easy way to do this is append your AWS access key to "glance". +# S3 buckets in AWS *must* be lowercased, so remember to lowercase +# your AWS access key if you use it in your bucket name below! +s3_store_bucket = glance + +# Do we create the bucket if it does not exist? +s3_store_create_bucket_on_put = False + +# ============ Image Cache Options ======================== + +image_cache_enabled = False + +# Directory that the Image Cache writes data to +# Make sure this is also set in glance-pruner.conf +image_cache_datadir = /var/lib/glance/image-cache/ + +# Number of seconds after which we should consider an incomplete image to be +# stalled and eligible for reaping +image_cache_stall_timeout = 86400 + +# ============ Delayed Delete Options ============================= + +# Turn on/off delayed delete +delayed_delete = False + +# Delayed delete time in seconds +scrub_time = 43200 + +# Directory that the scrubber will use to remind itself of what to delete +# Make sure this is also set in glance-scrubber.conf +scrubber_datadir = /var/lib/glance/scrubber + +[pipeline:glance-api] +#pipeline = versionnegotiation context apiv1app +# NOTE: use the following pipeline for keystone +pipeline = versionnegotiation authtoken auth-context apiv1app + +# To enable Image Cache Management API replace pipeline with below: +# pipeline = versionnegotiation context imagecache apiv1app +# NOTE: use the following pipeline for keystone auth (with caching) +# pipeline = versionnegotiation authtoken auth-context imagecache apiv1app + +[app:apiv1app] +paste.app_factory = glance.common.wsgi:app_factory +glance.app_factory = glance.api.v1.router:API + +[filter:versionnegotiation] +paste.filter_factory = glance.common.wsgi:filter_factory +glance.filter_factory = glance.api.middleware.version_negotiation:VersionNegotiationFilter + +[filter:cache] +paste.filter_factory = glance.common.wsgi:filter_factory +glance.filter_factory = glance.api.middleware.cache:CacheFilter + +[filter:cachemanage] +paste.filter_factory = glance.common.wsgi:filter_factory +glance.filter_factory = glance.api.middleware.cache_manage:CacheManageFilter + +[filter:context] +paste.filter_factory = glance.common.wsgi:filter_factory +glance.filter_factory = glance.common.context:ContextMiddleware + +[filter:authtoken] +paste.filter_factory = keystone.middleware.auth_token:filter_factory +service_protocol = http +service_host = 127.0.0.1 +service_port = 5000 +auth_host = 127.0.0.1 +auth_port = 35357 +auth_protocol = http +auth_uri = http://127.0.0.1:5000/ +admin_token = %SERVICE_TOKEN% + +[filter:auth-context] +paste.filter_factory = glance.common.wsgi:filter_factory +glance.filter_factory = keystone.middleware.glance_auth_token:KeystoneContextMiddleware diff --git a/conf/glance/glance-registry.conf b/conf/glance/glance-registry.conf new file mode 100644 index 00000000..e732e869 --- /dev/null +++ b/conf/glance/glance-registry.conf @@ -0,0 +1,74 @@ +[DEFAULT] +# Show more verbose log output (sets INFO log level output) +verbose = True + +# Show debugging output in logs (sets DEBUG log level output) +debug = True + +# Address to bind the registry server +bind_host = 0.0.0.0 + +# Port the bind the registry server to +bind_port = 9191 + +# Log to this file. Make sure you do not set the same log +# file for both the API and registry servers! +#log_file = %DEST%/glance/registry.log + +# Where to store images +filesystem_store_datadir = %DEST%/glance/images + +# Send logs to syslog (/dev/log) instead of to file specified by `log_file` +use_syslog = %SYSLOG% + +# SQLAlchemy connection string for the reference implementation +# registry server. Any valid SQLAlchemy connection string is fine. +# See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine +sql_connection = %SQL_CONN% + +# Period in seconds after which SQLAlchemy should reestablish its connection +# to the database. +# +# MySQL uses a default `wait_timeout` of 8 hours, after which it will drop +# idle connections. This can result in 'MySQL Gone Away' exceptions. If you +# notice this, you can lower this value to ensure that SQLAlchemy reconnects +# before MySQL can drop the connection. +sql_idle_timeout = 3600 + +# Limit the api to return `param_limit_max` items in a call to a container. If +# a larger `limit` query param is provided, it will be reduced to this value. +api_limit_max = 1000 + +# If a `limit` query param is not provided in an api request, it will +# default to `limit_param_default` +limit_param_default = 25 + +[pipeline:glance-registry] +#pipeline = context registryapp +# NOTE: use the following pipeline for keystone +pipeline = authtoken auth-context context registryapp + +[app:registryapp] +paste.app_factory = glance.common.wsgi:app_factory +glance.app_factory = glance.registry.api.v1:API + +[filter:context] +context_class = glance.registry.context.RequestContext +paste.filter_factory = glance.common.wsgi:filter_factory +glance.filter_factory = glance.common.context:ContextMiddleware + +[filter:authtoken] +paste.filter_factory = keystone.middleware.auth_token:filter_factory +service_protocol = http +service_host = 127.0.0.1 +service_port = 5000 +auth_host = 127.0.0.1 +auth_port = 35357 +auth_protocol = http +auth_uri = http://127.0.0.1:5000/ +admin_token = %SERVICE_TOKEN% + +[filter:auth-context] +context_class = glance.registry.context.RequestContext +paste.filter_factory = glance.common.wsgi:filter_factory +glance.filter_factory = keystone.middleware.glance_auth_token:KeystoneContextMiddleware diff --git a/conf/horizon/horizon_settings.py b/conf/horizon/horizon_settings.py new file mode 100644 index 00000000..05ddfe7b --- /dev/null +++ b/conf/horizon/horizon_settings.py @@ -0,0 +1,110 @@ +import os + +DEBUG = True +TEMPLATE_DEBUG = DEBUG +PROD = False +USE_SSL = False + +LOCAL_PATH = os.path.dirname(os.path.abspath(__file__)) + +# FIXME: We need to change this to mysql, instead of sqlite. +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(LOCAL_PATH, 'dashboard_openstack.sqlite3'), + 'TEST_NAME': os.path.join(LOCAL_PATH, 'test.sqlite3'), + }, +} + +# The default values for these two settings seem to cause issues with apache +CACHE_BACKEND = 'dummy://' +SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db' + +# Send email to the console by default +EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +# Or send them to /dev/null +#EMAIL_BACKEND = 'django.core.mail.backends.dummy.EmailBackend' + +# django-mailer uses a different settings attribute +MAILER_EMAIL_BACKEND = EMAIL_BACKEND + +# Configure these for your outgoing email host +# EMAIL_HOST = 'smtp.my-company.com' +# EMAIL_PORT = 25 +# EMAIL_HOST_USER = 'djangomail' +# EMAIL_HOST_PASSWORD = 'top-secret!' + +HORIZON_CONFIG = { + 'dashboards': ('nova', 'syspanel', 'settings',), + 'default_dashboard': 'nova', + 'user_home': 'dashboard.views.user_home', +} + +OPENSTACK_HOST = "127.0.0.1" +OPENSTACK_KEYSTONE_URL = "http://%s:5000/v2.0" % OPENSTACK_HOST +# FIXME: this is only needed until keystone fixes its GET /tenants call +# so that it doesn't return everything for admins +OPENSTACK_KEYSTONE_ADMIN_URL = "http://%s:35357/v2.0" % OPENSTACK_HOST +OPENSTACK_KEYSTONE_DEFAULT_ROLE = "Member" + +SWIFT_PAGINATE_LIMIT = 100 + +# Configure quantum connection details for networking +QUANTUM_ENABLED = False +QUANTUM_URL = '%s' % OPENSTACK_HOST +QUANTUM_PORT = '9696' +QUANTUM_TENANT = '1234' +QUANTUM_CLIENT_VERSION='0.1' + +# If you have external monitoring links, eg: +# EXTERNAL_MONITORING = [ +# ['Nagios','http://foo.com'], +# ['Ganglia','http://bar.com'], +# ] + +#LOGGING = { +# 'version': 1, +# # When set to True this will disable all logging except +# # for loggers specified in this configuration dictionary. Note that +# # if nothing is specified here and disable_existing_loggers is True, +# # django.db.backends will still log unless it is disabled explicitly. +# 'disable_existing_loggers': False, +# 'handlers': { +# 'null': { +# 'level': 'DEBUG', +# 'class': 'django.utils.log.NullHandler', +# }, +# 'console': { +# # Set the level to "DEBUG" for verbose output logging. +# 'level': 'INFO', +# 'class': 'logging.StreamHandler', +# }, +# }, +# 'loggers': { +# # Logging from django.db.backends is VERY verbose, send to null +# # by default. +# 'django.db.backends': { +# 'handlers': ['null'], +# 'propagate': False, +# }, +# 'horizon': { +# 'handlers': ['console'], +# 'propagate': False, +# }, +# 'novaclient': { +# 'handlers': ['console'], +# 'propagate': False, +# }, +# 'keystoneclient': { +# 'handlers': ['console'], +# 'propagate': False, +# }, +# 'nose.plugins.manager': { +# 'handlers': ['console'], +# 'propagate': False, +# } +# } +#} + +# How much ram on each compute host? +COMPUTE_HOST_RAM_GB = 16 diff --git a/conf/keystone/keystone.conf b/conf/keystone/keystone.conf new file mode 100644 index 00000000..0c0d0e26 --- /dev/null +++ b/conf/keystone/keystone.conf @@ -0,0 +1,116 @@ +[DEFAULT] +# Show more verbose log output (sets INFO log level output) +verbose = False + +# Show debugging output in logs (sets DEBUG log level output) +debug = False + +# Which backend store should Keystone use by default. +# Default: 'sqlite' +# Available choices are 'sqlite' [future will include LDAP, PAM, etc] +default_store = sqlite + +# Log to this file. Make sure you do not set the same log +# file for both the API and registry servers! +log_file = %DEST%/keystone/keystone.log + +# List of backends to be configured +backends = keystone.backends.sqlalchemy +#For LDAP support, add: ,keystone.backends.ldap + +# Dictionary Maps every service to a header.Missing services would get header +# X_(SERVICE_NAME) Key => Service Name, Value => Header Name +service-header-mappings = { + 'nova' : 'X-Server-Management-Url', + 'swift' : 'X-Storage-Url', + 'cdn' : 'X-CDN-Management-Url'} + +#List of extensions currently supported +extensions= osksadm,oskscatalog + +# Address to bind the API server +# TODO Properties defined within app not available via pipeline. +service_host = 0.0.0.0 + +# Port the bind the API server to +service_port = 5000 + +# SSL for API server +service_ssl = False + +# Address to bind the Admin API server +admin_host = 0.0.0.0 + +# Port the bind the Admin API server to +admin_port = 35357 + +# SSL for API Admin server +admin_ssl = False + +# Keystone certificate file (modify as needed) +# Only required if *_ssl is set to True +certfile = /etc/keystone/ssl/certs/keystone.pem + +# Keystone private key file (modify as needed) +# Only required if *_ssl is set to True +keyfile = /etc/keystone/ssl/private/keystonekey.pem + +# Keystone trusted CA certificates (modify as needed) +# Only required if *_ssl is set to True +ca_certs = /etc/keystone/ssl/certs/ca.pem + +# Client certificate required +# Only relevant if *_ssl is set to True +cert_required = True + +#Role that allows to perform admin operations. +keystone-admin-role = Admin + +#Role that allows to perform service admin operations. +keystone-service-admin-role = KeystoneServiceAdmin + +#Tells whether password user need to be hashed in the backend +hash-password = True + +[keystone.backends.sqlalchemy] +# SQLAlchemy connection string for the reference implementation registry +# server. Any valid SQLAlchemy connection string is fine. +# See: http://bit.ly/ideIpI +sql_connection = %SQL_CONN% +backend_entities = ['UserRoleAssociation', 'Endpoints', 'Role', 'Tenant', + 'User', 'Credentials', 'EndpointTemplates', 'Token', + 'Service'] + +# Period in seconds after which SQLAlchemy should reestablish its connection +# to the database. +sql_idle_timeout = 30 + +[pipeline:admin] +pipeline = + urlrewritefilter + admin_api + +[pipeline:keystone-legacy-auth] +pipeline = + urlrewritefilter + legacy_auth + RAX-KEY-extension + service_api + +[app:service_api] +paste.app_factory = keystone.server:service_app_factory + +[app:admin_api] +paste.app_factory = keystone.server:admin_app_factory + +[filter:urlrewritefilter] +paste.filter_factory = keystone.middleware.url:filter_factory + +[filter:legacy_auth] +paste.filter_factory = keystone.frontends.legacy_token_auth:filter_factory + +[filter:RAX-KEY-extension] +paste.filter_factory = keystone.contrib.extensions.service.raxkey.frontend:filter_factory + +[filter:debug] +paste.filter_factory = keystone.common.wsgi:debug_filter_factory diff --git a/conf/keystone/keystone_data.sh b/conf/keystone/keystone_data.sh new file mode 100755 index 00000000..6a495518 --- /dev/null +++ b/conf/keystone/keystone_data.sh @@ -0,0 +1,54 @@ +#!/bin/bash +BIN_DIR=${BIN_DIR:-.} +# Tenants +$BIN_DIR/keystone-manage $* tenant add admin +$BIN_DIR/keystone-manage $* tenant add demo +$BIN_DIR/keystone-manage $* tenant add invisible_to_admin + +# Users +$BIN_DIR/keystone-manage $* user add admin %ADMIN_PASSWORD% +$BIN_DIR/keystone-manage $* user add demo %ADMIN_PASSWORD% + +# Roles +$BIN_DIR/keystone-manage $* role add Admin +$BIN_DIR/keystone-manage $* role add Member +$BIN_DIR/keystone-manage $* role add KeystoneAdmin +$BIN_DIR/keystone-manage $* role add KeystoneServiceAdmin +$BIN_DIR/keystone-manage $* role add sysadmin +$BIN_DIR/keystone-manage $* role add netadmin +$BIN_DIR/keystone-manage $* role grant Admin admin admin +$BIN_DIR/keystone-manage $* role grant Member demo demo +$BIN_DIR/keystone-manage $* role grant sysadmin demo demo +$BIN_DIR/keystone-manage $* role grant netadmin demo demo +$BIN_DIR/keystone-manage $* role grant Member demo invisible_to_admin +$BIN_DIR/keystone-manage $* role grant Admin admin demo +$BIN_DIR/keystone-manage $* role grant Admin admin +$BIN_DIR/keystone-manage $* role grant KeystoneAdmin admin +$BIN_DIR/keystone-manage $* role grant KeystoneServiceAdmin admin + +# Services +$BIN_DIR/keystone-manage $* service add nova compute "Nova Compute Service" +$BIN_DIR/keystone-manage $* service add ec2 ec2 "EC2 Compatability Layer" +$BIN_DIR/keystone-manage $* service add glance image "Glance Image Service" +$BIN_DIR/keystone-manage $* service add keystone identity "Keystone Identity Service" +if [[ "$ENABLED_SERVICES" =~ "swift" ]]; then + $BIN_DIR/keystone-manage $* service add swift object-store "Swift Service" +fi + +#endpointTemplates +$BIN_DIR/keystone-manage $* endpointTemplates add RegionOne nova http://%HOST_IP%:8774/v1.1/%tenant_id% http://%HOST_IP%:8774/v1.1/%tenant_id% http://%HOST_IP%:8774/v1.1/%tenant_id% 1 1 +$BIN_DIR/keystone-manage $* endpointTemplates add RegionOne ec2 http://%HOST_IP%:8773/services/Cloud http://%HOST_IP%:8773/services/Admin http://%HOST_IP%:8773/services/Cloud 1 1 +$BIN_DIR/keystone-manage $* endpointTemplates add RegionOne glance http://%HOST_IP%:9292/v1.1/%tenant_id% http://%HOST_IP%:9292/v1.1/%tenant_id% http://%HOST_IP%:9292/v1.1/%tenant_id% 1 1 +$BIN_DIR/keystone-manage $* endpointTemplates add RegionOne keystone http://%HOST_IP%:5000/v2.0 http://%HOST_IP%:35357/v2.0 http://%HOST_IP%:5000/v2.0 1 1 +if [[ "$ENABLED_SERVICES" =~ "swift" ]]; then + $BIN_DIR/keystone-manage $* endpointTemplates add RegionOne swift http://%HOST_IP%:8080/v1/AUTH_%tenant_id% http://%HOST_IP%:8080/ http://%HOST_IP%:8080/v1/AUTH_%tenant_id% 1 1 +fi + +# Tokens +$BIN_DIR/keystone-manage $* token add %SERVICE_TOKEN% admin admin 2015-02-05T00:00 + +# EC2 related creds - note we are setting the secret key to ADMIN_PASSWORD +# but keystone doesn't parse them - it is just a blob from keystone's +# point of view +$BIN_DIR/keystone-manage $* credentials add admin EC2 'admin' '%ADMIN_PASSWORD%' admin || echo "no support for adding credentials" +$BIN_DIR/keystone-manage $* credentials add demo EC2 'demo' '%ADMIN_PASSWORD%' demo || echo "no support for adding credentials" diff --git a/conf/nova/nova-api-paste.ini b/conf/nova/nova-api-paste.ini new file mode 100644 index 00000000..7f27fdcb --- /dev/null +++ b/conf/nova/nova-api-paste.ini @@ -0,0 +1,138 @@ +############ +# Metadata # +############ +[composite:metadata] +use = egg:Paste#urlmap +/: metaversions +/latest: meta +/2007-01-19: meta +/2007-03-01: meta +/2007-08-29: meta +/2007-10-10: meta +/2007-12-15: meta +/2008-02-01: meta +/2008-09-01: meta +/2009-04-04: meta + +[pipeline:metaversions] +pipeline = ec2faultwrap logrequest metaverapp + +[pipeline:meta] +pipeline = ec2faultwrap logrequest metaapp + +[app:metaverapp] +paste.app_factory = nova.api.metadata.handler:Versions.factory + +[app:metaapp] +paste.app_factory = nova.api.metadata.handler:MetadataRequestHandler.factory + +####### +# EC2 # +####### + +[composite:ec2] +use = egg:Paste#urlmap +/services/Cloud: ec2cloud +/services/Admin: ec2admin + +[pipeline:ec2cloud] +pipeline = ec2faultwrap logrequest totoken authtoken keystonecontext cloudrequest authorizer ec2executor + +[pipeline:ec2admin] +pipeline = ec2faultwrap logrequest totoken authtoken keystonecontext adminrequest authorizer ec2executor + +[pipeline:ec2metadata] +pipeline = ec2faultwrap logrequest ec2md + +[pipeline:ec2versions] +pipeline = ec2faultwrap logrequest ec2ver + +[filter:ec2faultwrap] +paste.filter_factory = nova.api.ec2:FaultWrapper.factory + +[filter:logrequest] +paste.filter_factory = nova.api.ec2:RequestLogging.factory + +[filter:ec2lockout] +paste.filter_factory = nova.api.ec2:Lockout.factory + +[filter:totoken] +paste.filter_factory = keystone.middleware.ec2_token:EC2Token.factory + +[filter:ec2noauth] +paste.filter_factory = nova.api.ec2:NoAuth.factory + +[filter:authenticate] +paste.filter_factory = nova.api.ec2:Authenticate.factory + +[filter:cloudrequest] +controller = nova.api.ec2.cloud.CloudController +paste.filter_factory = nova.api.ec2:Requestify.factory + +[filter:adminrequest] +controller = nova.api.ec2.admin.AdminController +paste.filter_factory = nova.api.ec2:Requestify.factory + +[filter:authorizer] +paste.filter_factory = nova.api.ec2:Authorizer.factory + +[app:ec2executor] +paste.app_factory = nova.api.ec2:Executor.factory + +############# +# Openstack # +############# + +[composite:osapi] +use = call:nova.api.openstack.v2.urlmap:urlmap_factory +/: osversions +/v1.1: openstack_api_v2 +/v2: openstack_api_v2 + +[pipeline:openstack_api_v2] +pipeline = faultwrap authtoken keystonecontext ratelimit serialize extensions osapi_app_v2 + +[filter:faultwrap] +paste.filter_factory = nova.api.openstack.v2:FaultWrapper.factory + +[filter:auth] +paste.filter_factory = nova.api.openstack.v2.auth:AuthMiddleware.factory + +[filter:noauth] +paste.filter_factory = nova.api.openstack.v2.auth:NoAuthMiddleware.factory + +[filter:ratelimit] +paste.filter_factory = nova.api.openstack.v2.limits:RateLimitingMiddleware.factory + +[filter:serialize] +paste.filter_factory = nova.api.openstack.wsgi:LazySerializationMiddleware.factory + +[filter:extensions] +paste.filter_factory = nova.api.openstack.v2.extensions:ExtensionMiddleware.factory + +[app:osapi_app_v2] +paste.app_factory = nova.api.openstack.v2:APIRouter.factory + +[pipeline:osversions] +pipeline = faultwrap osversionapp + +[app:osversionapp] +paste.app_factory = nova.api.openstack.v2.versions:Versions.factory + +########## +# Shared # +########## + +[filter:keystonecontext] +paste.filter_factory = keystone.middleware.nova_keystone_context:NovaKeystoneContext.factory + +[filter:authtoken] +paste.filter_factory = keystone.middleware.auth_token:filter_factory +service_protocol = http +service_host = 127.0.0.1 +service_port = 5000 +auth_host = 127.0.0.1 +auth_port = 35357 +auth_protocol = http +auth_uri = http://127.0.0.1:5000/ +admin_token = %SERVICE_TOKEN% diff --git a/conf/nova/sudo.allowed b/conf/nova/sudo.allowed new file mode 100644 index 00000000..0a79c210 --- /dev/null +++ b/conf/nova/sudo.allowed @@ -0,0 +1,47 @@ +Cmnd_Alias NOVADEVCMDS = /bin/chmod /var/lib/nova/tmp/*/root/.ssh, \ + /bin/chown /var/lib/nova/tmp/*/root/.ssh, \ + /bin/chown, \ + /bin/chmod, \ + /bin/dd, \ + /sbin/ifconfig, \ + /sbin/ip, \ + /sbin/route, \ + /sbin/iptables, \ + /sbin/iptables-save, \ + /sbin/iptables-restore, \ + /sbin/ip6tables-save, \ + /sbin/ip6tables-restore, \ + /sbin/kpartx, \ + /sbin/losetup, \ + /sbin/lvcreate, \ + /sbin/lvdisplay, \ + /sbin/lvremove, \ + /bin/mkdir, \ + /bin/mount, \ + /sbin/pvcreate, \ + /usr/bin/tee, \ + /sbin/tune2fs, \ + /bin/umount, \ + /sbin/vgcreate, \ + /usr/bin/virsh, \ + /usr/bin/qemu-nbd, \ + /usr/sbin/brctl, \ + /sbin/brctl, \ + /usr/sbin/radvd, \ + /usr/sbin/vblade-persist, \ + /sbin/pvcreate, \ + /sbin/aoe-discover, \ + /sbin/vgcreate, \ + /bin/aoe-stat, \ + /bin/kill, \ + /sbin/vconfig, \ + /usr/sbin/ietadm, \ + /sbin/vgs, \ + /sbin/iscsiadm, \ + /usr/bin/socat, \ + /sbin/parted, \ + /usr/sbin/dnsmasq, \ + /usr/sbin/arping + +%USER% ALL = (root) NOPASSWD: SETENV: NOVADEVCMDS + diff --git a/conf/pkgs/db.pkg b/conf/pkgs/db.pkg new file mode 100644 index 00000000..c8f19b93 --- /dev/null +++ b/conf/pkgs/db.pkg @@ -0,0 +1,10 @@ +{ + "ubuntu-oneiric": { + "mysql-server": { + "version": "5.1.58-1ubuntu1", + "allowed": ">=", + "removable": true + } + }, + "rhel-6": {} +} diff --git a/conf/pkgs/general.pkg b/conf/pkgs/general.pkg new file mode 100644 index 00000000..03016715 --- /dev/null +++ b/conf/pkgs/general.pkg @@ -0,0 +1,96 @@ +{ + "ubuntu-oneiric": { + "pep8": { + "version": "0.6.1-2ubuntu1", + "allowed": ">=", + "removable" : false + }, + "pylint": { + "version": "0.23.0-1", + "allowed": ">=", + "removable" : false + }, + "python-pip": { + "version": "1.0-1", + "allowed": ">=", + "removable" : false + }, + "screen": { + "version": "4.0.3-14ubuntu8", + "allowed": ">=", + "removable" : false + }, + "unzip": { + "version": "6.0-4ubuntu1", + "allowed": ">=", + "removable" : false + }, + "wget": { + "version": "1.12-3.1ubuntu1", + "allowed": ">=", + "removable" : false + }, + "psmisc": { + "version": "22.14-1", + "allowed": ">=", + "removable" : false + }, + "git-core": { + "version": "1:1.7.5.4-1", + "allowed": ">=", + "removable" : false + }, + "lsof": { + "version": "4.81.dfsg.1-1build1", + "allowed": ">=", + "removable" : false + }, + "openssh-server": { + "version": "1:5.8p1-7ubuntu1", + "allowed": ">=", + "removable" : false + }, + "vim-nox": { + "version": "2:7.3.154+hg~74503f6ee649-2ubuntu3", + "allowed": ">=", + "removable" : false + }, + "locate": { + "version": "4.4.2-1ubuntu3", + "allowed": ">=", + "removable" : false + }, + "python-virtualenv": { + "version": "1.6.4-0ubuntu1", + "allowed": ">=", + "removable" : false + }, + "python-unittest2": { + "version": "0.5.1-1", + "allowed": ">=", + "removable" : false + }, + "iputils-ping": { + "version": "3:20101006-1", + "allowed": ">=", + "removable" : false + }, + "curl": { + "version": "7.21.6-3ubuntu3", + "allowed": ">=", + "removable" : false + }, + "tcpdump": { + "version": "4.1.1-2ubuntu2", + "allowed": ">=", + "removable" : false + }, + "euca2ools": { + "version": "2.0.0~bzr464-0ubuntu2", + "allowed": ">=", + "removable" : false + } + }, + "rhel-6": { + } +} diff --git a/conf/pkgs/glance.pkg b/conf/pkgs/glance.pkg new file mode 100644 index 00000000..a29fcc39 --- /dev/null +++ b/conf/pkgs/glance.pkg @@ -0,0 +1,48 @@ +{ + "ubuntu-oneiric": { + "python-eventlet": { + "version": "0.9.15-0ubuntu4", + "allowed": ">=", + "removable": true + }, + "python-routes": { + "version": "1.12.3-1", + "allowed": ">=", + "removable": true + }, + "python-greenlet": { + "version": "0.3.1-1ubuntu4", + "allowed": ">=", + "removable": true + }, + "python-argparse": { + "version": "1.1-1ubuntu1", + "allowed": ">=", + "removable": true + }, + "python-sqlalchemy": { + "version": "0.6.8-1", + "allowed": ">=", + "removable": true + }, + "python-wsgiref": { + "removable": false + }, + "python-pastedeploy": { + "version": "1.5.0-2", + "allowed": ">=", + "removable": true + }, + "python-xattr": { + "version": "0.6-1ubuntu2", + "allowed": ">=", + "removable": true + }, + "python-httplib2": { + "version": "0.7.1-1ubuntu1", + "allowed": ">=", + "removable": true + } + }, + "rhel-6": {} +} diff --git a/conf/pkgs/horizon.pkg b/conf/pkgs/horizon.pkg new file mode 100644 index 00000000..8939b10f --- /dev/null +++ b/conf/pkgs/horizon.pkg @@ -0,0 +1,106 @@ +{ + "ubuntu-oneiric": { + "apache2": { + "version": "2.2.20-1ubuntu1", + "allowed": ">=" + }, + "libapache2-mod-wsgi": { + "version": "3.3-2ubuntu3", + "allowed": ">=" + }, + "python-dateutil": { + "version": "1.4.1-4", + "allowed": ">=" + }, + "python-paste": { + "version": "1.7.5.1-4ubuntu1", + "allowed": ">=" + }, + "python-pastedeploy": { + "version": "1.5.0-2", + "allowed": ">=" + }, + "python-anyjson": { + "version": "0.3.1-1", + "allowed": ">=" + }, + "python-routes": { + "version": "1.12.3-1", + "allowed": ">=" + }, + "python-xattr": { + "version": "0.6-1ubuntu2", + "allowed": ">=" + }, + "python-sqlalchemy": { + "version": "0.6.8-1", + "allowed": ">=" + }, + "python-webob": { + "version": "1.0.8-1", + "allowed": ">=" + }, + "python-kombu": { + "version": "1.0.4-2", + "allowed": ">=" + }, + "pylint": { + "version": "0.23.0-1", + "allowed": ">=" + }, + "pep8": { + "version": "0.6.1-2ubuntu1", + "allowed": ">=" + }, + "python-eventlet": { + "version": "0.9.15-0ubuntu4", + "allowed": ">=" + }, + "python-nose": { + "version": "1.0.0-1ubuntu1", + "allowed": ">=" + }, + "python-sphinx": { + "version": "1.0.7+dfsg-1", + "allowed": ">=" + }, + "python-mox": { + "version": "0.5.3-1ubuntu4", + "allowed": ">=" + }, + "python-coverage": { + "version": "3.4-1", + "allowed": ">=" + }, + "python-cherrypy3": { + "version": "3.1.2-1", + "allowed": ">=" + }, + "python-django": { + "version": "1.3-2ubuntu1", + "allowed": ">=" + }, + "python-django-mailer": { + "version": "0.2a1.dev3-0ubuntu1", + "allowed": ">=" + }, + "python-django-nose": { + "version": "0.1.2-2", + "allowed": ">=" + }, + "python-django-registration": { + "version": "0.7-2", + "allowed": ">=" + }, + "python-cloudfiles": { + "version": "1.7.9.2-0ubuntu1", + "allowed": ">=" + }, + "python-migrate": { + "version": "0.7.1-1", + "allowed": ">=" + } + }, + "rhel-6": { + } +} diff --git a/conf/pkgs/keystone.pkg b/conf/pkgs/keystone.pkg new file mode 100644 index 00000000..ea029078 --- /dev/null +++ b/conf/pkgs/keystone.pkg @@ -0,0 +1,73 @@ +{ + "ubuntu-oneiric": { + "python-setuptools": { + "version": "0.6.16-1", + "allowed": ">=", + "removable": true + }, + "python-dev": { + "version": "2.7.2-7ubuntu2", + "allowed": ">=", + "removable": true + }, + "python-lxml": { + "version": "2.3-0.1build1", + "allowed": ">=", + "removable": true + }, + "python-pastescript": { + "version": "1.7.3-7", + "allowed": ">=", + "removable": true + }, + "python-pastedeploy": { + "version": "1.5.0-2", + "allowed": ">=", + "removable": true + }, + "python-paste": { + "version": "1.7.5.1-4ubuntu1", + "allowed": ">=", + "removable": true + }, + "sqlite3": { + "version": "3.7.7-2ubuntu2", + "allowed": ">=", + "removable": true + }, + "python-pysqlite2": { + "version": "2.6.3-2", + "allowed": ">=", + "removable": true + }, + "python-sqlalchemy": { + "version": "0.6.8-1", + "allowed": ">=", + "removable": true + }, + "python-webob": { + "version": "1.0.8-1", + "allowed": ">=", + "removable": true + }, + "python-greenlet": { + "version": "0.3.1-1ubuntu4", + "allowed": ">=", + "removable": true + }, + "python-routes": { + "version": "1.12.3-1", + "allowed": ">=", + "removable": true + }, + "libldap2-dev": { + "removable": true + }, + "libsasl2-dev": { + "version": "2.1.24~rc1.dfsg1+cvs2011-05-23-4ubuntu2", + "allowed": ">=", + "removable": true + } + }, + "rhel-6": {} +} diff --git a/conf/pkgs/n-cpu.pkg b/conf/pkgs/n-cpu.pkg new file mode 100644 index 00000000..0d08389a --- /dev/null +++ b/conf/pkgs/n-cpu.pkg @@ -0,0 +1,18 @@ +{ + "ubuntu-oneiric": { + "lvm2": { + "version": "2.02.66-4ubuntu3", + "allowed": ">=" + }, + "open-iscsi": { + "version": "2.0.871-0ubuntu8", + "allowed": ">=" + }, + "open-iscsi-utils": { + "version": "2.0.871-0ubuntu8", + "allowed": ">=" + } + }, + "rhel-6": { + } +} diff --git a/conf/pkgs/n-vnc.pkg b/conf/pkgs/n-vnc.pkg new file mode 100644 index 00000000..4990959b --- /dev/null +++ b/conf/pkgs/n-vnc.pkg @@ -0,0 +1,10 @@ +{ + "ubuntu-oneiric": { + "python-numpy": { + "version": "1:1.5.1-2ubuntu2", + "allowed": ">=" + } + }, + "rhel-6": { + } +} diff --git a/conf/pkgs/n-vol.pkg b/conf/pkgs/n-vol.pkg new file mode 100644 index 00000000..13a702c1 --- /dev/null +++ b/conf/pkgs/n-vol.pkg @@ -0,0 +1,18 @@ +{ + "ubuntu-oneiric": { + "iscsitarget": { + "version": "1.4.20.2-5ubuntu1", + "allowed": ">=" + }, + "iscsitarget-dkms": { + "version": "1.4.20.2-5ubuntu1", + "allowed": ">=" + }, + "lvm2": { + "version": "2.02.66-4ubuntu3", + "allowed": ">=" + } + }, + "rhel-6": { + } +} diff --git a/conf/pkgs/nova.pkg b/conf/pkgs/nova.pkg new file mode 100644 index 00000000..0607ca33 --- /dev/null +++ b/conf/pkgs/nova.pkg @@ -0,0 +1,174 @@ +{ + "ubuntu-oneiric": { + "dnsmasq-base": { + "version": "2.57-1ubuntu1", + "allowed": ">=" + }, + "dnsmasq-utils": { + "version": "2.57-1ubuntu1", + "allowed": ">=" + }, + "kpartx": { + "version": "0.4.9-2ubuntu1", + "allowed": ">=" + }, + "parted": { + "version": "2.3-6ubuntu3", + "allowed": ">=" + }, + "arping": { + "version": "2.09-2", + "allowed": ">=" + }, + "iputils-arping": { + "version": "3:20101006-1", + "allowed": ">=" + }, + "mysql-server": { + "version": "5.1.58-1ubuntu1", + "allowed": ">=" + }, + "python-mysqldb": { + "version": "1.2.3-0ubuntu1", + "allowed": ">=" + }, + "python-xattr": { + "version": "0.6-1ubuntu2", + "allowed": ">=" + }, + "python-lxml": { + "version": "2.3-0.1build1", + "allowed": ">=" + }, + "kvm": { + "version": "1:84+dfsg-0ubuntu16+0.14.1+noroms+0ubuntu6", + "allowed": ">=" + }, + "gawk": { + "version": "1:3.1.8+dfsg-0.1build1", + "allowed": ">=" + }, + "iptables": { + "version": "1.4.10-1ubuntu1", + "allowed": ">=" + }, + "ebtables": { + "version": "2.0.9.2-2", + "allowed": ">=" + }, + "sqlite3": { + "version": "3.7.7-2ubuntu2", + "allowed": ">=" + }, + "sudo": { + "version": "1.7.4p6-1ubuntu2", + "allowed": ">=" + }, + "libvirt-bin": { + "version": "0.9.2-4ubuntu15", + "allowed": ">=" + }, + "vlan": { + "version": "1.9-3ubuntu3", + "allowed": ">=" + }, + "curl": { + "version": "7.21.6-3ubuntu3", + "allowed": ">=" + }, + "rabbitmq-server": { + "version": "2.5.0-1ubuntu2", + "allowed": ">=" + }, + "socat": { + "version": "1.7.1.3-1.1ubuntu1", + "allowed": ">=" + }, + "python-mox": { + "version": "0.5.3-1ubuntu4", + "allowed": ">=" + }, + "python-paste": { + "version": "1.7.5.1-4ubuntu1", + "allowed": ">=" + }, + "python-migrate": { + "version": "0.7.1-1", + "allowed": ">=" + }, + "python-gflags": { + "version": "1.5.1-1", + "allowed": ">=" + }, + "python-greenlet": { + "version": "0.3.1-1ubuntu4", + "allowed": ">=" + }, + "python-libvirt": { + "version": "0.9.2-4ubuntu15", + "allowed": ">=" + }, + "python-libxml2": { + "version": "2.7.8.dfsg-4", + "allowed": ">=" + }, + "python-routes": { + "version": "1.12.3-1", + "allowed": ">=" + }, + "python-netaddr": { + "version": "0.7.5-4", + "allowed": ">=" + }, + "python-pastedeploy": { + "version": "1.5.0-2", + "allowed": ">=" + }, + "python-eventlet": { + "version": "0.9.15-0ubuntu4", + "allowed": ">=" + }, + "python-cheetah": { + "version": "2.4.4-2ubuntu1", + "allowed": ">=" + }, + "python-carrot": { + "version": "0.10.7-0ubuntu1", + "allowed": ">=" + }, + "python-tempita": { + "version": "0.5.1-1", + "allowed": ">=" + }, + "python-sqlalchemy": { + "version": "0.6.8-1", + "allowed": ">=" + }, + "python-suds": { + "version": "0.4.1-2", + "allowed": ">=" + }, + "python-lockfile": { + "version": "1:0.8-2", + "allowed": ">=" + }, + "python-m2crypto": { + "version": "0.20.1+dfsg1-1.1ubuntu1", + "allowed": ">=" + }, + "python-boto": { + "version": "2.0-0ubuntu1", + "allowed": ">=" + }, + "python-kombu": { + "version": "1.0.4-2", + "allowed": ">=" + }, + "python-feedparser": { + "version": "5.0.1-1", + "allowed": ">=" + } + }, + "rhel-6": { + } +} diff --git a/conf/pkgs/rabbitmq.pkg b/conf/pkgs/rabbitmq.pkg new file mode 100644 index 00000000..8c111bda --- /dev/null +++ b/conf/pkgs/rabbitmq.pkg @@ -0,0 +1,10 @@ +{ + "ubuntu-oneiric": { + "rabbitmq-server": { + "version": "2.5.0-1ubuntu2", + "allowed": ">=", + "removable": true + } + }, + "rhel-6": {} +} diff --git a/conf/pkgs/swift.pkg b/conf/pkgs/swift.pkg new file mode 100644 index 00000000..834e21ef --- /dev/null +++ b/conf/pkgs/swift.pkg @@ -0,0 +1,74 @@ +{ + "ubuntu-oneiric": { + "curl": { + "version": "7.21.6-3ubuntu3", + "allowed": ">=" + }, + "gcc": { + "version": "4:4.6.1-2ubuntu5", + "allowed": ">=" + }, + "memcached": { + "version": "1.4.7-0.1ubuntu1", + "allowed": ">=" + }, + "python-configobj": { + "version": "4.7.2+ds-3", + "allowed": ">=" + }, + "python-coverage": { + "version": "3.4-1", + "allowed": ">=" + }, + "python-dev": { + "version": "2.7.2-7ubuntu2", + "allowed": ">=" + }, + "python-eventlet": { + "version": "0.9.15-0ubuntu4", + "allowed": ">=" + }, + "python-greenlet": { + "version": "0.3.1-1ubuntu4", + "allowed": ">=" + }, + "python-netifaces": { + "version": "0.5-2.1ubuntu2", + "allowed": ">=" + }, + "python-nose": { + "version": "1.0.0-1ubuntu1", + "allowed": ">=" + }, + "python-pastedeploy": { + "version": "1.5.0-2", + "allowed": ">=" + }, + "python-setuptools": { + "version": "0.6.16-1", + "allowed": ">=" + }, + "python-simplejson": { + "version": "2.1.6-1", + "allowed": ">=" + }, + "python-webob": { + "version": "1.0.8-1", + "allowed": ">=" + }, + "python-xattr": { + "version": "0.6-1ubuntu2", + "allowed": ">=" + }, + "sqlite3": { + "version": "3.7.7-2ubuntu2", + "allowed": ">=" + }, + "xfsprogs": { + "version": "3.1.5ubuntu1", + "allowed": ">=" + } + }, + "rhel-6": { + } +} diff --git a/conf/stack.ini b/conf/stack.ini new file mode 100644 index 00000000..2753e831 --- /dev/null +++ b/conf/stack.ini @@ -0,0 +1,159 @@ +# Devstack2 local configuration + +# When a value looks like a bash variable + default then it is parsed like a bash +# variable and will perform similar lookups. Ie ${SQL_HOST:-localhost} will +# look in environment variable SQL_HOST and if that does not exist then +# localhost will be used instead. + +[default] + +# Set api host endpoint +host_ip = ${HOST_IP:-127.0.0.1} + +#Sys log enabled or not +syslog = 0 + +[db] + +sql_host = ${SQL_HOST:-localhost} +sql_user = ${SQL_USER:-root} + +#internal commands are dependent on this... +type = mysql + +[nova] + +# Nova original used project_id as the *account* that owned resources (servers, +# ip address, ...) With the addition of Keystone we have standardized on the +# term **tenant** as the entity that owns the resources. **novaclient** still +# uses the old deprecated terms project_id. Note that this field should now be +# set to tenant_name, not tenant_id. +nova_project_id = ${TENANT:-demo} + +# In addition to the owning entity (tenant), nova stores the entity performing +# the action as the **user**. +nova_username = ${USERNAME:-demo} + +# With Keystone you pass the keystone password instead of an api key. +# The most recent versions of novaclient use NOVA_PASSWORD instead of NOVA_API_KEY +nova_password = ${ADMIN_PASSWORD:-secrete} + +# With the addition of Keystone, to use an openstack cloud you should +# authenticate against keystone, which returns a **Token** and **Service +# Catalog**. The catalog contains the endpoint for all services the user/tenant +# has access to - including nova, glance, keystone, swift, ... We currently +# recommend using the 2.0 *auth api*. +# +# *NOTE*: Using the 2.0 *auth api* does not mean that compute api is 2.0. We +# will use the 1.1 *compute api* +nova_url = ${NOVA_URL:-http://$HOST_IP:5000/v2.0/} + +# Currently novaclient needs you to specify the *compute api* version. This +# needs to match the config of your catalog returned by Keystone. +nova_version = ${NOVA_VERSION:-1.1} + +[ec2] + +# Set the ec2 url so euca2ools works +ec2_url = ${EC2_URL:-http://$HOST_IP:8773/services/Cloud} + +# Access key is set in the initial keystone data to be the same as username +ec2_access_key = ${USERNAME:-demo} + +# Secret key is set in the initial keystone data to the admin password +ec2_secret_key = ${ADMIN_PASSWORD:-secrete} + +[vm] + +# Max time till the vm is bootable +boot_timeout = ${BOOT_TIMEOUT:-15} + +# Max time to wait while vm goes from build to active state +active_timeout = ${ACTIVE_TIMEOUT:-10} + +# Max time from run instance command until it is running +running_timeout = ${RUNNING_TIMEOUT:-$(($active_timeout + $active_timeout))} + +# Max time to wait for proper IP association and dis-association. +associate_timeout = ${ASSOCIATE_TIMEOUT:-10} + +[git] + +# compute service +nova_repo = https://github.com/openstack/nova.git +nova_branch = master + +# storage service +swift_repo = https://github.com/openstack/swift.git +swift_branch = master + +# swift and keystone integration +swift_keystone_repo = https://github.com/cloudbuilders/swift-keystone2.git +swift_keystone_branch = master + +# image catalog service +glance_repo = https://github.com/openstack/glance.git +glance_branch = master + +# unified auth system (manages accounts/tokens) +keystone_repo = https://github.com/openstack/keystone.git +keystone_branch = stable/diablo + +# a websockets/html5 or flash powered VNC console for vm instances +novnc_repo = https://github.com/cloudbuilders/noVNC.git +novnc_branch = master + +# django powered web control panel for openstack +horizon_repo = https://github.com/openstack/horizon.git +horizon_branch = master + +# python client library to nova that horizon (and others) use +novaclient_repo = https://github.com/openstack/python-novaclient.git +novaclient_branch = master + +# openstackx is a collection of extensions to openstack.compute & nova +# that is *deprecated*. The code is being moved into python-novaclient & nova. +openstackx_repo = https://github.com/cloudbuilders/openstackx.git +openstackx_branch = master + +# quantum service +quantum_repo = https://github.com/openstack/quantum +quantum_branch = master + +[ci] + +# CI test suite +citest_repo = https://github.com/openstack/tempest.git +citest_branch = master + +[img] + +# Specify a comma-separated list of uec images to download and install into glance. +# supported urls here are: +# * "uec-style" images: +# If the file ends in .tar.gz, uncompress the tarball and and select the first +# .img file inside it as the image. If present, use "*-vmlinuz*" as the kernel +# and "*-initrd*" as the ramdisk +# example: http://cloud-images.ubuntu.com/releases/oneiric/release/ubuntu-11.10-server-cloudimg-amd64.tar.gz +# * disk image (*.img,*.img.gz) +# if file ends in .img, then it will be uploaded and registered as a to +# glance as a disk image. If it ends in .gz, it is uncompressed first. +# example: +# http://cloud-images.ubuntu.com/releases/oneiric/release/ubuntu-11.10-server-cloudimg-armel-disk1.img +# http://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-rootfs.img.gz + +# old ttylinux-uec image +#image_urls="http://smoser.brickies.net/ubuntu/ttylinux-uec/ttylinux-uec-amd64-11.2_2.6.35-15_1.tar.gz" +# cirros full disk image +#image_urls="http://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img" + +# uec style cirros image +image_urls = "http://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-uec.tar.gz" + +[passwords] + +sql = ${MYSQL_PASSWORD:-} +rabbit = ${RABBIT_PASSWORD:-} +horizon = ${ADMIN_PASSWORD:-} +service_token = ${SERVICE_TOKEN:-} + diff --git a/conf/swift/account-server.conf b/conf/swift/account-server.conf new file mode 100644 index 00000000..db0f097f --- /dev/null +++ b/conf/swift/account-server.conf @@ -0,0 +1,20 @@ +[DEFAULT] +devices = %NODE_PATH%/node +mount_check = false +bind_port = %BIND_PORT% +user = %USER% +log_facility = LOG_LOCAL%LOG_FACILITY% +swift_dir = %SWIFT_CONFIG_LOCATION% + +[pipeline:main] +pipeline = account-server + +[app:account-server] +use = egg:swift#account + +[account-replicator] +vm_test_mode = yes + +[account-auditor] + +[account-reaper] diff --git a/conf/swift/container-server.conf b/conf/swift/container-server.conf new file mode 100644 index 00000000..bdc3e3a0 --- /dev/null +++ b/conf/swift/container-server.conf @@ -0,0 +1,22 @@ +[DEFAULT] +devices = %NODE_PATH%/node +mount_check = false +bind_port = %BIND_PORT% +user = %USER% +log_facility = LOG_LOCAL%LOG_FACILITY% +swift_dir = %SWIFT_CONFIG_LOCATION% + +[pipeline:main] +pipeline = container-server + +[app:container-server] +use = egg:swift#container + +[container-replicator] +vm_test_mode = yes + +[container-updater] + +[container-auditor] + +[container-sync] diff --git a/conf/swift/object-server.conf b/conf/swift/object-server.conf new file mode 100644 index 00000000..06fbffea --- /dev/null +++ b/conf/swift/object-server.conf @@ -0,0 +1,20 @@ +[DEFAULT] +devices = %NODE_PATH%/node +mount_check = false +bind_port = %BIND_PORT% +user = %USER% +log_facility = LOG_LOCAL%LOG_FACILITY% +swift_dir = %SWIFT_CONFIG_LOCATION% + +[pipeline:main] +pipeline = object-server + +[app:object-server] +use = egg:swift#object + +[object-replicator] +vm_test_mode = yes + +[object-updater] + +[object-auditor] diff --git a/conf/swift/proxy-server.conf b/conf/swift/proxy-server.conf new file mode 100644 index 00000000..d7ed4851 --- /dev/null +++ b/conf/swift/proxy-server.conf @@ -0,0 +1,33 @@ +[DEFAULT] +bind_port = 8080 +user = %USER% +log_facility = LOG_LOCAL1 +swift_dir = %SWIFT_CONFIG_LOCATION% + +[pipeline:main] +pipeline = healthcheck cache %AUTH_SERVER% proxy-server + +[app:proxy-server] +use = egg:swift#proxy +allow_account_management = true +account_autocreate = true + +[filter:keystone] +use = egg:swiftkeystone2#keystone2 +keystone_admin_token = %SERVICE_TOKEN% +keystone_url = http://localhost:35357/v2.0 +keystone_swift_operator_roles = Member + +[filter:tempauth] +use = egg:swift#tempauth +user_admin_admin = admin .admin .reseller_admin +user_test_tester = testing .admin +user_test2_tester2 = testing2 .admin +user_test_tester3 = testing3 +bind_ip = 0.0.0.0 + +[filter:healthcheck] +use = egg:swift#healthcheck + +[filter:cache] +use = egg:swift#memcache diff --git a/conf/swift/rsyncd.conf b/conf/swift/rsyncd.conf new file mode 100644 index 00000000..66215c7f --- /dev/null +++ b/conf/swift/rsyncd.conf @@ -0,0 +1,79 @@ +uid = %USER% +gid = %GROUP% +log file = /var/log/rsyncd.log +pid file = /var/run/rsyncd.pid +address = 127.0.0.1 + +[account6012] +max connections = 25 +path = %SWIFT_DATA_LOCATION%/1/node/ +read only = false +lock file = /var/lock/account6012.lock + +[account6022] +max connections = 25 +path = %SWIFT_DATA_LOCATION%/2/node/ +read only = false +lock file = /var/lock/account6022.lock + +[account6032] +max connections = 25 +path = %SWIFT_DATA_LOCATION%/3/node/ +read only = false +lock file = /var/lock/account6032.lock + +[account6042] +max connections = 25 +path = %SWIFT_DATA_LOCATION%/4/node/ +read only = false +lock file = /var/lock/account6042.lock + + +[container6011] +max connections = 25 +path = %SWIFT_DATA_LOCATION%/1/node/ +read only = false +lock file = /var/lock/container6011.lock + +[container6021] +max connections = 25 +path = %SWIFT_DATA_LOCATION%/2/node/ +read only = false +lock file = /var/lock/container6021.lock + +[container6031] +max connections = 25 +path = %SWIFT_DATA_LOCATION%/3/node/ +read only = false +lock file = /var/lock/container6031.lock + +[container6041] +max connections = 25 +path = %SWIFT_DATA_LOCATION%/4/node/ +read only = false +lock file = /var/lock/container6041.lock + + +[object6010] +max connections = 25 +path = %SWIFT_DATA_LOCATION%/1/node/ +read only = false +lock file = /var/lock/object6010.lock + +[object6020] +max connections = 25 +path = %SWIFT_DATA_LOCATION%/2/node/ +read only = false +lock file = /var/lock/object6020.lock + +[object6030] +max connections = 25 +path = %SWIFT_DATA_LOCATION%/3/node/ +read only = false +lock file = /var/lock/object6030.lock + +[object6040] +max connections = 25 +path = %SWIFT_DATA_LOCATION%/4/node/ +read only = false +lock file = /var/lock/object6040.lock diff --git a/conf/swift/swift-remakerings b/conf/swift/swift-remakerings new file mode 100755 index 00000000..c65353ce --- /dev/null +++ b/conf/swift/swift-remakerings @@ -0,0 +1,26 @@ +#!/bin/bash + +cd %SWIFT_CONFIG_LOCATION% + +rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz + +swift-ring-builder object.builder create %SWIFT_PARTITION_POWER_SIZE% 3 1 +swift-ring-builder object.builder add z1-127.0.0.1:6010/sdb1 1 +swift-ring-builder object.builder add z2-127.0.0.1:6020/sdb2 1 +swift-ring-builder object.builder add z3-127.0.0.1:6030/sdb3 1 +swift-ring-builder object.builder add z4-127.0.0.1:6040/sdb4 1 +swift-ring-builder object.builder rebalance + +swift-ring-builder container.builder create %SWIFT_PARTITION_POWER_SIZE% 3 1 +swift-ring-builder container.builder add z1-127.0.0.1:6011/sdb1 1 +swift-ring-builder container.builder add z2-127.0.0.1:6021/sdb2 1 +swift-ring-builder container.builder add z3-127.0.0.1:6031/sdb3 1 +swift-ring-builder container.builder add z4-127.0.0.1:6041/sdb4 1 +swift-ring-builder container.builder rebalance + +swift-ring-builder account.builder create %SWIFT_PARTITION_POWER_SIZE% 3 1 +swift-ring-builder account.builder add z1-127.0.0.1:6012/sdb1 1 +swift-ring-builder account.builder add z2-127.0.0.1:6022/sdb2 1 +swift-ring-builder account.builder add z3-127.0.0.1:6032/sdb3 1 +swift-ring-builder account.builder add z4-127.0.0.1:6042/sdb4 1 +swift-ring-builder account.builder rebalance diff --git a/conf/swift/swift-startmain b/conf/swift/swift-startmain new file mode 100755 index 00000000..69efebd9 --- /dev/null +++ b/conf/swift/swift-startmain @@ -0,0 +1,3 @@ +#!/bin/bash + +swift-init all restart diff --git a/conf/swift/swift.conf b/conf/swift/swift.conf new file mode 100644 index 00000000..98df4663 --- /dev/null +++ b/conf/swift/swift.conf @@ -0,0 +1,3 @@ +[swift-hash] +# random unique string that can never change (DO NOT LOSE) +swift_hash_path_suffix = %SWIFT_HASH% diff --git a/devstack/Component.py b/devstack/Component.py new file mode 100644 index 00000000..7fd30c76 --- /dev/null +++ b/devstack/Component.py @@ -0,0 +1,67 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import Util + +""" +An abstraction that different components +can inherit from to perform or basic install +and configure and uninstall actions. +""" + + +class ComponentBase(): + def __init__(self, component_name, *args, **kargs): + self.cfg = kargs.get("cfg") + self.packager = kargs.get("pkg") + self.distro = kargs.get("distro") + self.root = kargs.get("root") + self.othercomponents = kargs.get("components") + pths = Util.component_pths(self.root, component_name) + self.componentroot = pths.get('root_dir') + self.tracedir = pths.get("trace_dir") + self.appdir = pths.get("app_dir") + self.cfgdir = pths.get('config_dir') + +# +#the following are just interfaces... +# + + +class InstallComponent(): + def download(self): + raise NotImplementedError() + + def configure(self): + raise NotImplementedError() + + def install(self): + raise NotImplementedError() + + +class UninstallComponent(): + def unconfigure(self): + raise NotImplementedError() + + def uninstall(self): + raise NotImplementedError() + + +class RuntimeComponent(): + def start(self): + raise NotImplementedError() + + def stop(self): + raise NotImplementedError() diff --git a/devstack/Config.py b/devstack/Config.py new file mode 100644 index 00000000..d7276fb4 --- /dev/null +++ b/devstack/Config.py @@ -0,0 +1,82 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import re +import ConfigParser + +import Shell +import Logger +import Exceptions +from Exceptions import (BadParamException) + +LOG = Logger.getLogger("install.config") +PW_TMPL = "Enter a password for %s: " +ENV_PAT = re.compile(r"^\s*\$\{([\w\d]+):\-(.*)\}\s*$") + + +class EnvConfigParser(ConfigParser.RawConfigParser): + def __init__(self): + ConfigParser.RawConfigParser.__init__(self) + self.pws = dict() + + def _makekey(self, section, option): + key = option + "@" + section + return key + + def get(self, section, option): + key = self._makekey(section, option) + LOG.debug("Fetching value for param %s" % (key)) + v = self._get_special(section, option) + LOG.debug("Fetched \"%s\" for %s" % (v, key)) + return v + + def _get_special(self, section, option): + key = self._makekey(section, option) + v = ConfigParser.RawConfigParser.get(self, section, option) + if(v == None): + return v + mtch = ENV_PAT.match(v) + if(mtch): + env = mtch.group(1).strip() + defv = mtch.group(2) + if(len(defv) == 0 and len(env) == 0): + msg = "Invalid bash-like value %s for %s" % (v, key) + raise BadParamException(msg) + if(len(env) == 0): + return defv + LOG.debug("Looking up environment variable %s" % (env)) + v = os.getenv(env) + if(v == None): + LOG.debug("Could not find anything in environment variable %s (using default value)" % (env)) + v = defv + return v + else: + return v + + def getpw(self, section, option): + key = self._makekey(section, option) + pw = self.pws.get(key) + if(pw != None): + return pw + pw = self.get(section, option) + if(pw == None): + pw = "" + if(len(pw) == 0): + while(len(pw) == 0): + pw = Shell.password(PW_TMPL % (key)) + LOG.debug("Password for %s will be %s" % (key, pw)) + self.pws[key] = pw + return pw diff --git a/devstack/Db.py b/devstack/Db.py new file mode 100644 index 00000000..2f1da6c2 --- /dev/null +++ b/devstack/Db.py @@ -0,0 +1,244 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import Logger +import Component +from Component import (ComponentBase, RuntimeComponent, + UninstallComponent, InstallComponent) +import Util +from Util import (DB, + get_pkg_list, param_replace, + joinlinesep) +import Trace +from Trace import (TraceWriter, TraceReader) +import Shell +from Shell import (mkdirslist, execute, deldir) + +LOG = Logger.getLogger("install.mysql") +TYPE = DB + +#TODO maybe someday this should be in the pkg info? +TYPE_ACTIONS = { + 'mysql': { + 'start': ["/etc/init.d/mysql", "start"], + 'stop' : ["/etc/init.d/mysql", "stop"], + 'create_db': 'CREATE DATABASE %s;', + 'drop_db': 'DROP DATABASE IF EXISTS %s;', + "before_install": [ + { + 'cmd': ["debconf-set-selections"], + 'stdin': [ + "mysql-server-5.1 mysql-server/root_password password %PASSWORD%", + "mysql-server-5.1 mysql-server/root_password_again password %PASSWORD%", + "mysql-server-5.1 mysql-server/start_on_boot boolean %BOOT_START%", + ], + 'run_as_root': True, + }, + ], + 'after_install': [ + { + 'cmd': [ + "mysql", + '-uroot', + '-p%PASSWORD%', + '-h127.0.0.1', + '-e', + "GRANT ALL PRIVILEGES ON *.* TO '%USER%'@'%' identified by '%PASSWORD%';", + ], + 'stdin': [], + 'run_as_root': False, + } + ] + }, +} + +BASE_ERROR = 'Currently we do not know how to %s for database type [%s]' + + +class DBUninstaller(ComponentBase, UninstallComponent): + def __init__(self, *args, **kargs): + ComponentBase.__init__(self, TYPE, *args, **kargs) + self.tracereader = TraceReader(self.tracedir, Trace.IN_TRACE) + + def unconfigure(self): + #nothing to unconfigure, we are just a pkg + pass + + def uninstall(self): + #clean out removeable packages + pkgsfull = self.tracereader.packages_installed() + if(len(pkgsfull)): + am = len(pkgsfull) + LOG.info("Removing %s packages" % (am)) + self.packager.remove_batch(pkgsfull) + dirsmade = self.tracereader.dirs_made() + if(len(dirsmade)): + am = len(dirsmade) + LOG.info("Removing %s created directories" % (am)) + for dirname in dirsmade: + deldir(dirname) + LOG.info("Removed %s" % (dirname)) + + +class DBInstaller(ComponentBase, InstallComponent): + def __init__(self, *args, **kargs): + ComponentBase.__init__(self, TYPE, *args, **kargs) + self.tracewriter = TraceWriter(self.tracedir, Trace.IN_TRACE) + + def download(self): + #nothing to download, we are just a pkg + pass + + def configure(self): + #nothing to configure, we are just a pkg + pass + + def _run_install_cmds(self, cmds): + if(not cmds or len(cmds) == 0): + return + installparams = self._get_install_params() + for cmdinfo in cmds: + cmd_to_run_templ = cmdinfo.get("cmd") + if(not cmd_to_run_templ): + continue + cmd_to_run = list() + for piece in cmd_to_run_templ: + cmd_to_run.append(param_replace(piece, installparams)) + stdin_templ = cmdinfo.get('stdin') + stdin = None + if(stdin_templ): + stdin_full = list() + for piece in stdin_templ: + stdin_full.append(param_replace(piece, installparams)) + stdin = joinlinesep(stdin_full) + root_run = cmdinfo.get('run_as_root', True) + execute(*cmd_to_run, process_input=stdin, run_as_root=root_run) + + def _get_install_params(self): + out = dict() + out['PASSWORD'] = self.cfg.getpw("passwords", "sql") + out['BOOT_START'] = str(True).lower() + out['USER'] = self.cfg.get("db", "sql_user") + return out + + def _pre_install(self, pkgs): + dbtype = self.cfg.get("db", "type") + dbactions = TYPE_ACTIONS.get(dbtype) + if(dbactions and dbactions.get("before_install")): + LOG.info("Running pre-install commands.") + self._run_install_cmds(dbactions.get("before_install")) + + def _post_install(self, pkgs): + dbtype = self.cfg.get("db", "type") + dbactions = TYPE_ACTIONS.get(dbtype) + if(dbactions and dbactions.get("after_install")): + LOG.info("Running post-install commands.") + self._run_install_cmds(dbactions.get("after_install")) + + def install(self): + #just install the pkgs + pkgs = get_pkg_list(self.distro, TYPE) + #run any pre-installs cmds + self._pre_install(pkgs) + #now install the pkgs + pkgnames = sorted(pkgs.keys()) + LOG.debug("Installing packages %s" % (", ".join(pkgnames))) + installparams = self._get_install_params() + self.packager.install_batch(pkgs, installparams) + for name in pkgnames: + packageinfo = pkgs.get(name) + version = packageinfo.get("version", "") + remove = packageinfo.get("removable", True) + # This trace is used to remove the pkgs + self.tracewriter.package_install(name, remove, version) + dirsmade = mkdirslist(self.tracedir) + # This trace is used to remove the dirs created + self.tracewriter.dir_made(*dirsmade) + #run any post-installs cmds + self._post_install(pkgs) + #TODO + # # Update the DB to give user "$MYSQL_USER"@"%" full control of the all databases: + #sudo mysql -uroot -p$MYSQL_PASSWORD -h127.0.0.1 -e "GRANT ALL PRIVILEGES ON *.* TO '$MYSQL_USER'@'%' identified by '$MYSQL_PASSWORD';" + #TODO + # Edit /etc/mysql/my.cnf to change "bind-address" from localhost (127.0.0.1) to any (0.0.0.0) and stop the mysql service: + #sudo sed -i 's/127.0.0.1/0.0.0.0/g' /etc/mysql/my.cnf + return self.tracedir + + +class DBRuntime(ComponentBase, RuntimeComponent): + def __init__(self, *args, **kargs): + ComponentBase.__init__(self, TYPE, *args, **kargs) + self.tracereader = TraceReader(self.tracedir, Trace.IN_TRACE) + + def start(self): + pkgsinstalled = self.tracereader.packages_installed() + if(len(pkgsinstalled) == 0): + msg = "Can not start %s since it was not installed" % (TYPE) + raise StartException(msg) + dbtype = cfg.get("db", "type") + typeactions = TYPE_ACTIONS.get(dbtype.lower()) + if(typeactions == None): + msg = BASE_ERROR % ('start', dbtype) + raise NotImplementedError(msg) + startcmd = typeactions.get('start') + if(startcmd): + execute(*startcmd, run_as_root=True) + return None + + def stop(self): + pkgsinstalled = self.tracereader.packages_installed() + if(len(pkgsinstalled) == 0): + msg = "Can not stop %s since it was not installed" % (TYPE) + raise StopException(msg) + dbtype = cfg.get("db", "type") + typeactions = TYPE_ACTIONS.get(dbtype.lower()) + if(typeactions == None): + msg = BASE_ERROR % ('start', dbtype) + stopcmd = typeactions.get('stop') + if(stopcmd): + execute(*stopcmd, run_as_root=True) + return None + + +def drop_db(cfg, dbname): + dbtype = cfg.get("db", "type") + dbtypelo = dbtype.lower() + if(dbtypelo == 'mysql'): + #drop it + basesql = TYPE_ACTIONS.get(dbtypelo).get('drop_db') + sql = basesql % (dbname) + user = cfg.get("db", "sql_user") + pw = cfg.get("passwords", "sql") + cmd = ['mysql', '-u' + user, '-p' + pw, '-e', sql] + execute(*cmd) + else: + msg = BASE_ERROR % ('drop', dbtype) + raise NotImplementedError(msg) + + +def create_db(cfg, dbname): + dbtype = cfg.get("db", "type") + dbtypelo = dbtype.lower() + if(dbtypelo == 'mysql'): + #create it + basesql = TYPE_ACTIONS.get(dbtypelo).get('create_db') + sql = basesql % (dbname) + user = cfg.get("db", "sql_user") + pw = cfg.get("passwords", "sql") + cmd = ['mysql', '-u' + user, '-p' + pw, '-e', sql] + execute(*cmd) + else: + msg = BASE_ERROR % ('create', dbtype) + raise NotImplementedError(msg) diff --git a/devstack/Downloader.py b/devstack/Downloader.py new file mode 100644 index 00000000..41aaa719 --- /dev/null +++ b/devstack/Downloader.py @@ -0,0 +1,32 @@ +from urlparse import urlparse +import re + +from Shell import (execute, mkdirslist) +from Util import (create_regex, MASTER_BRANCH) +import Logger + +LOG = Logger.getLogger("install.downloader") +EXT_REG = create_regex(r"/^(.*?)\.git\s*$/i") + + +def _gitdownload(storewhere, uri, branch=None): + dirsmade = mkdirslist(storewhere) + LOG.info("Downloading from %s to %s" % (uri, storewhere)) + cmd = ["git", "clone"] + [uri, storewhere] + execute(*cmd) + if(branch and branch != MASTER_BRANCH): + LOG.info("Adjusting git branch to %s" % (branch)) + cmd = ['git', 'checkout'] + [branch] + execute(*cmd, cwd=storewhere) + return dirsmade + + +def download(storewhere, uri, branch=None): + #figure out which type + up = urlparse(uri) + if(up and up.scheme.lower() == "git" or + EXT_REG.match(up.path)): + return _gitdownload(storewhere, uri, branch) + else: + msg = "Currently we do not know how to download %s" % (uri) + raise NotImplementedError(msg) diff --git a/devstack/Exceptions.py b/devstack/Exceptions.py new file mode 100644 index 00000000..d8bbafdf --- /dev/null +++ b/devstack/Exceptions.py @@ -0,0 +1,60 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class InstallException(Exception): + pass + + +class BadRegexException(Exception): + pass + + +class BadParamException(Exception): + pass + + +class NoReplacementException(Exception): + pass + + +class StartException(Exception): + pass + + +class StopException(Exception): + pass + + +class FileException(Exception): + pass + + +class ProcessExecutionError(IOError): + def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, + description=None): + self.exit_code = exit_code + self.stderr = stderr + self.stdout = stdout + self.cmd = cmd + self.description = description + if description is None: + description = ('Unexpected error while running command.') + if exit_code is None: + exit_code = '-' + message = ('%(description)s\nCommand: %(cmd)s\n' + 'Exit code: %(exit_code)s\nStdout: %(stdout)r\n' + 'Stderr: %(stderr)r' % locals()) + IOError.__init__(self, message) diff --git a/devstack/Glance.py b/devstack/Glance.py new file mode 100644 index 00000000..db2eeb31 --- /dev/null +++ b/devstack/Glance.py @@ -0,0 +1,307 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import os.path + +import Logger +import Component +import Shell +import Util +import Trace +from Trace import (TraceWriter, TraceReader) +import Downloader +import Runner +import runners.Foreground as Foreground +from runners.Foreground import (ForegroundRunner) +from Util import (GLANCE, + get_pkg_list, + param_replace, get_dbdsn, + ) +from Shell import (execute, deldir, mkdirslist, unlink, + joinpths, load_file, write_file, touch_file) +import Exceptions +from Exceptions import (StopException, StartException, InstallException) + +LOG = Logger.getLogger("install.glance") + +#naming + config files +TYPE = GLANCE +API_CONF = "glance-api.conf" +REG_CONF = "glance-registry.conf" +CONFIGS = [API_CONF, REG_CONF] +DB_NAME = "glance" + +#why doesn't --record do anything?? +PY_INSTALL = ['python', 'setup.py', 'develop'] +PY_UNINSTALL = ['python', 'setup.py', 'develop', '--uninstall'] + +#what to start +APPS_TO_START = ['glance-api', 'glance-registry'] +APP_OPTIONS = { + 'glance-api': ['--config-file', joinpths('%ROOT%', "etc", API_CONF)], + 'glance-registry': ['--config-file', joinpths('%ROOT%', "etc", REG_CONF)] +} + + +class GlanceBase(Component.ComponentBase): + def __init__(self, *args, **kargs): + Component.ComponentBase.__init__(self, TYPE, *args, **kargs) + #note not config + self.cfgdir = joinpths(self.appdir, "etc") + + +class GlanceUninstaller(GlanceBase, Component.UninstallComponent): + def __init__(self, *args, **kargs): + GlanceBase.__init__(self, *args, **kargs) + self.tracereader = TraceReader(self.tracedir, Trace.IN_TRACE) + + def unconfigure(self): + #get rid of all files configured + cfgfiles = self.tracereader.files_configured() + if(len(cfgfiles)): + am = len(cfgfiles) + LOG.info("Removing %s configuration files" % (am)) + for fn in cfgfiles: + if(len(fn)): + unlink(fn) + LOG.info("Removed %s" % (fn)) + + def uninstall(self): + #clean out removeable packages + pkgsfull = self.tracereader.packages_installed() + if(len(pkgsfull)): + am = len(pkgsfull) + LOG.info("Removing %s packages" % (am)) + self.packager.remove_batch(pkgsfull) + #clean out files touched + filestouched = self.tracereader.files_touched() + if(len(filestouched)): + am = len(pkgsfull) + LOG.info("Removing %s touched files" % (am)) + for fn in filestouched: + if(len(fn)): + unlink(fn) + LOG.info("Removed %s" % (fn)) + #undevelop python??? + #how should this be done?? + pylisting = self.tracereader.py_listing() + if(pylisting != None): + execute(*PY_UNINSTALL, cwd=self.appdir, run_as_root=True) + #clean out dirs created + dirsmade = self.tracereader.dirs_made() + if(len(dirsmade)): + am = len(dirsmade) + LOG.info("Removing %s created directories" % (am)) + for dirname in dirsmade: + deldir(dirname) + LOG.info("Removed %s" % (dirname)) + + +class GlanceRuntime(GlanceBase, Component.RuntimeComponent): + def __init__(self, *args, **kargs): + GlanceBase.__init__(self, *args, **kargs) + self.foreground = kargs.get("foreground", True) + self.tracereader = TraceReader(self.tracedir, Trace.IN_TRACE) + self.tracewriter = TraceWriter(self.tracedir, Trace.START_TRACE) + self.starttracereader = TraceReader(self.tracedir, Trace.START_TRACE) + + def start(self): + #ensure it was installed + pylisting = self.tracereader.py_listing() + if(len(pylisting) == 0): + msg = "Can not start %s since it was not installed" % (TYPE) + raise StartException(msg) + #select how we are going to start i + if(self.foreground): + starter = ForegroundRunner() + else: + raise NotImplementedError("Can not yet start in screen mode") + #start all apps + fns = list() + replacements = dict() + replacements['ROOT'] = self.appdir + for app in APPS_TO_START: + #adjust the program options now that we have real locations + program_opts = [] + for opt in APP_OPTIONS.get(app): + program_opts.append(param_replace(opt, replacements)) + LOG.info("Starting %s with options [%s]" % (app, ", ".join(program_opts))) + #start it with the given settings + fn = starter.start(app, app, *program_opts, + app_dir=self.appdir, trace_dir=self.tracedir) + if(fn): + fns.append(fn) + LOG.info("Started %s, details are in %s" % (app, fn)) + # This trace is used to locate details about what to stop + self.tracewriter.started_info(app, fn) + else: + LOG.info("Started %s" % (app)) + return fns + + def stop(self): + #ensure it was installed + pylisting = self.tracereader.py_listing() + if(pylisting == None or len(pylisting) == 0): + msg = "Can not start %s since it was not installed" % (TYPE) + raise StopException(msg) + #we can only stop what has a started trace + start_traces = self.starttracereader.apps_started() + killedam = 0 + for mp in start_traces: + #extract the apps name and where its trace is + fn = mp.get('trace_fn') + name = mp.get('name') + #missing some key info, skip it + if(fn == None or name == None): + continue + #figure out which class will stop it + contents = Trace.parse_fn(fn) + runtype = None + for (cmd, action) in contents: + if(cmd == Runner.RUN_TYPE and action == Foreground.RUN_TYPE): + runtype = ForegroundRunner + break + #we can try to stop it + if(runtype != None): + LOG.info("Stopping %s with %s in %s" % (name, runtype, self.tracedir)) + killer = runtype() + killer.stop(name, trace_dir=self.tracedir) + killedam += 1 + #if we got rid of them all get rid of the trace + if(killedam == len(start_traces)): + fn = self.starttracereader.trace_fn + LOG.info("Deleting trace file %s" % (fn)) + unlink(fn) + + +class GlanceInstaller(GlanceBase, Component.InstallComponent): + def __init__(self, *args, **kargs): + GlanceBase.__init__(self, *args, **kargs) + self.gitloc = self.cfg.get("git", "glance_repo") + self.brch = self.cfg.get("git", "glance_branch") + self.tracewriter = TraceWriter(self.tracedir, Trace.IN_TRACE) + + def download(self): + dirsmade = Downloader.download(self.appdir, self.gitloc, self.brch) + # This trace isn't used yet but could be + self.tracewriter.downloaded(self.appdir, self.gitloc) + # This trace is used to remove the dirs created + self.tracewriter.dir_made(*dirsmade) + return self.tracedir + + def install(self): + #get all the packages for glance for the specified distro + pkgs = get_pkg_list(self.distro, TYPE) + pkgnames = pkgs.keys() + pkgnames.sort() + LOG.debug("Installing packages %s" % (", ".join(pkgnames))) + self.packager.install_batch(pkgs) + for name in pkgnames: + packageinfo = pkgs.get(name) + version = packageinfo.get("version", "") + remove = packageinfo.get("removable", True) + # This trace is used to remove the pkgs + self.tracewriter.package_install(name, remove, version) + #make a directory for the python trace file (if its not already there) + dirsmade = mkdirslist(self.tracedir) + # This trace is used to remove the dirs created + self.tracewriter.dir_made(*dirsmade) + recordwhere = Trace.touch_trace(self.tracedir, Trace.PY_TRACE) + # This trace is used to remove the trace created + self.tracewriter.py_install(recordwhere) + (sysout, stderr) = execute(*PY_INSTALL, cwd=self.appdir, run_as_root=True) + write_file(recordwhere, sysout) + return self.tracedir + + def configure(self): + dirsmade = mkdirslist(self.cfgdir) + # This trace is used to remove the dirs created + self.tracewriter.dir_made(*dirsmade) + for fn in CONFIGS: + #go through each config in devstack (which is really a template) + #and adjust that template to have real values and then go through + #the resultant config file and perform and adjustments (directory creation...) + #and then write that to the glance configuration directory. + sourcefn = joinpths(Util.STACK_CONFIG_DIR, TYPE, fn) + tgtfn = joinpths(self.cfgdir, fn) + LOG.info("Configuring template file %s" % (sourcefn)) + contents = load_file(sourcefn) + pmap = self._get_param_map(fn) + LOG.info("Replacing parameters in file %s" % (sourcefn)) + LOG.debug("Replacements = %s" % (pmap)) + contents = param_replace(contents, pmap) + LOG.debug("Applying side-effects of param replacement for template %s" % (sourcefn)) + self._config_apply(contents, fn) + LOG.info("Writing configuration file %s" % (tgtfn)) + write_file(tgtfn, contents) + # This trace is used to remove the files configured + self.tracewriter.cfg_write(tgtfn) + return self.tracedir + + def _config_apply(self, contents, fn): + lines = contents.splitlines() + for line in lines: + cleaned = line.strip() + if(len(cleaned) == 0 or cleaned[0] == '#' or cleaned[0] == '['): + #not useful to examine these + continue + pieces = cleaned.split("=", 1) + if(len(pieces) != 2): + continue + key = pieces[0].strip() + val = pieces[1].strip() + if(len(key) == 0 or len(val) == 0): + continue + #now we take special actions + if(key == 'filesystem_store_datadir'): + # Delete existing images + deldir(val) + # Recreate + dirsmade = mkdirslist(val) + self.tracewriter.dir_made(*dirsmade) + elif(key == 'log_file'): + # Ensure that we can write to the log file + dirname = os.path.dirname(val) + if(len(dirname)): + dirsmade = mkdirslist(dirname) + # This trace is used to remove the dirs created + self.tracewriter.dir_made(*dirsmade) + # Destroy then recreate it + unlink(val) + touch_file(val) + self.tracewriter.file_touched(val) + elif(key == 'image_cache_datadir'): + # Destroy then recreate it + deldir(val) + dirsmade = mkdirslist(val) + # This trace is used to remove the dirs created + self.tracewriter.dir_made(*dirsmade) + elif(key == 'scrubber_datadir'): + # Destroy then recreate it + deldir(val) + dirsmade = mkdirslist(val) + # This trace is used to remove the dirs created + self.tracewriter.dir_made(*dirsmade) + + def _get_param_map(self, fn): + # These be used to fill in the configuration + # params with actual values + mp = dict() + mp['DEST'] = self.appdir + mp['SYSLOG'] = self.cfg.getboolean("default", "syslog") + mp['SERVICE_TOKEN'] = self.cfg.getpw("passwords", "service_token") + mp['SQL_CONN'] = get_dbdsn(self.cfg, DB_NAME) + return mp diff --git a/devstack/Horizon.py b/devstack/Horizon.py new file mode 100644 index 00000000..190938bd --- /dev/null +++ b/devstack/Horizon.py @@ -0,0 +1,45 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import Logger +import Component + +LOG = Logger.getLogger("install.horizon") + + +class HorizonTraceWriter(): + def __init__(self, root): + pass + + +class HorizonTraceReader(): + def __init__(self, root): + pass + + +class HorizonUninstaller(Component.UninstallComponent): + def __init__(self, *args, **kargs): + pass + + +class HorizonInstaller(Component.InstallComponent): + def __init__(self, *args, **kargs): + pass + + +class HorizonRuntime(Component.RuntimeComponent): + def __init__(self, *args, **kargs): + pass diff --git a/devstack/Img.py b/devstack/Img.py new file mode 100644 index 00000000..c79014af --- /dev/null +++ b/devstack/Img.py @@ -0,0 +1,33 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class Img(): + def download(): + pass + + def install(): + pass + + def remove(): + pass + + +class ImgDB(): + def availableImages(): + pass + + def installedImages(): + pass diff --git a/devstack/Keystone.py b/devstack/Keystone.py new file mode 100644 index 00000000..dc68988c --- /dev/null +++ b/devstack/Keystone.py @@ -0,0 +1,158 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import os.path + +import Util +from Util import (KEYSTONE, get_pkg_list, get_dbdsn, + param_replace) +import Logger +import Component +import Downloader +import Trace +from Trace import (TraceWriter, TraceReader) +import Shell +from Shell import (execute, mkdirslist, write_file, + load_file, joinpths, touch_file, + unlink) + +LOG = Logger.getLogger("install.keystone") +TYPE = KEYSTONE +PY_INSTALL = ['python', 'setup.py', 'develop'] +ROOT_CONF = "keystone.conf" +CONFIGS = [ROOT_CONF] +BIN_DIR = "bin" +DATA_SCRIPT = "keystone_data.sh" +DB_NAME = "keystone" + +class KeystoneBase(Component.ComponentBase): + def __init__(self, *args, **kargs): + Component.ComponentBase.__init__(self, TYPE, *args, **kargs) + self.cfgdir = joinpths(self.appdir, Util.CONFIG_DIR) + self.bindir = joinpths(self.appdir, BIN_DIR) + + +class KeystoneUninstaller(KeystoneBase, Component.UninstallComponent): + def __init__(self, *args, **kargs): + KeystoneBase.__init__(self, *args, **kargs) + + +class KeystoneInstaller(KeystoneBase, Component.InstallComponent): + def __init__(self, *args, **kargs): + KeystoneBase.__init__(self, *args, **kargs) + self.gitloc = self.cfg.get("git", "keystone_repo") + self.brch = self.cfg.get("git", "keystone_branch") + self.tracewriter = TraceWriter(self.tracedir, Trace.IN_TRACE) + + def download(self): + dirsmade = Downloader.download(self.appdir, self.gitloc, self.brch) + # This trace isn't used yet but could be + self.tracewriter.downloaded(self.appdir, self.gitloc) + # This trace is used to remove the dirs created + self.tracewriter.dir_made(*dirsmade) + return self.tracedir + + def install(self): + pkgs = get_pkg_list(self.distro, TYPE) + pkgnames = pkgs.keys() + pkgnames.sort() + LOG.debug("Installing packages %s" % (", ".join(pkgnames))) + self.packager.install_batch(pkgs) + for name in pkgnames: + packageinfo = pkgs.get(name) + version = packageinfo.get("version", "") + remove = packageinfo.get("removable", True) + # This trace is used to remove the pkgs + self.tracewriter.package_install(name, remove, version) + dirsmade = mkdirslist(self.tracedir) + # This trace is used to remove the dirs created + self.tracewriter.dir_made(*dirsmade) + recordwhere = Trace.touch_trace(self.tracedir, Trace.PY_TRACE) + # This trace is used to remove the trace created + self.tracewriter.py_install(recordwhere) + (sysout, stderr) = execute(*PY_INSTALL, cwd=self.appdir, run_as_root=True) + write_file(recordwhere, sysout) + #adjust db + self._setup_db() + #setup any data + self._setup_data() + return self.tracedir + + def configure(self): + dirsmade = mkdirslist(self.cfgdir) + self.tracewriter.dir_made(*dirsmade) + for fn in CONFIGS: + sourcefn = joinpths(Util.STACK_CONFIG_DIR, TYPE, fn) + tgtfn = joinpths(self.cfgdir, fn) + LOG.info("Configuring template file %s" % (sourcefn)) + contents = load_file(sourcefn) + pmap = self._get_param_map(fn) + LOG.info("Replacing parameters in file %s" % (sourcefn)) + LOG.debug("Replacements = %s" % (pmap)) + contents = param_replace(contents, pmap) + LOG.debug("Applying side-effects of param replacement for template %s" % (sourcefn)) + self._config_apply(contents, fn) + LOG.info("Writing configuration file %s" % (tgtfn)) + write_file(tgtfn, contents) + # This trace is used to remove the files configured + self.tracewriter.cfg_write(tgtfn) + return self.tracedir + + def _setup_db(self): + pass + + def _setup_data(self): + pass + + def _config_apply(self, contents, fn): + lines = contents.splitlines() + for line in lines: + cleaned = line.strip() + if(len(cleaned) == 0 or cleaned[0] == '#' or cleaned[0] == '['): + #not useful to examine these + continue + pieces = cleaned.split("=", 1) + if(len(pieces) != 2): + continue + key = pieces[0].strip() + val = pieces[1].strip() + if(len(key) == 0 or len(val) == 0): + continue + #now we take special actions + if(key == 'log_file'): + # Ensure that we can write to the log file + dirname = os.path.dirname(val) + if(len(dirname)): + dirsmade = mkdirslist(dirname) + # This trace is used to remove the dirs created + self.tracewriter.dir_made(*dirsmade) + # Destroy then recreate it + unlink(val) + touch_file(val) + self.tracewriter.file_touched(val) + + def _get_param_map(self, fn): + # These be used to fill in the configuration + # params with actual values + mp = dict() + mp['DEST'] = self.appdir + mp['SQL_CONN'] = get_dbdsn(self.cfg, DB_NAME) + return mp + + +class KeystoneRuntime(KeystoneBase, Component.RuntimeComponent): + def __init__(self, *args, **kargs): + KeystoneBase.__init__(self, *args, **kargs) diff --git a/devstack/Logger.py b/devstack/Logger.py new file mode 100644 index 00000000..2f57c795 --- /dev/null +++ b/devstack/Logger.py @@ -0,0 +1,85 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import sys + +#requires http://pypi.python.org/pypi/termcolor +#but the colors make it worth it :-) +from termcolor import colored + +#take this in from config?? +LOG_LEVEL = logging.DEBUG +LOG_FORMAT = '%(levelname)s: @%(name)s : %(message)s' + + +class TermFormatter(logging.Formatter): + def __init__(self, fmt): + logging.Formatter.__init__(self, fmt) + + def format(self, record): + lvl = record.levelno + lvlname = record.levelname + if(lvl == logging.DEBUG): + lvlname = colored(lvlname, 'blue') + elif(lvl == logging.INFO): + lvlname = colored(lvlname, 'cyan') + elif(lvl == logging.WARNING): + lvlname = colored(lvlname, 'yellow') + elif(lvl == logging.ERROR): + lvlname = colored(lvlname, 'red') + elif(lvl == logging.CRITICAL): + lvlname = colored(lvlname, 'red') + record.msg = colored(record.msg, attrs=['bold', 'blink']) + record.levelname = lvlname + return logging.Formatter.format(self, record) + + +class TermHandler(logging.Handler): + STREAM = sys.stdout + DO_FLUSH = True + NL = "\n" + + def __init__(self): + logging.Handler.__init__(self) + + def emit(self, record): + lvl = record.levelno + msg = self.format(record) + if(len(msg)): + TermHandler.STREAM.write(msg + TermHandler.NL) + if(TermHandler.DO_FLUSH): + TermHandler.STREAM.flush() + + +def setupLogging(): + logger = logging.getLogger() + handler = TermHandler() + formatter = TermFormatter(LOG_FORMAT) + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(LOG_LEVEL) + + +def getLogger(name): + logger = logging.getLogger(name) + return logger + + +#this should happen first (and once) +INIT_LOGGING = False +if(not INIT_LOGGING): + setupLogging() + INIT_LOGGING = True diff --git a/devstack/Nova.py b/devstack/Nova.py new file mode 100644 index 00000000..151a73cb --- /dev/null +++ b/devstack/Nova.py @@ -0,0 +1,35 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import Logger +import Component + +LOG = Logger.getLogger("install.nova") + + +class NovaUninstaller(Component.UninstallComponent): + def __init__(self, *args, **kargs): + pass + + +class NovaInstaller(Component.InstallComponent): + def __init__(self, *args, **kargs): + pass + + +class NovaRuntime(Component.RuntimeComponent): + def __init__(self, *args, **kargs): + pass diff --git a/devstack/Options.py b/devstack/Options.py new file mode 100644 index 00000000..757a6048 --- /dev/null +++ b/devstack/Options.py @@ -0,0 +1,51 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from optparse import OptionParser + +import Util + +KNOWN_COMPONENTS = set(Util.NAMES) +KNOWN_ACTIONS = set(Util.ACTIONS) + + +def parse(): + parser = OptionParser() + actions = "(" + ", ".join(KNOWN_ACTIONS) + ")" + parser.add_option("-a", "--action", + action="store", + type="string", + dest="action", + metavar="ACTION", + help="action to perform, ie %s" % (actions)) + + parser.add_option("-d", "--directory", + action="store", + type="string", + dest="dir", + metavar="DIR", + help="root DIR for new components or DIR with existing components (ACTION dependent)") + + components = "(" + ", ".join(KNOWN_COMPONENTS) + ")" + parser.add_option("-c", "--component", + action="append", + dest="component", + help="stack component, ie %s" % (components)) + + (options, args) = parser.parse_args() + output = dict() + if(options != None): + output = vars(options) + return output diff --git a/devstack/Packager.py b/devstack/Packager.py new file mode 100644 index 00000000..220b1cc3 --- /dev/null +++ b/devstack/Packager.py @@ -0,0 +1,36 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +An abstraction that different packaging +frameworks (ie apt, yum) can inherit from +""" + +import Logger +import Shell +from Shell import execute + +LOG = Logger.getLogger("install.packager") + +class Packager(): + def __init__(self): + pass + + def install_batch(self, pkgs): + raise NotImplementedError() + + def remove_batch(self, pkgs): + raise NotImplementedError() diff --git a/devstack/Quantum.py b/devstack/Quantum.py new file mode 100644 index 00000000..50a5f0d5 --- /dev/null +++ b/devstack/Quantum.py @@ -0,0 +1,45 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import Logger +import Component + +LOG = Logger.getLogger("install.quantum") + + +class QuantumTraceWriter(): + def __init__(self, root): + pass + + +class QuantumTraceReader(): + def __init__(self, root): + pass + + +class QuantumUninstaller(Component.UninstallComponent): + def __init__(self, *args, **kargs): + pass + + +class QuantumInstaller(Component.InstallComponent): + def __init__(self, *args, **kargs): + pass + + +class QuantumRuntime(Component.RuntimeComponent): + def __init__(self, *args, **kargs): + pass diff --git a/devstack/Rabbit.py b/devstack/Rabbit.py new file mode 100644 index 00000000..62d89318 --- /dev/null +++ b/devstack/Rabbit.py @@ -0,0 +1,120 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import Logger +import Component +from Component import (ComponentBase, RuntimeComponent, + UninstallComponent, InstallComponent) +import Util +from Util import (RABBIT, + get_pkg_list) +import Trace +from Trace import (TraceWriter, TraceReader) +import Shell +from Shell import (mkdirslist, execute, deldir) + +LOG = Logger.getLogger("install.rabbit") +TYPE = RABBIT +START_CMD = ["/etc/init.d/rabbitmq-server", "start"] +STOP_CMD = ["/etc/init.d/rabbitmq-server", "stop"] +PWD_CMD = ['rabbitmqctl', 'change_password', 'guest'] + + +class RabbitUninstaller(ComponentBase, UninstallComponent): + def __init__(self, *args, **kargs): + ComponentBase.__init__(self, TYPE, *args, **kargs) + self.tracereader = TraceReader(self.tracedir, Trace.IN_TRACE) + + def unconfigure(self): + #nothing to unconfigure, we are just a pkg + pass + + def uninstall(self): + #clean out removeable packages + pkgsfull = self.tracereader.packages_installed() + if(len(pkgsfull)): + am = len(pkgsfull) + LOG.info("Removing %s packages" % (am)) + self.packager.remove_batch(pkgsfull) + dirsmade = self.tracereader.dirs_made() + if(len(dirsmade)): + am = len(dirsmade) + LOG.info("Removing %s created directories" % (am)) + for dirname in dirsmade: + deldir(dirname) + LOG.info("Removed %s" % (dirname)) + + +class RabbitInstaller(ComponentBase, InstallComponent): + def __init__(self, *args, **kargs): + ComponentBase.__init__(self, TYPE, *args, **kargs) + self.tracewriter = TraceWriter(self.tracedir, Trace.IN_TRACE) + + def download(self): + #nothing to download, we are just a pkg + pass + + def configure(self): + #nothing to configure, we are just a pkg + pass + + def _setup_pw(self): + passwd = self.cfg.getpw("passwords", "rabbit") + cmd = PWD_CMD + [passwd] + execute(*cmd, run_as_root=True) + + def install(self): + #just install the pkg + pkgs = get_pkg_list(self.os, TYPE) + pkgnames = pkgs.keys() + pkgnames.sort() + LOG.debug("Installing packages %s" % (", ".join(pkgnames))) + self.packager.install_batch(pkgs) + for name in pkgnames: + packageinfo = pkgs.get(name) + version = packageinfo.get("version", "") + remove = packageinfo.get("removable", True) + # This trace is used to remove the pkgs + self.tracewriter.package_install(name, remove, version) + dirsmade = mkdirslist(self.tracedir) + # This trace is used to remove the dirs created + self.tracewriter.dir_made(*dirsmade) + self._setup_pw() + # TODO - stop it (since it usually autostarts) + # so that we control the start/stop, not it + return self.tracedir + + +class RabbitRuntime(ComponentBase, RuntimeComponent): + def __init__(self, *args, **kargs): + ComponentBase.__init__(self, TYPE, *args, **kargs) + self.tracereader = TraceReader(self.tracedir, Trace.IN_TRACE) + + def start(self): + pkgsinstalled = self.tracereader.packages_installed() + if(len(pkgsinstalled) == 0): + msg = "Can not start %s since it was not installed" % (TYPE) + raise StartException(msg) + execute(*START_CMD, run_as_root=True) + return None + + def stop(self): + pkgsinstalled = self.tracereader.packages_installed() + if(len(pkgsinstalled) == 0): + msg = "Can not stop %s since it was not installed" % (TYPE) + raise StopException(msg) + execute(*STOP_CMD, run_as_root=True) + return None diff --git a/devstack/Runner.py b/devstack/Runner.py new file mode 100644 index 00000000..ee3eae06 --- /dev/null +++ b/devstack/Runner.py @@ -0,0 +1,34 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +An abstraction that allows different methods +of starting and stopping python applications +""" + +#trace actions shared +RUN_TYPE = "RUN" + + +class Runner(): + def __init__(self): + pass + + def start(self): + raise NotImplementedError() + + def stop(self, pkgs): + raise NotImplementedError() diff --git a/devstack/Shell.py b/devstack/Shell.py new file mode 100644 index 00000000..9553f1ed --- /dev/null +++ b/devstack/Shell.py @@ -0,0 +1,196 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import subprocess +import shlex +import getpass +import sys +import os.path +import os +import shutil +import json + +import Exceptions +from Exceptions import ProcessExecutionError, FileException +import Logger + +ROOT_HELPER = ["sudo"] +MKPW_CMD = ["openssl", 'rand', '-hex'] + +LOG = Logger.getLogger("install.shell") + + +def execute(*cmd, **kwargs): + process_input = kwargs.pop('process_input', None) + check_exit_code = kwargs.pop('check_exit_code', [0]) + cwd = kwargs.pop('cwd', None) + ignore_exit_code = False + if isinstance(check_exit_code, bool): + ignore_exit_code = not check_exit_code + check_exit_code = [0] + elif isinstance(check_exit_code, int): + check_exit_code = [check_exit_code] + + run_as_root = kwargs.pop('run_as_root', False) + shell = kwargs.pop('shell', False) + + if run_as_root: + cmd = ROOT_HELPER + list(cmd) + + cmd = map(str, cmd) + + LOG.debug(('Running cmd: %s') % (' '.join(cmd))) + if(process_input != None): + LOG.debug(('With stdin: %s') % (process_input)) + + + _PIPE = subprocess.PIPE # pylint: disable=E1101 + obj = subprocess.Popen(cmd, + stdin=_PIPE, + stdout=_PIPE, + stderr=_PIPE, + close_fds=True, + cwd=cwd, + shell=shell) + + result = None + if process_input is not None: + result = obj.communicate(process_input) + else: + result = obj.communicate() + + obj.stdin.close() # pylint: disable=E1101 + _returncode = obj.returncode # pylint: disable=E1101 + LOG.debug(('Cmd result had return code %s') % _returncode) + if not ignore_exit_code \ + and _returncode not in check_exit_code: + (stdout, stderr) = result + raise ProcessExecutionError( + exit_code=_returncode, + stdout=stdout, + stderr=stderr, + cmd=' '.join(cmd)) + else: + return result + + +def isfile(fn): + return os.path.isfile(fn) + + +def joinpths(*pths): + return os.path.join(*pths) + + +def password(prompt=None, genlen=8): + if(prompt): + rd = getpass.getpass(prompt) + else: + rd = getpass.getpass() + if(len(rd) == 0): + LOG.debug("Generating you a password of length %s" % (genlen)) + cmd = MKPW_CMD + [genlen] + (stdout, stderr) = execute(*cmd) + return stdout.strip() + else: + return rd + + +def mkdirslist(pth): + dirsmade = list() + if(os.path.isdir(pth)): + #already there... + return dirsmade + dirspossible = set() + dirspossible.add(pth) + while(True): + splitup = os.path.split(pth) + pth = splitup[0] + base = splitup[1] + dirspossible.add(pth) + if(len(base) == 0): + break + dirstobe = list(dirspossible) + dirstobe.sort() + for pth in dirstobe: + if(not os.path.isdir(pth)): + os.mkdir(pth) + dirsmade.append(pth) + return dirsmade + + +def load_json(fn): + data = load_file(fn) + return json.loads(data) + + +def append_file(fn, text, flush=True): + with open(fn, "a") as f: + f.write(text) + if(flush): + f.flush() + + +def write_file(fn, text, flush=True): + with open(fn, "w") as f: + f.write(text) + if(flush): + f.flush() + + +def touch_file(fn, diethere=True): + if(not os.path.exists(fn)): + with open(fn, "w") as f: + f.truncate(0) + else: + if(diethere): + msg = "Can not touch file %s since it already exists" % (fn) + raise FileException(msg) + + +def load_file(fn): + data = "" + with open(fn, "r") as f: + data = f.read() + return data + + +def mkdir(pth, recurse=True): + if(not os.path.isdir(pth)): + if(recurse): + os.makedirs(pth) + else: + os.mkdir(pth) + + +def deldir(pth, force=True): + if(os.path.isdir(pth)): + if(force): + shutil.rmtree(pth) + else: + os.removedirs(pth) + + +def prompt(prompt): + rd = raw_input(prompt) + return rd + + +def unlink(pth, ignore=True): + try: + os.unlink(pth) + except OSError as (errono, emsg): + if(not ignore): + raise diff --git a/devstack/Swift.py b/devstack/Swift.py new file mode 100644 index 00000000..4faf8629 --- /dev/null +++ b/devstack/Swift.py @@ -0,0 +1,35 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import Logger +import Component + +LOG = Logger.getLogger("install.swift") + + +class SwiftUninstaller(Component.UninstallComponent): + def __init__(self, *args, **kargs): + pass + + +class SwiftInstaller(Component.InstallComponent): + def __init__(self, *args, **kargs): + pass + + +class SwiftRuntime(Component.RuntimeComponent): + def __init__(self, *args, **kargs): + pass diff --git a/devstack/Trace.py b/devstack/Trace.py new file mode 100644 index 00000000..08a1a6b0 --- /dev/null +++ b/devstack/Trace.py @@ -0,0 +1,272 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os.path +import json + +import Util +from Util import (rcf8222date) + +import Shell +from Shell import (touch_file, append_file, joinpths, load_file, mkdirslist) + +TRACE_FMT = "%s - %s\n" +TRACE_EXT = ".trace" + +#common trace actions +CFG_WRITING_FILE = "CFG_WRITING_FILE" +PKG_INSTALL = "PKG_INSTALL" +PYTHON_INSTALL = "PYTHON_INSTALL" +DIR_MADE = "DIR_MADE" +FILE_TOUCHED = "FILE_TOUCHED" +DOWNLOADED = "DOWNLOADED" +AP_STARTED = "AP_STARTED" + +#trace file types +PY_TRACE = "python" +IN_TRACE = "install" +START_TRACE = "start" + +#used to note version of trace +TRACE_VERSION = "TRACE_VERSION" +TRACE_VER = 0x1 + + +class Trace(): + def __init__(self, tracefn): + self.tracefn = tracefn + + def fn(self): + return self.tracefn + + def trace(self, cmd, action=None): + if(action == None): + action = rcf8222date() + line = TRACE_FMT % (cmd, action) + append_file(self.tracefn, line) + + +class TraceWriter(): + def __init__(self, root, name): + self.tracer = None + self.root = root + self.name = name + self.started = False + + def _start(self): + if(self.started): + return + else: + dirs = mkdirslist(self.root) + fn = touch_trace(self.root, self.name) + self.tracer = Trace(fn) + cmd = TRACE_VERSION + action = str(TRACE_VER) + self.tracer.trace(cmd, action) + cmd = DIR_MADE + for d in dirs: + action = d + self.tracer.trace(cmd, action) + self.started = True + + def py_install(self, where): + self._start() + cmd = PYTHON_INSTALL + action = where + self.tracer.trace(cmd, action) + + def cfg_write(self, cfgfile): + self._start() + cmd = CFG_WRITING_FILE + action = cfgfile + self.tracer.trace(cmd, action) + + def downloaded(self, tgt, fromwhere): + self._start() + cmd = DOWNLOADED + action = dict() + action['target'] = tgt + action['from'] = fromwhere + store = json.dumps(action) + self.tracer.trace(cmd, store) + + def dir_made(self, *dirs): + self._start() + cmd = DIR_MADE + for d in dirs: + action = d + self.tracer.trace(cmd, action) + + def file_touched(self, fn): + self._start() + cmd = FILE_TOUCHED + action = fn + self.tracer.trace(cmd, action) + + def package_install(self, name, removeable, version): + self._start() + pkgmeta = dict() + pkgmeta['name'] = name + pkgmeta['removable'] = removeable + pkgmeta['version'] = version + tracedata = json.dumps(pkgmeta) + cmd = PKG_INSTALL + action = tracedata + self.tracer.trace(cmd, action) + + def started_info(self, name, info_fn): + self._start() + cmd = AP_STARTED + out = dict() + out['name'] = name + out['trace_fn'] = info_fn + action = json.dumps(out) + self.tracer.trace(cmd, action) + + +class TraceReader(): + def __init__(self, root, name): + self.root = root + self.name = name + self.trace_fn = trace_fn(root, name) + + def _readpy(self): + lines = self._read() + pyfn = None + pylines = list() + for (cmd, action) in lines: + if(cmd == PYTHON_INSTALL and len(action)): + pyfn = action + break + if(pyfn != None): + lines = load_file(pyfn).splitlines() + pylines = lines + return pylines + + def _read(self): + return parse_name(self.root, self.name) + + def py_listing(self): + return self._readpy() + + def files_touched(self): + lines = self._read() + files = list() + for (cmd, action) in lines: + if(cmd == FILE_TOUCHED and len(action)): + files.append(action) + #ensure no dups + files = list(set(files)) + files.sort() + return files + + def dirs_made(self): + lines = self._read() + dirs = list() + for (cmd, action) in lines: + if(cmd == DIR_MADE and len(action)): + dirs.append(action) + #ensure not dups + dirs = list(set(dirs)) + #ensure in ok order (ie /tmp is before /) + dirs.sort() + dirs.reverse() + return dirs + + def apps_started(self): + lines = self._read() + files = list() + for (cmd, action) in lines: + if(cmd == AP_STARTED and len(action)): + jdec = json.loads(action) + if(type(jdec) is dict): + files.append(jdec) + return files + + def files_configured(self): + lines = self._read() + files = list() + for (cmd, action) in lines: + if(cmd == CFG_WRITING_FILE and len(action)): + files.append(action) + #ensure not dups + files = list(set(files)) + files.sort() + return files + + def packages_installed(self): + lines = self._read() + pkgsinstalled = dict() + actions = list() + for (cmd, action) in lines: + if(cmd == PKG_INSTALL and len(action)): + actions.append(action) + for action in actions: + pv = json.loads(action) + if(type(pv) is dict): + name = pv.get("name", "") + remove = pv.get("removable", True) + version = pv.get("version", "") + if(remove and len(name)): + if(len(version)): + pkgsinstalled[name] = {"version": version} + else: + pkgsinstalled[name] = {} + return pkgsinstalled + + +def trace_fn(rootdir, name): + fullname = name + TRACE_EXT + tracefn = joinpths(rootdir, fullname) + return tracefn + + +def touch_trace(rootdir, name): + tracefn = trace_fn(rootdir, name) + touch_file(tracefn) + return tracefn + + +def split_line(line): + pieces = line.split("-", 1) + if(len(pieces) == 2): + cmd = pieces[0].rstrip() + action = pieces[1].lstrip() + return (cmd, action) + else: + return None + + +def read(rootdir, name): + pth = trace_fn(rootdir, name) + contents = load_file(pth) + lines = contents.splitlines() + return lines + + +def parse_fn(fn): + contents = load_file(fn) + lines = contents.splitlines() + accum = list() + for line in lines: + ep = split_line(line) + if(ep == None): + continue + accum.append(tuple(ep)) + return accum + + +def parse_name(rootdir, name): + return parse_fn(trace_fn(rootdir, name)) diff --git a/devstack/Util.py b/devstack/Util.py new file mode 100644 index 00000000..ebb498b3 --- /dev/null +++ b/devstack/Util.py @@ -0,0 +1,294 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import Exceptions +from Exceptions import (BadRegexException, + NoReplacementException, + FileException) +import Logger +import Shell +from Shell import (joinpths, load_json) + +from time import (localtime, strftime) +from termcolor import colored + +import subprocess +import platform +import re +import os + +#constant goodies +VERSION = 0x2 + +#these also have meaning outside python +#ie in the pkg listings so update there also! +UBUNTU12 = "ubuntu-oneiric" +RHEL6 = "rhel-6" + +#GIT master +MASTER_BRANCH = "master" + +#other constants +DB_DSN = '%s://%s:%s@%s/%s' + +#component name mappings +NOVA = "nova" +GLANCE = "glance" +QUANTUM = "quantum" +SWIFT = "swift" +HORIZON = "horizon" +KEYSTONE = "keystone" +DB = "db" +RABBIT = "rabbit" + +NAMES = [NOVA, GLANCE, QUANTUM, + SWIFT, HORIZON, KEYSTONE, + DB, RABBIT] + +#ordering of install (lower priority means earlier) +NAMES_PRIORITY = { + DB: 1, + RABBIT: 1, + KEYSTONE: 2, + GLANCE: 3, + QUANTUM: 4, + NOVA: 5, + SWIFT: 6, + HORIZON: 7, +} + +#when a component is asked for it may +#need another component, that dependency +#map is listed here... +COMPONENT_DEPENDENCIES = { + DB: [], + RABBIT: [], + GLANCE: [KEYSTONE, DB], + KEYSTONE: [DB], + NOVA: [KEYSTONE, GLANCE, DB, RABBIT], + SWIFT: [], + HORIZON: [], + QUANTUM: [], +} + +#program +#actions +INSTALL = "install" +UNINSTALL = "uninstall" +START = "start" +STOP = "stop" + +ACTIONS = [INSTALL, UNINSTALL, START, STOP] + +#this is used to map an action to a useful string for +#the welcome display... +WELCOME_MAP = { + INSTALL: "Installer", + UNINSTALL: "Uninstaller", + START: "Runner", + STOP: "Stopper", +} + +#where we should get the config file... +STACK_CONFIG_DIR = "conf" +CFG_LOC = joinpths(STACK_CONFIG_DIR, "stack.ini") + +#this regex is how we match python platform output to +#a known constant +KNOWN_OS = { + UBUNTU12: '/Ubuntu(.*)oneiric/i', + RHEL6: '/redhat-6\.(\d+)/i', +} + +#the pkg files that each component +#needs +PKG_MAP = { + NOVA: + [ + joinpths(STACK_CONFIG_DIR, "pkgs", "nova.pkg"), + joinpths(STACK_CONFIG_DIR, "pkgs", "general.pkg"), + ], + GLANCE: + [ + joinpths(STACK_CONFIG_DIR, "pkgs", "general.pkg"), + joinpths(STACK_CONFIG_DIR, "pkgs", 'glance.pkg'), + ], + KEYSTONE: + [ + joinpths(STACK_CONFIG_DIR, "pkgs", "general.pkg"), + joinpths(STACK_CONFIG_DIR, "pkgs", 'keystone.pkg'), + ], + HORIZON: + [ + joinpths(STACK_CONFIG_DIR, "pkgs", "general.pkg"), + joinpths(STACK_CONFIG_DIR, "pkgs", 'horizon.pkg'), + ], + SWIFT: + [ + joinpths(STACK_CONFIG_DIR, "pkgs", "general.pkg"), + joinpths(STACK_CONFIG_DIR, "pkgs", 'swift.pkg'), + ], + DB: + [ + joinpths(STACK_CONFIG_DIR, "pkgs", 'db.pkg'), + ], + RABBIT: + [ + joinpths(STACK_CONFIG_DIR, "pkgs", 'rabbitmq.pkg'), + ], +} + +#subdirs of a components dir +TRACE_DIR = "traces" +APP_DIR = "app" +CONFIG_DIR = "config" + +#our ability to create regexes +#which is more like php, which is nicer +#for modifiers... +REGEX_MATCHER = re.compile("^/(.*?)/([a-z]*)$") + +LOG = Logger.getLogger("install.util") + + +def fetch_deps(component, add=False): + if(add): + deps = list([component]) + else: + deps = list() + cdeps = COMPONENT_DEPENDENCIES.get(component) + if(cdeps and len(cdeps)): + for d in cdeps: + deps = deps + fetch_deps(d, True) + return deps + + +def component_pths(root, compnent_type): + component_root = joinpths(root, compnent_type) + tracedir = joinpths(component_root, TRACE_DIR) + appdir = joinpths(component_root, APP_DIR) + cfgdir = joinpths(component_root, CONFIG_DIR) + out = dict() + out['root_dir'] = component_root + out['trace_dir'] = tracedir + out['app_dir'] = appdir + out['config_dir'] = cfgdir + return out + + +def get_interfaces(): + import netifaces + interfaces = dict() + for intfc in netifaces.interfaces(): + interfaces[intfc] = netifaces.ifaddresses(intfc) + return interfaces + + +def create_regex(format): + mtch = REGEX_MATCHER.match(format) + if(not mtch): + raise BadRegexException("Badly formatted pre-regex: " + format) + else: + toberegex = mtch.group(1) + options = mtch.group(2).lower() + flags = 0 + if(options.find("i") != -1): + flags = flags | re.IGNORECASE + if(options.find("m") != -1): + flags = flags | re.MULTILINE + if(options.find("u") != -1): + flags = flags | re.UNICODE + return re.compile(toberegex, flags) + + +def get_dbdsn(cfg, dbname): + user = cfg.get("db", "sql_user") + host = cfg.get("db", "sql_host") + dbtype = cfg.get("db", "type") + pw = cfg.getpw("passwords", "sql") + return DB_DSN % (dbtype, user, pw, host, dbname) + + +def determine_os(): + os = None + plt = platform.platform() + for aos, pat in KNOWN_OS.items(): + reg = create_regex(pat) + if(reg.search(plt)): + os = aos + break + return (os, plt) + + +def get_pkg_list(distro, component): + LOG.info("Getting packages for distro %s and component %s." % (distro, component)) + all_pkgs = dict() + fns = PKG_MAP.get(component) + if(fns == None): + #guess none needed + return all_pkgs + #load + merge them + for fn in fns: + js = load_json(fn) + if(type(js) is dict): + distromp = js.get(distro) + if(distromp != None and type(distromp) is dict): + all_pkgs = dict(all_pkgs.items() + distromp.items()) + return all_pkgs + + +def joinlinesep(*pieces): + return os.linesep.join(*pieces) + +def param_replace(text, replacements): + if(len(replacements) == 0 or len(text) == 0): + return text + + def replacer(m): + org = m.group() + name = m.group(1) + v = replacements.get(name) + if(v == None): + msg = "No replacement found for parameter %s" % (org) + raise NoReplacementException(msg) + return str(v) + + ntext = re.sub("%([\\w\\d]+?)%", replacer, text) + return ntext + + +def welcome(program_action): + formatted_action = WELCOME_MAP.get(program_action) + lower = "!%s v%s!" % (formatted_action.upper(), VERSION) + welcome = r''' + ___ ____ _____ _ _ ____ _____ _ ____ _ __ + / _ \| _ \| ____| \ | / ___|_ _|/ \ / ___| |/ / +| | | | |_) | _| | \| \___ \ | | / _ \| | | ' / +| |_| | __/| |___| |\ |___) || |/ ___ \ |___| . \ + \___/|_| |_____|_| \_|____/ |_/_/ \_\____|_|\_\ + +''' + welcome = " " + welcome.strip() + lowerc = " " * 21 + colored(lower, 'blue') + msg = welcome + "\n" + lowerc + print(msg) + + +def rcf8222date(): + return strftime("%a, %d %b %Y %H:%M:%S", localtime()) + + +def fsSafeDate(): + return strftime("%m_%d_%G-%H-%M-%S", localtime()) diff --git a/devstack/__init__.py b/devstack/__init__.py new file mode 100644 index 00000000..e8e40359 --- /dev/null +++ b/devstack/__init__.py @@ -0,0 +1,14 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/devstack/packaging/Apt.py b/devstack/packaging/Apt.py new file mode 100644 index 00000000..9b94fef8 --- /dev/null +++ b/devstack/packaging/Apt.py @@ -0,0 +1,60 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import os + +import Packager +import Util +from Util import param_replace +import Shell +from Shell import execute + +APT_GET = ['apt-get'] +APT_REMOVE = ["purge", "-y"] # should we use remove or purge? +APT_INSTALL = ["install", "-y"] + +#make sure its non-interactive +os.putenv('DEBIAN_FRONTEND', 'noninteractive') + + +class AptPackager(Packager.Packager): + def __init__(self): + Packager.Packager.__init__(self) + + def _form_cmd(self, name, version): + cmd = name + if(version and len(version)): + cmd = cmd + "=" + version + return cmd + + def _do_cmd(self, base_cmd, pkgs): + pkgnames = pkgs.keys() + pkgnames.sort() + cmds = [] + for name in pkgnames: + version = None + info = pkgs.get(name) + if(info): + version = info.get("version") + torun = self._form_cmd(name, version) + cmds.append(torun) + if(len(cmds)): + cmd = APT_GET + base_cmd + cmds + execute(*cmd, run_as_root=True) + + def remove_batch(self, pkgs): + self._do_cmd(APT_REMOVE, pkgs) + + def install_batch(self, pkgs, params=None): + self._do_cmd(APT_INSTALL, pkgs) diff --git a/devstack/packaging/Yum.py b/devstack/packaging/Yum.py new file mode 100644 index 00000000..cba40b03 --- /dev/null +++ b/devstack/packaging/Yum.py @@ -0,0 +1,21 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import Packager + + +class YumPackager(Packager.Packager): + def __init__(self): + Packager.Packager.__init__(self) diff --git a/devstack/packaging/__init__.py b/devstack/packaging/__init__.py new file mode 100644 index 00000000..e8e40359 --- /dev/null +++ b/devstack/packaging/__init__.py @@ -0,0 +1,14 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/devstack/runners/Foreground.py b/devstack/runners/Foreground.py new file mode 100644 index 00000000..d0b1205b --- /dev/null +++ b/devstack/runners/Foreground.py @@ -0,0 +1,143 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import sys +import resource +import signal +import errno + +import Runner +import Util +import Exceptions +from Exceptions import (StartException, StopException) +import Logger +import Shell +from Shell import (unlink, mkdir, joinpths, write_file, + load_file, isfile) +import Trace + +# Maximum for the number of available file descriptors (when not found) +MAXFD = 2048 +MAX_KILL_TRY = 4 + +LOG = Logger.getLogger("install.runners.foreground") + +#trace constants +RUN = Runner.RUN_TYPE +RUN_TYPE = "FORK" +PID_FN = "PID_FN" +STDOUT_FN = "STDOUT_FN" +STDERR_FN = "STDERR_FN" +NAME = "NAME" + + +class ForegroundRunner(Runner.Runner): + def __init__(self): + Runner.Runner.__init__(self) + + def stop(self, name, *args, **kargs): + rootdir = kargs.get("trace_dir") + pidfile = joinpths(rootdir, name + ".pid") + stderr = joinpths(rootdir, name + ".stderr") + stdout = joinpths(rootdir, name + ".stdout") + tfname = Trace.trace_fn(rootdir, name) + if(isfile(pidfile) and isfile(tfname)): + pid = int(load_file(pidfile).strip()) + killed = False + lastmsg = "" + attempts = 1 + for attempt in range(0, MAX_KILL_TRY): + try: + os.kill(pid, signal.SIGKILL) + attempts += 1 + except OSError as (ec, msg): + if(ec == errno.ESRCH): + killed = True + break + lastmsg = msg + #trash the files + if(killed): + LOG.info("Killed pid %s in %s attempts" % (str(pid), str(attempts))) + LOG.info("Removing pid file %s" % (pidfile)) + unlink(pidfile) + LOG.info("Removing stderr file %s" % (stderr)) + unlink(stderr) + LOG.info("Removing stdout file %s" % (stdout)) + unlink(stdout) + LOG.info("Removing %s trace file %s" % (name, tfname)) + unlink(tfname) + else: + msg = "Could not stop program named %s after %s attempts - [%s]" % (name, MAX_KILL_TRY, lastmsg) + raise StopException(msg) + else: + msg = "No pid file could be found to terminate at %s" % (pidfile) + raise StopException(msg) + + def start(self, name, program, *args, **kargs): + tracedir = kargs.get("trace_dir") + appdir = kargs.get("app_dir") + pidfile = joinpths(tracedir, name + ".pid") + stderr = joinpths(tracedir, name + ".stderr") + stdout = joinpths(tracedir, name + ".stdout") + tracefn = Trace.trace_fn(tracedir, name) + tracefn = Trace.touch_trace(tracedir, name) + runtrace = Trace.Trace(tracefn) + runtrace.trace(RUN, RUN_TYPE) + runtrace.trace(PID_FN, pidfile) + runtrace.trace(STDERR_FN, stderr) + runtrace.trace(STDOUT_FN, stdout) + #fork to get daemon out + pid = os.fork() + if(pid == 0): + os.setsid() + pid = os.fork() + #fork to get daemon out - this time under init control + #and now fully detached (no shell possible) + if(pid == 0): + #move to where application should be + os.chdir(appdir) + #close other fds + limits = resource.getrlimit(resource.RLIMIT_NOFILE) + mkfd = limits[1] + if(mkfd == resource.RLIM_INFINITY): + mkfd = MAXFD + for fd in range(0, mkfd): + try: + os.close(fd) + except OSError: + #not open, thats ok + pass + #now adjust stderr and stdout + stdoh = open(stdout, "w") + stdeh = open(stderr, "w") + os.dup2(stdoh.fileno(), sys.stdout.fileno()) + os.dup2(stdeh.fileno(), sys.stderr.fileno()) + #now exec... + #the arguments to the child process should + #start with the name of the command being run + actualargs = [program] + list(args) + os.execlp(program, *actualargs) + else: + #write out the child pid + contents = str(pid) + "\n" + write_file(pidfile, contents) + #not exit or sys.exit, this is recommended + #since it will do the right cleanups that we want + #not calling any atexit functions, which would + #be bad right now + os._exit(0) + else: + return tracefn diff --git a/devstack/runners/Screen.py b/devstack/runners/Screen.py new file mode 100644 index 00000000..aa761713 --- /dev/null +++ b/devstack/runners/Screen.py @@ -0,0 +1,21 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import Runner + + +class Screen(Runner.Runner): + def __init__(self): + Runner.Runner.__init__(self) diff --git a/devstack/runners/__init__.py b/devstack/runners/__init__.py new file mode 100644 index 00000000..e8e40359 --- /dev/null +++ b/devstack/runners/__init__.py @@ -0,0 +1,14 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/stack b/stack new file mode 100755 index 00000000..c1996688 --- /dev/null +++ b/stack @@ -0,0 +1,263 @@ +#!/usr/bin/env python + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import os.path +import sys +import operator + +#TODO is this needed? +sys.path.append("devstack") + +import Logger +import Options +import Util +from Util import ( + welcome, + rcf8222date, + fsSafeDate, + determine_os, + get_pkg_list + ) +import Shell +from Shell import (mkdir, joinpths) +import Config +import Glance +import Horizon +import Keystone +import Nova +import Quantum +import Config +import Swift +import Db +import Rabbit +import Trace + +LOG = Logger.getLogger("install") + +# This determines what classes to use to install/uninstall/... +ACTION_CLASSES = { + Util.INSTALL: { + Util.NOVA: Nova.NovaInstaller, + Util.GLANCE: Glance.GlanceInstaller, + Util.QUANTUM: Quantum.QuantumInstaller, + Util.SWIFT: Swift.SwiftInstaller, + Util.HORIZON: Horizon.HorizonInstaller, + Util.KEYSTONE: Keystone.KeystoneInstaller, + Util.DB: Db.DBInstaller, + Util.RABBIT: Rabbit.RabbitInstaller, + }, + Util.UNINSTALL: { + Util.NOVA: Nova.NovaUninstaller, + Util.GLANCE: Glance.GlanceUninstaller, + Util.QUANTUM: Quantum.QuantumUninstaller, + Util.SWIFT: Swift.SwiftUninstaller, + Util.HORIZON: Horizon.HorizonUninstaller, + Util.KEYSTONE: Keystone.KeystoneUninstaller, + Util.DB: Db.DBUninstaller, + Util.RABBIT: Rabbit.RabbitUninstaller, + }, + Util.START: { + Util.NOVA: Nova.NovaRuntime, + Util.GLANCE: Glance.GlanceRuntime, + Util.QUANTUM: Quantum.QuantumRuntime, + Util.SWIFT: Swift.SwiftRuntime, + Util.HORIZON: Horizon.HorizonRuntime, + Util.KEYSTONE: Keystone.KeystoneRuntime, + Util.DB: Db.DBRuntime, + Util.RABBIT: Rabbit.RabbitRuntime, + }, + Util.STOP: { + Util.NOVA: Nova.NovaRuntime, + Util.GLANCE: Glance.GlanceRuntime, + Util.QUANTUM: Quantum.QuantumRuntime, + Util.SWIFT: Swift.SwiftRuntime, + Util.HORIZON: Horizon.HorizonRuntime, + Util.KEYSTONE: Keystone.KeystoneRuntime, + Util.DB: Db.DBRuntime, + Util.RABBIT: Rabbit.RabbitRuntime, + }, +} + +# Actions which need dependent actions to occur +DEP_ACTIONS_NEEDED = set([Util.START, Util.STOP, Util.INSTALL]) + + +def get_pkg_manager(distro): + klass = None + if(distro == Util.UBUNTU12): + #late import required + #TODO better way to do this? + from packaging import Apt + klass = Apt.AptPackager + elif(distro == Util.RHEL6): + #late import required + #TODO better way to do this? + from packaging import Yum + klass = Yum.YumPackager + return klass() + + +def get_config(): + LOG.info("Loading config from %s" % (Util.CFG_LOC)) + cfg = Config.EnvConfigParser() + cfg.read(Util.CFG_LOC) + return cfg + + +def stop(components, distro, rootdir): + pkg_manager = get_pkg_manager(distro) + cfg = get_config() + LOG.info("Will stop [%s] from %s" % (", ".join(components), rootdir)) + klass_lookup = ACTION_CLASSES.get(Util.START) + for c in components: + klass = klass_lookup.get(c) + instance = klass(components=components, distro=distro, pkg=pkg_manager, cfg=cfg, root=rootdir) + LOG.info("Stopping %s." % (c)) + instance.stop() + LOG.info("Finished stop of %s" % (c)) + return None + + +def start(components, distro, rootdir): + pkg_manager = get_pkg_manager(distro) + cfg = get_config() + LOG.info("Will start [%s] from %s" % (", ".join(components), rootdir)) + klass_lookup = ACTION_CLASSES.get(Util.START) + locations = [] + for c in components: + klass = klass_lookup.get(c) + instance = klass(components=components, distro=distro, pkg=pkg_manager, cfg=cfg, root=rootdir) + LOG.info("Starting %s." % (c)) + trace_locs = instance.start() or [] + LOG.info("Finished start of %s - check [%s] for traces of what happened." % (c, ", ".join(trace_locs))) + locations = locations + trace_locs + return locations + + +def install(components, distro, rootdir): + pkg_manager = get_pkg_manager(distro) + cfg = get_config() + mkdir(rootdir) + LOG.info("Will install [%s] and store in %s." % (", ".join(components), rootdir)) + klass_lookup = ACTION_CLASSES.get(Util.INSTALL) + traces = [] + for c in components: + klass = klass_lookup.get(c) + instance = klass(components=components, distro=distro, pkg=pkg_manager, cfg=cfg, root=rootdir) + LOG.info("Downloading %s." % (c)) + instance.download() + LOG.info("Configuring %s." % (c)) + instance.configure() + LOG.info("Installing %s." % (c)) + trace = instance.install() + LOG.info("Finished install of %s - check %s for traces of what happened." % (c, trace)) + traces.append(trace) + return traces + + +def uninstall(components, distro, uninstalldir): + pkg_manager = get_pkg_manager(distro) + cfg = get_config() + LOG.info("Will uninstall [%s] with traces from directory %s." % (", ".join(components), uninstalldir)) + klass_lookup = ACTION_CLASSES.get(Util.UNINSTALL) + for c in components: + klass = klass_lookup.get(c) + instance = klass(components=components, distro=distro, pkg=pkg_manager, cfg=cfg, root=uninstalldir) + LOG.info("Unconfiguring %s." % (c)) + instance.unconfigure() + LOG.info("Uninstalling %s." % (c)) + instance.uninstall() + return None + + +#what functions to activate for each action +FUNC_MAP = { + Util.INSTALL: install, + Util.UNINSTALL: uninstall, + Util.START: start, + Util.STOP: stop +} + + +def main(): + me = __file__ + args = Options.parse() + components = args.get("component") or [] + if(len(components) == 0): + #assume them all?? + components = list(Util.NAMES) + components = set([x.lower() for x in components]) + applicable = set(Util.NAMES).intersection(components) + if(len(applicable) == 0): + LOG.error("No valid components specified!") + LOG.info("Perhaps you should try %s --help" % (me)) + return 1 + action = args.get("action") or "" + action = action.strip() + action = action.lower() + if(not (action in Util.ACTIONS)): + LOG.error("No valid action specified!") + LOG.info("Perhaps you should try %s --help" % (me)) + return 1 + rootdir = args.get("dir") or "" + if(len(rootdir) == 0): + LOG.error("No valid root directory specified!") + LOG.info("Perhaps you should try %s --help" % (me)) + return 1 + #check if implemented yet + if(not action in ACTION_CLASSES or not action in FUNC_MAP): + LOG.error("Action %s not implemented yet!" % (action)) + return 1 + #ensure os is known + (install_os, plt) = determine_os() + if(install_os == None): + LOG.error("Unsupported operating system/distro: %s" % (plt)) + return 1 + if(os.path.isdir(rootdir) and action == Util.INSTALL): + LOG.error("Root directory [%s] already exists! Please remove it!" % (rootdir)) + return 1 + #start it + welcome(action) + if(action in DEP_ACTIONS_NEEDED): + # need to figure out deps for components (if any) + deps = list() + for c in applicable: + cdeps = list(set(Util.fetch_deps(c))) + if(len(cdeps)): + LOG.info("Having to %s [%s] since they are dependencies for %s." % (action, ", ".join(cdeps), c)) + deps = deps + cdeps + deps = deps + [c] + applicable = set(deps) + #get the right component order (by priority) + mporder = dict() + for c in applicable: + mporder[c] = Util.NAMES_PRIORITY.get(c) + #sort by priority value + priororder = sorted(mporder.iteritems(), key=operator.itemgetter(1)) + componentorder = [x[0] for x in priororder] + funcAction = FUNC_MAP.get(action) + LOG.info("Starting action [%s] on %s for operating system/distro [%s]" % (action, rcf8222date(), install_os)) + resultList = funcAction(componentorder, install_os, rootdir) + LOG.info("Finished action [%s] on %s" % (action, rcf8222date())) + if(resultList): + msg = "Check [%s] for traces of what happened." % (", ".join(resultList)) + LOG.info(msg) + return 0 + + +if __name__ == "__main__": + rc = main() + sys.exit(rc) diff --git a/utils/pkgfinder.pl b/utils/pkgfinder.pl new file mode 100755 index 00000000..786d19e6 --- /dev/null +++ b/utils/pkgfinder.pl @@ -0,0 +1,170 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; + +use FileHandle; +use Term::ANSIColor qw(:constants); + + +sub printinfo +{ + print BOLD, BLUE, "INFO: "."", RESET; + println("@_"); +} + +sub printerror +{ + print BOLD, RED, "ERROR: @_"."\n", RESET; +} + +sub run +{ + my ($prog, $die) = @_; + #printinfo("Runing command: $prog"); + my $res = qx/$prog/; + my $ok = 0; + my $rc = $? >> 8; + if($rc == 0) + { + $ok = 1; + } + if($ok == 0 && $die == 1) + { + printerror("Failed running $prog"); + exit(1); + } + $res = trim($res); + my $out = {}; + $out->{'status'} = $rc; + $out->{'output'} = $res; + return $out; +} + +sub println +{ + my $arg = shift; + if(!defined($arg)) + { + $arg = ''; + } + return print($arg."\n"); +} + +sub trim +{ + my $string = shift; + $string =~ s/^\s+//; + $string =~ s/\s+$//; + return $string; +} + +my $argc = scalar(@ARGV); +if($argc == 0) +{ + println($0. " pkglist"); + exit(1); +} + + +my $fn = $ARGV[0]; +my $fh = new FileHandle($fn, "r") || die("Could not open $fn");; +my @lines = <$fh>; +$fh->close(); + +my @all = (); +my $ks = {}; + +for my $line (@lines) +{ + $line = trim($line); + if(length($line) == 0) + { + next; + } + my @pieces = split /\s+/, $line; + for my $piece (@pieces) + { + $piece = trim($piece); + if(length($piece) == 0) + { + next; + } + if(defined($ks->{$piece})) + { + next; + } + push(@all, $piece); + $ks->{$piece} = 1; + } +} + +@all = sort(@all); +printinfo("Finding info about packages:"); +println(join(", ", @all).""); + +my $info = {}; +for my $pkg (@all) +{ + printinfo("Finding information about $pkg"); + my $cmd = "apt-cache showpkg $pkg"; + my $out = run($cmd, 1)->{'output'}; + my $version = undef; + if($out =~ /Versions:\s+([\S]+)\s+/msi) + { + $version = $1; + } + else + { + printerror("No version found for $pkg"); + exit(1); + } + $cmd = "apt-cache depends $pkg"; + $out = run($cmd, 1)->{'output'}; + my @tmplines = split /\n|\r/, $out; + my @deps = (); + for my $aline (@tmplines) + { + if($aline =~ /\s+Depends:\s*(\S+)\s*/i) + { + my $dep = trim($1); + if(length($dep) > 0) + { + if($dep =~ /[<>]/) + { + #not sure why we get these... + next; + } + push(@deps, $dep); + } + } + } + my $d = {}; + $d->{'deps'} = \@deps; + $d->{'version'} = $version; + $info->{$pkg} = $d; +} + +for my $pkg (@all) +{ + my $data = $info->{$pkg}; + my $version = $data->{version}; + print STDERR ("+Package name: $pkg\n"); + print STDERR ("+Package version: $version\n"); + my @deps = @{$data->{deps}}; + @deps = sort(@deps); + my $tmpk = {}; + print STDERR ("+Dependencies:\n"); + for my $dep (@deps) + { + if(defined($tmpk->{$dep})) + { + next; + } + print STDERR ("\t"."$dep\n"); + $tmpk->{$dep} = 1; + } +} + +exit(0); +