diff --git a/README.rst b/README.rst index 4a99ffbc3..b4d55639e 100644 --- a/README.rst +++ b/README.rst @@ -47,29 +47,6 @@ If the cloud being used has no default_floating_pool defined in nova.conf, you will need to define a pool name using the nodepool yaml file to use floating ips. - -Set up database for interactive testing: - -.. code-block:: bash - - mysql -u root - - mysql> create database nodepool; - mysql> GRANT ALL ON nodepool.* TO 'nodepool'@'localhost'; - mysql> flush privileges; - -Set up database for unit tests: - -.. code-block:: bash - - mysql -u root - mysql> grant all privileges on *.* to 'openstack_citest'@'localhost' identified by 'openstack_citest' with grant option; - mysql> flush privileges; - mysql> create database openstack_citest; - -Note that the script tools/test-setup.sh can be used for the step -above. - Export variable for your ssh key so you can log into the created instances: .. code-block:: bash @@ -92,9 +69,3 @@ Use the following tool to check on progress: .. code-block:: bash nodepool image-list - -After each run (the fake nova provider is only in-memory): - -.. code-block:: bash - - mysql> delete from snapshot_image; delete from node; diff --git a/bindep.txt b/bindep.txt index 426c5db61..81d5723a5 100644 --- a/bindep.txt +++ b/bindep.txt @@ -1,8 +1,6 @@ # This is a cross-platform list tracking distribution packages needed by tests; # see http://docs.openstack.org/infra/bindep/ for additional information. -mysql-client [test] -mysql-server [test] python-dev [platform:dpkg test] python-devel [platform:rpm test] zookeeperd [platform:dpkg test] diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 7149208e0..0f68b1be0 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -101,7 +101,6 @@ EOF function nodepool_write_config { sudo mkdir -p $(dirname $NODEPOOL_CONFIG) sudo mkdir -p $(dirname $NODEPOOL_SECURE) - local dburi=$(database_connection_url nodepool) cat > /tmp/logging.conf < /tmp/secure.conf << EOF -[database] -# The mysql password here may be different depending on your -# devstack install, you should double check it (the devstack var -# is MYSQL_PASSWORD and if unset devstack should prompt you for -# the value). -dburi: $dburi +# Empty EOF sudo mv /tmp/secure.conf $NODEPOOL_SECURE @@ -174,11 +168,6 @@ EOF # example element. elements-dir: $(dirname $NODEPOOL_CONFIG)/elements images-dir: $NODEPOOL_DIB_BASE_PATH/images -# The mysql password here may be different depending on your -# devstack install, you should double check it (the devstack var -# is MYSQL_PASSWORD and if unset devstack should prompt you for -# the value). -dburi: '$dburi' zookeeper-servers: - host: localhost @@ -378,7 +367,6 @@ EOF mkdir -p $HOME/.cache/openstack/ } -# Initialize database # Create configs # Setup custom flavor function configure_nodepool { @@ -390,10 +378,6 @@ function configure_nodepool { # write the elements nodepool_write_elements - - # builds a fresh db - recreate_database nodepool - } function start_nodepool { diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index e41d4a80f..017d15751 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -4,23 +4,8 @@ Configuration ============= Nodepool reads its secure configuration from ``/etc/nodepool/secure.conf`` -by default. The secure file is a standard ini config file, with -one section for the database. - - [database] - dburi={dburi} - -Following settings are available:: - -**required** - - ``dburi`` - Indicates the URI for the database connection. See the `SQLAlchemy - documentation - `_ - for the syntax. Example:: - - dburi='mysql+pymysql://nodepool@localhost/nodepool' +by default. The secure file is a standard ini config file. Note that this +file is currently unused, but may be in the future. Nodepool reads its configuration from ``/etc/nodepool/nodepool.yaml`` by default. The configuration file follows the standard YAML syntax diff --git a/doc/source/installation.rst b/doc/source/installation.rst index fc9f25cfc..040c748d0 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -55,8 +55,7 @@ Configuration Nodepool has two required configuration files: secure.conf and nodepool.yaml, and an optional logging configuration file logging.conf. The secure.conf file is used to store nodepool configurations that contain -sensitive data, such as the Nodepool database password and Jenkins -api key. The nodepool.yaml files is used to store all other +sensitive data. The nodepool.yaml files is used to store all other configurations. The logging configuration file is in the standard python logging diff --git a/nodepool/cmd/config_validator.py b/nodepool/cmd/config_validator.py index f8b463762..285ac7da2 100644 --- a/nodepool/cmd/config_validator.py +++ b/nodepool/cmd/config_validator.py @@ -105,7 +105,6 @@ class ConfigValidator: top_level = { 'elements-dir': str, 'images-dir': str, - 'dburi': str, 'zookeeper-servers': [{ 'host': str, 'port': int, diff --git a/nodepool/config.py b/nodepool/config.py index aa77cc11e..ed5f1602b 100644 --- a/nodepool/config.py +++ b/nodepool/config.py @@ -131,12 +131,10 @@ def loadConfig(config_path): newconfig = Config() newconfig.db = None - newconfig.dburi = None newconfig.providers = {} newconfig.labels = {} newconfig.elementsdir = config.get('elements-dir') newconfig.imagesdir = config.get('images-dir') - newconfig.dburi = None newconfig.provider_managers = {} newconfig.zookeeper_servers = {} newconfig.diskimages = {} @@ -277,7 +275,7 @@ def loadSecureConfig(config, secure_config_path): secure = ConfigParser.ConfigParser() secure.readfp(open(secure_config_path)) - config.dburi = secure.get('database', 'dburi') + #config.dburi = secure.get('database', 'dburi') def _cloudKwargsFromProvider(provider): diff --git a/nodepool/nodedb.py b/nodepool/nodedb.py deleted file mode 100644 index d7ae959b7..000000000 --- a/nodepool/nodedb.py +++ /dev/null @@ -1,247 +0,0 @@ -# Copyright (C) 2011-2014 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -import time - -# States: -# The cloud provider is building this machine. We have an ID, but it's -# not ready for use. -BUILDING = 1 -# The machine is ready for use. -READY = 2 -# This can mean in-use, or used but complete. -USED = 3 -# Delete this machine immediately. -DELETE = 4 -# Keep this machine indefinitely. -HOLD = 5 -# Acceptance testing (pre-ready) -TEST = 6 - - -STATE_NAMES = { - BUILDING: 'building', - READY: 'ready', - USED: 'used', - DELETE: 'delete', - HOLD: 'hold', - TEST: 'test', - } - -from sqlalchemy import Table, Column, Integer, String, \ - MetaData, create_engine -from sqlalchemy.orm import scoped_session, mapper -from sqlalchemy.orm.session import Session, sessionmaker - -metadata = MetaData() - -node_table = Table( - 'node', metadata, - Column('id', Integer, primary_key=True), - Column('provider_name', String(255), index=True, nullable=False), - Column('label_name', String(255), index=True, nullable=False), - Column('target_name', String(255), index=True, nullable=False), - Column('manager_name', String(255)), - # Machine name - Column('hostname', String(255), index=True), - # Eg, jenkins node name - Column('nodename', String(255), index=True), - # Provider assigned id for this machine - Column('external_id', String(255)), - # Provider availability zone for this machine - Column('az', String(255)), - # Primary IP address - Column('ip', String(255)), - # Internal/fixed IP address - Column('ip_private', String(255)), - # One of the above values - Column('state', Integer), - # Time of last state change - Column('state_time', Integer), - # Comment about the state of the node - used to annotate held nodes - Column('comment', String(255)), - mysql_engine='InnoDB', - ) -job_table = Table( - 'job', metadata, - Column('id', Integer, primary_key=True), - # The name of the job - Column('name', String(255), index=True), - # Automatically hold up to this number of nodes that fail this job - Column('hold_on_failure', Integer), - mysql_engine='InnoDB', - ) - - -class Node(object): - def __init__(self, provider_name, label_name, target_name, az, - hostname=None, external_id=None, ip=None, ip_private=None, - manager_name=None, state=BUILDING, comment=None): - self.provider_name = provider_name - self.label_name = label_name - self.target_name = target_name - self.manager_name = manager_name - self.external_id = external_id - self.az = az - self.ip = ip - self.ip_private = ip_private - self.hostname = hostname - self.state = state - self.comment = comment - - def delete(self): - session = Session.object_session(self) - session.delete(self) - session.commit() - - @property - def state(self): - return self._state - - @state.setter - def state(self, state): - self._state = state - self.state_time = int(time.time()) - session = Session.object_session(self) - if session: - session.commit() - - -class Job(object): - def __init__(self, name=None, hold_on_failure=0): - self.name = name - self.hold_on_failure = hold_on_failure - - def delete(self): - session = Session.object_session(self) - session.delete(self) - session.commit() - - -mapper(Job, job_table) - - -mapper(Node, node_table, - properties=dict( - _state=node_table.c.state)) - - -class NodeDatabase(object): - def __init__(self, dburi): - engine_kwargs = dict(echo=False, pool_recycle=3600) - if 'sqlite:' not in dburi: - engine_kwargs['max_overflow'] = -1 - - self.engine = create_engine(dburi, **engine_kwargs) - metadata.create_all(self.engine) - self.session_factory = sessionmaker(bind=self.engine) - self.session = scoped_session(self.session_factory) - - def getSession(self): - return NodeDatabaseSession(self.session) - - -class NodeDatabaseSession(object): - def __init__(self, session): - self.session = session - - def __enter__(self): - return self - - def __exit__(self, etype, value, tb): - if etype: - self.session().rollback() - else: - self.session().commit() - self.session().close() - self.session = None - - def abort(self): - self.session().rollback() - - def commit(self): - self.session().commit() - - def delete(self, obj): - self.session().delete(obj) - - def getNodes(self, provider_name=None, label_name=None, target_name=None, - state=None): - exp = self.session().query(Node).order_by( - node_table.c.provider_name, - node_table.c.label_name) - if provider_name: - exp = exp.filter_by(provider_name=provider_name) - if label_name: - exp = exp.filter_by(label_name=label_name) - if target_name: - exp = exp.filter_by(target_name=target_name) - if state: - exp = exp.filter(node_table.c.state == state) - return exp.all() - - def createNode(self, *args, **kwargs): - new = Node(*args, **kwargs) - self.session().add(new) - self.commit() - return new - - def getNode(self, id): - nodes = self.session().query(Node).filter_by(id=id).all() - if not nodes: - return None - return nodes[0] - - def getNodeByHostname(self, hostname): - nodes = self.session().query(Node).filter_by(hostname=hostname).all() - if not nodes: - return None - return nodes[0] - - def getNodeByNodename(self, nodename): - nodes = self.session().query(Node).filter_by(nodename=nodename).all() - if not nodes: - return None - return nodes[0] - - def getNodeByExternalID(self, provider_name, external_id): - nodes = self.session().query(Node).filter_by( - provider_name=provider_name, - external_id=external_id).all() - if not nodes: - return None - return nodes[0] - - def getJob(self, id): - jobs = self.session().query(Job).filter_by(id=id).all() - if not jobs: - return None - return jobs[0] - - def getJobByName(self, name): - jobs = self.session().query(Job).filter_by(name=name).all() - if not jobs: - return None - return jobs[0] - - def getJobs(self): - return self.session().query(Job).all() - - def createJob(self, *args, **kwargs): - new = Job(*args, **kwargs) - self.session().add(new) - self.commit() - return new diff --git a/nodepool/tests/__init__.py b/nodepool/tests/__init__.py index 8317c36d5..3d8cb6d2a 100644 --- a/nodepool/tests/__init__.py +++ b/nodepool/tests/__init__.py @@ -18,17 +18,14 @@ import glob import logging import os -import pymysql import random import string import subprocess import threading import tempfile import time -import uuid import fixtures -import lockfile import kazoo.client import testtools @@ -230,47 +227,6 @@ class RoundRobinTestCase(object): self.allocations)) -class MySQLSchemaFixture(fixtures.Fixture): - def setUp(self): - super(MySQLSchemaFixture, self).setUp() - - random_bits = ''.join(random.choice(string.ascii_lowercase + - string.ascii_uppercase) - for x in range(8)) - self.name = '%s_%s' % (random_bits, os.getpid()) - self.passwd = uuid.uuid4().hex - lock = lockfile.LockFile('/tmp/nodepool-db-schema-lockfile') - with lock: - db = pymysql.connect(host="localhost", - user="openstack_citest", - passwd="openstack_citest", - db="openstack_citest") - cur = db.cursor() - cur.execute("create database %s" % self.name) - cur.execute( - "grant all on %s.* to '%s'@'localhost' identified by '%s'" % - (self.name, self.name, self.passwd)) - cur.execute("flush privileges") - - self.dburi = 'mysql+pymysql://%s:%s@localhost/%s' % (self.name, - self.passwd, - self.name) - self.addDetail('dburi', testtools.content.text_content(self.dburi)) - self.addCleanup(self.cleanup) - - def cleanup(self): - lock = lockfile.LockFile('/tmp/nodepool-db-schema-lockfile') - with lock: - db = pymysql.connect(host="localhost", - user="openstack_citest", - passwd="openstack_citest", - db="openstack_citest") - cur = db.cursor() - cur.execute("drop database %s" % self.name) - cur.execute("drop user '%s'@'localhost'" % self.name) - cur.execute("flush privileges") - - class BuilderFixture(fixtures.Fixture): def __init__(self, configfile, cleanup_interval): super(BuilderFixture, self).__init__() @@ -296,9 +252,6 @@ class DBTestCase(BaseTestCase): def setUp(self): super(DBTestCase, self).setUp() self.log = logging.getLogger("tests") - f = MySQLSchemaFixture() - self.useFixture(f) - self.dburi = f.dburi self.secure_conf = self._setup_secure() self.setupZK() @@ -333,7 +286,8 @@ class DBTestCase(BaseTestCase): (fd, path) = tempfile.mkstemp() with open(configfile) as conf_fd: config = conf_fd.read() - os.write(fd, config.format(dburi=self.dburi)) + os.write(fd, config) + #os.write(fd, config.format(dburi=self.dburi)) os.close(fd) return path diff --git a/nodepool/tests/fixtures/secure.conf b/nodepool/tests/fixtures/secure.conf index 08f7d8909..b7db25411 100644 --- a/nodepool/tests/fixtures/secure.conf +++ b/nodepool/tests/fixtures/secure.conf @@ -1,2 +1 @@ -[database] -dburi={dburi} +# Empty diff --git a/requirements.txt b/requirements.txt index a11843862..53ebc2e75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ python-daemon>=2.0.4,<2.1.0 extras statsd>=3.0 sqlalchemy>=0.8.2,<1.1.0 -PyMySQL PrettyTable>=0.6,<0.8 # shade has a looser requirement on six than nodepool, so install six first six>=1.7.0 diff --git a/tools/fake-secure.conf b/tools/fake-secure.conf index d29d9c094..b7db25411 100644 --- a/tools/fake-secure.conf +++ b/tools/fake-secure.conf @@ -1,2 +1 @@ -[database] -dburi=mysql+pymysql://nodepool@localhost/nodepool +# Empty diff --git a/tools/test-setup.sh b/tools/test-setup.sh deleted file mode 100755 index f4a045869..000000000 --- a/tools/test-setup.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -xe - -# This script will be run by OpenStack CI before unit tests are run, -# it sets up the test system as needed. -# Developers should setup their test systems in a similar way. - -# This setup needs to be run as a user that can run sudo. - -# The root password for the MySQL database; pass it in via -# MYSQL_ROOT_PW. -DB_ROOT_PW=${MYSQL_ROOT_PW:-insecure_slave} - -# This user and its password are used by the tests, if you change it, -# your tests might fail. -DB_USER=openstack_citest -DB_PW=openstack_citest - -sudo -H mysqladmin -u root password $DB_ROOT_PW - -# It's best practice to remove anonymous users from the database. If -# a anonymous user exists, then it matches first for connections and -# other connections from that host will not work. -sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e " - DELETE FROM mysql.user WHERE User=''; - FLUSH PRIVILEGES; - GRANT ALL PRIVILEGES ON *.* - TO '$DB_USER'@'%' identified by '$DB_PW' WITH GRANT OPTION;" - -# Now create our database. -mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e " - SET default_storage_engine=MYISAM; - DROP DATABASE IF EXISTS openstack_citest; - CREATE DATABASE openstack_citest CHARACTER SET utf8;" diff --git a/tools/zuul-nodepool-integration/secure.conf b/tools/zuul-nodepool-integration/secure.conf index d29d9c094..b7db25411 100644 --- a/tools/zuul-nodepool-integration/secure.conf +++ b/tools/zuul-nodepool-integration/secure.conf @@ -1,2 +1 @@ -[database] -dburi=mysql+pymysql://nodepool@localhost/nodepool +# Empty