Merge "Remove MySQL" into feature/zuulv3

README.rst

@@ -47,29 +47,6 @@ If the cloud being used has no default_floating_pool defined in nova.conf,
 you will need to define a pool name using the nodepool yaml file to use
 floating ips.
 
-
-Set up database for interactive testing:
-
-.. code-block:: bash
-
-    mysql -u root
-
-    mysql> create database nodepool;
-    mysql> GRANT ALL ON nodepool.* TO 'nodepool'@'localhost';
-    mysql> flush privileges;
-
-Set up database for unit tests:
-
-.. code-block:: bash
-
-    mysql -u root
-    mysql> grant all privileges on *.* to 'openstack_citest'@'localhost' identified by 'openstack_citest' with grant option;
-    mysql> flush privileges;
-    mysql> create database openstack_citest;
-
-Note that the script tools/test-setup.sh can be used for the step
-above.
-
 Export variable for your ssh key so you can log into the created instances:
 
 .. code-block:: bash
@@ -92,9 +69,3 @@ Use the following tool to check on progress:
 .. code-block:: bash
 
     nodepool image-list
-
-After each run (the fake nova provider is only in-memory):
-
-.. code-block:: bash
-
-    mysql> delete from snapshot_image; delete from node;

bindep.txt

@@ -1,8 +1,6 @@
 # This is a cross-platform list tracking distribution packages needed by tests;
 # see http://docs.openstack.org/infra/bindep/ for additional information.
 
-mysql-client [test]
-mysql-server [test]
 python-dev [platform:dpkg test]
 python-devel [platform:rpm test]
 zookeeperd [platform:dpkg test]

devstack/plugin.sh

@@ -101,7 +101,6 @@ EOF
 function nodepool_write_config {
     sudo mkdir -p $(dirname $NODEPOOL_CONFIG)
     sudo mkdir -p $(dirname $NODEPOOL_SECURE)
-    local dburi=$(database_connection_url nodepool)
 
     cat > /tmp/logging.conf <<EOF
 [formatters]
@@ -149,12 +148,7 @@ EOF
     sudo mv /tmp/logging.conf $NODEPOOL_LOGGING
 
     cat > /tmp/secure.conf << EOF
-[database]
+# Empty
-# The mysql password here may be different depending on your
-# devstack install, you should double check it (the devstack var
-# is MYSQL_PASSWORD and if unset devstack should prompt you for
-# the value).
-dburi: $dburi
 EOF
     sudo mv /tmp/secure.conf $NODEPOOL_SECURE
 
@@ -174,11 +168,6 @@ EOF
 # example element.
 elements-dir: $(dirname $NODEPOOL_CONFIG)/elements
 images-dir: $NODEPOOL_DIB_BASE_PATH/images
-# The mysql password here may be different depending on your
-# devstack install, you should double check it (the devstack var
-# is MYSQL_PASSWORD and if unset devstack should prompt you for
-# the value).
-dburi: '$dburi'
 
 zookeeper-servers:
   - host: localhost
@@ -378,7 +367,6 @@ EOF
     mkdir -p $HOME/.cache/openstack/
 }
 
-# Initialize database
 # Create configs
 # Setup custom flavor
 function configure_nodepool {
@@ -390,10 +378,6 @@ function configure_nodepool {
 
     # write the elements
     nodepool_write_elements
-
-    # builds a fresh db
-    recreate_database nodepool
-
 }
 
 function start_nodepool {

doc/source/configuration.rst

@@ -4,23 +4,8 @@ Configuration
 =============
 
 Nodepool reads its secure configuration from ``/etc/nodepool/secure.conf``
-by default. The secure file is a standard ini config file, with
+by default. The secure file is a standard ini config file. Note that this
-one section for the database.
+file is currently unused, but may be in the future.
-
-  [database]
-  dburi={dburi}
-
-Following settings are available::
-
-**required**
-
-  ``dburi``
-  Indicates the URI for the database connection.  See the `SQLAlchemy
-  documentation
-  <http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>`_
-  for the syntax.  Example::
-
-    dburi='mysql+pymysql://nodepool@localhost/nodepool'
 
 Nodepool reads its configuration from ``/etc/nodepool/nodepool.yaml``
 by default.  The configuration file follows the standard YAML syntax

doc/source/installation.rst

@@ -55,8 +55,7 @@ Configuration
 Nodepool has two required configuration files: secure.conf and
 nodepool.yaml, and an optional logging configuration file logging.conf.
 The secure.conf file is used to store nodepool configurations that contain
-sensitive data, such as the Nodepool database password and Jenkins
+sensitive data. The nodepool.yaml files is used to store all other
-api key. The nodepool.yaml files is used to store all other
 configurations.
 
 The logging configuration file is in the standard python logging

nodepool/cmd/config_validator.py

@@ -105,7 +105,6 @@ class ConfigValidator:
         top_level = {
             'elements-dir': str,
             'images-dir': str,
-            'dburi': str,
             'zookeeper-servers': [{
                 'host': str,
                 'port': int,

nodepool/config.py

@@ -131,12 +131,10 @@ def loadConfig(config_path):
 
     newconfig = Config()
     newconfig.db = None
-    newconfig.dburi = None
     newconfig.providers = {}
     newconfig.labels = {}
     newconfig.elementsdir = config.get('elements-dir')
     newconfig.imagesdir = config.get('images-dir')
-    newconfig.dburi = None
     newconfig.provider_managers = {}
     newconfig.zookeeper_servers = {}
     newconfig.diskimages = {}
@@ -277,7 +275,7 @@ def loadSecureConfig(config, secure_config_path):
     secure = ConfigParser.ConfigParser()
     secure.readfp(open(secure_config_path))
 
-    config.dburi = secure.get('database', 'dburi')
+    #config.dburi = secure.get('database', 'dburi')
 
 
 def _cloudKwargsFromProvider(provider):

nodepool/nodedb.py

@@ -1,247 +0,0 @@
-# Copyright (C) 2011-2014 OpenStack Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-#
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import time
-
-# States:
-# The cloud provider is building this machine.  We have an ID, but it's
-# not ready for use.
-BUILDING = 1
-# The machine is ready for use.
-READY = 2
-# This can mean in-use, or used but complete.
-USED = 3
-# Delete this machine immediately.
-DELETE = 4
-# Keep this machine indefinitely.
-HOLD = 5
-# Acceptance testing (pre-ready)
-TEST = 6
-
-
-STATE_NAMES = {
-    BUILDING: 'building',
-    READY: 'ready',
-    USED: 'used',
-    DELETE: 'delete',
-    HOLD: 'hold',
-    TEST: 'test',
-    }
-
-from sqlalchemy import Table, Column, Integer, String, \
-    MetaData, create_engine
-from sqlalchemy.orm import scoped_session, mapper
-from sqlalchemy.orm.session import Session, sessionmaker
-
-metadata = MetaData()
-
-node_table = Table(
-    'node', metadata,
-    Column('id', Integer, primary_key=True),
-    Column('provider_name', String(255), index=True, nullable=False),
-    Column('label_name', String(255), index=True, nullable=False),
-    Column('target_name', String(255), index=True, nullable=False),
-    Column('manager_name', String(255)),
-    # Machine name
-    Column('hostname', String(255), index=True),
-    # Eg, jenkins node name
-    Column('nodename', String(255), index=True),
-    # Provider assigned id for this machine
-    Column('external_id', String(255)),
-    # Provider availability zone for this machine
-    Column('az', String(255)),
-    # Primary IP address
-    Column('ip', String(255)),
-    # Internal/fixed IP address
-    Column('ip_private', String(255)),
-    # One of the above values
-    Column('state', Integer),
-    # Time of last state change
-    Column('state_time', Integer),
-    # Comment about the state of the node - used to annotate held nodes
-    Column('comment', String(255)),
-    mysql_engine='InnoDB',
-    )
-job_table = Table(
-    'job', metadata,
-    Column('id', Integer, primary_key=True),
-    # The name of the job
-    Column('name', String(255), index=True),
-    # Automatically hold up to this number of nodes that fail this job
-    Column('hold_on_failure', Integer),
-    mysql_engine='InnoDB',
-    )
-
-
-class Node(object):
-    def __init__(self, provider_name, label_name, target_name, az,
-                 hostname=None, external_id=None, ip=None, ip_private=None,
-                 manager_name=None, state=BUILDING, comment=None):
-        self.provider_name = provider_name
-        self.label_name = label_name
-        self.target_name = target_name
-        self.manager_name = manager_name
-        self.external_id = external_id
-        self.az = az
-        self.ip = ip
-        self.ip_private = ip_private
-        self.hostname = hostname
-        self.state = state
-        self.comment = comment
-
-    def delete(self):
-        session = Session.object_session(self)
-        session.delete(self)
-        session.commit()
-
-    @property
-    def state(self):
-        return self._state
-
-    @state.setter
-    def state(self, state):
-        self._state = state
-        self.state_time = int(time.time())
-        session = Session.object_session(self)
-        if session:
-            session.commit()
-
-
-class Job(object):
-    def __init__(self, name=None, hold_on_failure=0):
-        self.name = name
-        self.hold_on_failure = hold_on_failure
-
-    def delete(self):
-        session = Session.object_session(self)
-        session.delete(self)
-        session.commit()
-
-
-mapper(Job, job_table)
-
-
-mapper(Node, node_table,
-       properties=dict(
-           _state=node_table.c.state))
-
-
-class NodeDatabase(object):
-    def __init__(self, dburi):
-        engine_kwargs = dict(echo=False, pool_recycle=3600)
-        if 'sqlite:' not in dburi:
-            engine_kwargs['max_overflow'] = -1
-
-        self.engine = create_engine(dburi, **engine_kwargs)
-        metadata.create_all(self.engine)
-        self.session_factory = sessionmaker(bind=self.engine)
-        self.session = scoped_session(self.session_factory)
-
-    def getSession(self):
-        return NodeDatabaseSession(self.session)
-
-
-class NodeDatabaseSession(object):
-    def __init__(self, session):
-        self.session = session
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, etype, value, tb):
-        if etype:
-            self.session().rollback()
-        else:
-            self.session().commit()
-        self.session().close()
-        self.session = None
-
-    def abort(self):
-        self.session().rollback()
-
-    def commit(self):
-        self.session().commit()
-
-    def delete(self, obj):
-        self.session().delete(obj)
-
-    def getNodes(self, provider_name=None, label_name=None, target_name=None,
-                 state=None):
-        exp = self.session().query(Node).order_by(
-            node_table.c.provider_name,
-            node_table.c.label_name)
-        if provider_name:
-            exp = exp.filter_by(provider_name=provider_name)
-        if label_name:
-            exp = exp.filter_by(label_name=label_name)
-        if target_name:
-            exp = exp.filter_by(target_name=target_name)
-        if state:
-            exp = exp.filter(node_table.c.state == state)
-        return exp.all()
-
-    def createNode(self, *args, **kwargs):
-        new = Node(*args, **kwargs)
-        self.session().add(new)
-        self.commit()
-        return new
-
-    def getNode(self, id):
-        nodes = self.session().query(Node).filter_by(id=id).all()
-        if not nodes:
-            return None
-        return nodes[0]
-
-    def getNodeByHostname(self, hostname):
-        nodes = self.session().query(Node).filter_by(hostname=hostname).all()
-        if not nodes:
-            return None
-        return nodes[0]
-
-    def getNodeByNodename(self, nodename):
-        nodes = self.session().query(Node).filter_by(nodename=nodename).all()
-        if not nodes:
-            return None
-        return nodes[0]
-
-    def getNodeByExternalID(self, provider_name, external_id):
-        nodes = self.session().query(Node).filter_by(
-            provider_name=provider_name,
-            external_id=external_id).all()
-        if not nodes:
-            return None
-        return nodes[0]
-
-    def getJob(self, id):
-        jobs = self.session().query(Job).filter_by(id=id).all()
-        if not jobs:
-            return None
-        return jobs[0]
-
-    def getJobByName(self, name):
-        jobs = self.session().query(Job).filter_by(name=name).all()
-        if not jobs:
-            return None
-        return jobs[0]
-
-    def getJobs(self):
-        return self.session().query(Job).all()
-
-    def createJob(self, *args, **kwargs):
-        new = Job(*args, **kwargs)
-        self.session().add(new)
-        self.commit()
-        return new

nodepool/tests/__init__.py

@@ -18,17 +18,14 @@
 import glob
 import logging
 import os
-import pymysql
 import random
 import string
 import subprocess
 import threading
 import tempfile
 import time
-import uuid
 
 import fixtures
-import lockfile
 import kazoo.client
 import testtools
 
@@ -230,47 +227,6 @@ class RoundRobinTestCase(object):
                                                          self.allocations))
 
 
-class MySQLSchemaFixture(fixtures.Fixture):
-    def setUp(self):
-        super(MySQLSchemaFixture, self).setUp()
-
-        random_bits = ''.join(random.choice(string.ascii_lowercase +
-                                            string.ascii_uppercase)
-                              for x in range(8))
-        self.name = '%s_%s' % (random_bits, os.getpid())
-        self.passwd = uuid.uuid4().hex
-        lock = lockfile.LockFile('/tmp/nodepool-db-schema-lockfile')
-        with lock:
-            db = pymysql.connect(host="localhost",
-                                 user="openstack_citest",
-                                 passwd="openstack_citest",
-                                 db="openstack_citest")
-            cur = db.cursor()
-            cur.execute("create database %s" % self.name)
-            cur.execute(
-                "grant all on %s.* to '%s'@'localhost' identified by '%s'" %
-                (self.name, self.name, self.passwd))
-            cur.execute("flush privileges")
-
-        self.dburi = 'mysql+pymysql://%s:%s@localhost/%s' % (self.name,
-                                                             self.passwd,
-                                                             self.name)
-        self.addDetail('dburi', testtools.content.text_content(self.dburi))
-        self.addCleanup(self.cleanup)
-
-    def cleanup(self):
-        lock = lockfile.LockFile('/tmp/nodepool-db-schema-lockfile')
-        with lock:
-            db = pymysql.connect(host="localhost",
-                                 user="openstack_citest",
-                                 passwd="openstack_citest",
-                                 db="openstack_citest")
-            cur = db.cursor()
-            cur.execute("drop database %s" % self.name)
-            cur.execute("drop user '%s'@'localhost'" % self.name)
-            cur.execute("flush privileges")
-
-
 class BuilderFixture(fixtures.Fixture):
     def __init__(self, configfile, cleanup_interval):
         super(BuilderFixture, self).__init__()
@@ -296,9 +252,6 @@ class DBTestCase(BaseTestCase):
     def setUp(self):
         super(DBTestCase, self).setUp()
         self.log = logging.getLogger("tests")
-        f = MySQLSchemaFixture()
-        self.useFixture(f)
-        self.dburi = f.dburi
         self.secure_conf = self._setup_secure()
         self.setupZK()
 
@@ -333,7 +286,8 @@ class DBTestCase(BaseTestCase):
         (fd, path) = tempfile.mkstemp()
         with open(configfile) as conf_fd:
             config = conf_fd.read()
-            os.write(fd, config.format(dburi=self.dburi))
+            os.write(fd, config)
+            #os.write(fd, config.format(dburi=self.dburi))
         os.close(fd)
         return path
 

nodepool/tests/fixtures/secure.conf

@@ -1,2 +1 @@
-[database]
+# Empty
-dburi={dburi}

requirements.txt

@@ -6,7 +6,6 @@ python-daemon>=2.0.4,<2.1.0
 extras
 statsd>=3.0
 sqlalchemy>=0.8.2,<1.1.0
-PyMySQL
 PrettyTable>=0.6,<0.8
 # shade has a looser requirement on six than nodepool, so install six first
 six>=1.7.0

tools/fake-secure.conf

@@ -1,2 +1 @@
-[database]
+# Empty
-dburi=mysql+pymysql://nodepool@localhost/nodepool

tools/test-setup.sh

@@ -1,33 +0,0 @@
-#!/bin/bash -xe
-
-# This script will be run by OpenStack CI before unit tests are run,
-# it sets up the test system as needed.
-# Developers should setup their test systems in a similar way.
-
-# This setup needs to be run as a user that can run sudo.
-
-# The root password for the MySQL database; pass it in via
-# MYSQL_ROOT_PW.
-DB_ROOT_PW=${MYSQL_ROOT_PW:-insecure_slave}
-
-# This user and its password are used by the tests, if you change it,
-# your tests might fail.
-DB_USER=openstack_citest
-DB_PW=openstack_citest
-
-sudo -H mysqladmin -u root password $DB_ROOT_PW
-
-# It's best practice to remove anonymous users from the database.  If
-# a anonymous user exists, then it matches first for connections and
-# other connections from that host will not work.
-sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e "
-    DELETE FROM mysql.user WHERE User='';
-    FLUSH PRIVILEGES;
-    GRANT ALL PRIVILEGES ON *.*
-        TO '$DB_USER'@'%' identified by '$DB_PW' WITH GRANT OPTION;"
-
-# Now create our database.
-mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e "
-    SET default_storage_engine=MYISAM;
-    DROP DATABASE IF EXISTS openstack_citest;
-    CREATE DATABASE openstack_citest CHARACTER SET utf8;"

tools/zuul-nodepool-integration/secure.conf

@@ -1,2 +1 @@
-[database]
+# Empty
-dburi=mysql+pymysql://nodepool@localhost/nodepool