Cleanup guestagent models

Cleaned up the guestagent database/user __init__()
methods by moving the deserialization and
error-checking code to the base.

* Also had to update CouchDB service to use
the new model's serialization/deserialization
calls (tested with int-tests).

Make use of the models in the Postgres manager

The guestagent should make use of these new classes internally.
Currently it creates serialized versions of those objects 'manually'
using Python dicts.

Change-Id: I8b1ae9207948f211de8fcc2a0d3097bd37e06d78
Related-Bug: 1498573
Closes-Bug: 1543739

trove/common/pagination.py

@@ -24,26 +24,48 @@ def url_quote(s):
     return urllib_parse.quote(str(s))
 
 
-def paginate_list(li, limit=None, marker=None, include_marker=False):
+def paginate_list(li, limit=None, marker=None, include_marker=False,
+                  key=lambda x: x):
     """Sort the given list and return a sublist containing a page of items.
 
     :param list li:             The list to be paginated.
     :param int limit:           Maximum number of iterms to be returned.
     :param marker:              Key of the first item to appear on the sublist.
     :param bool include_marker: Include the marker value itself in the sublist.
+    :param lambda key:          Sorting expression.
     :return:
     """
-    li.sort()
+    sli = sorted(li, key=key)
+    index = [key(item) for item in sli]
     if include_marker:
-        pos = bisect.bisect_left(li, marker)
+        pos = bisect.bisect_left(index, marker)
     else:
-        pos = bisect.bisect(li, marker)
+        pos = bisect.bisect(index, marker)
 
-    if limit and pos + limit < len(li):
+    if limit and pos + limit < len(sli):
-        page = li[pos:pos + limit]
+        page = sli[pos:pos + limit]
-        return page, page[-1]
+        return page, key(page[-1])
     else:
-        return li[pos:], None
+        return sli[pos:], None
+
+
+def paginate_object_list(li, attr_name, limit=None, marker=None,
+                         include_marker=False):
+    """Wrapper for paginate_list to handle lists of generic objects paginated
+    based on an attribute.
+    """
+    return paginate_list(li, limit=limit, marker=marker,
+                         include_marker=include_marker,
+                         key=lambda x: getattr(x, attr_name))
+
+
+def paginate_dict_list(li, key, limit=None, marker=None, include_marker=False):
+    """Wrapper for paginate_list to handle lists of dicts paginated
+    based on a key.
+    """
+    return paginate_list(li, limit=limit, marker=marker,
+                         include_marker=include_marker,
+                         key=lambda x: x[key])
 
 
 class PaginatedDataView(object):

trove/guestagent/datastore/experimental/couchdb/service.py

@@ -199,9 +199,7 @@ class CouchDBAdmin(object):
         if not type(self).admin_user:
             creds = CouchDBCredentials()
             creds.read(system.COUCHDB_ADMIN_CREDS_FILE)
-            user = models.CouchDBUser()
+            user = models.CouchDBUser(creds.username, creds.password)
-            user.name = creds.username
-            user.password = creds.password
             type(self).admin_user = user
         return type(self).admin_user
 
@@ -212,13 +210,15 @@ class CouchDBAdmin(object):
             return False
         return True
 
+    def _is_modifiable_database(self, name):
+        return name not in cfg.get_ignored_dbs()
+
     def create_user(self, users):
         LOG.debug("Creating user(s) for accessing CouchDB database(s).")
         self._admin_user()
         try:
             for item in users:
-                user = models.CouchDBUser()
+                user = models.CouchDBUser.deserialize_user(item)
-                user.deserialize(item)
                 try:
                     LOG.debug("Creating user: %s." % user.name)
                     utils.execute_with_timeout(
@@ -234,8 +234,7 @@ class CouchDBAdmin(object):
                     pass
 
                 for database in user.databases:
-                    mydb = models.CouchDBSchema()
+                    mydb = models.CouchDBSchema.deserialize_schema(database)
-                    mydb.deserialize(database)
                     try:
                         LOG.debug("Granting user: %s access to database: %s."
                                   % (user.name, mydb.name))
@@ -258,8 +257,7 @@ class CouchDBAdmin(object):
 
     def delete_user(self, user):
         LOG.debug("Delete a given CouchDB user.")
-        couchdb_user = models.CouchDBUser()
+        couchdb_user = models.CouchDBUser.deserialize_user(user)
-        couchdb_user.deserialize(user)
         db_names = self.list_database_names()
 
         for db in db_names:
@@ -346,8 +344,7 @@ class CouchDBAdmin(object):
             elif uname[17:]:
                 userlist.append(uname[17:])
         for i in range(len(userlist)):
-            user = models.CouchDBUser()
+            user = models.CouchDBUser(userlist[i])
-            user.name = userlist[i]
             for db in db_names:
                 try:
                     out2, err = utils.execute_with_timeout(
@@ -381,8 +378,7 @@ class CouchDBAdmin(object):
         return user.serialize()
 
     def _get_user(self, username, hostname):
-        user = models.CouchDBUser()
+        user = models.CouchDBUser(username)
-        user.name = username
         db_names = self.list_database_names()
         for db in db_names:
             try:
@@ -413,8 +409,7 @@ class CouchDBAdmin(object):
                 'Cannot grant access for non-existant user: '
                 '%(user)s') % {'user': username})
         else:
-            user = models.CouchDBUser()
+            user = models.CouchDBUser(username)
-            user.name = username
             if not self._is_modifiable_user(user.name):
                 LOG.warning(_('Cannot grant access for reserved user '
                               '%(user)s') % {'user': username})
@@ -462,12 +457,7 @@ class CouchDBAdmin(object):
 
     def enable_root(self, root_pwd=None):
         '''Create admin user root'''
-        if not root_pwd:
+        root_user = models.CouchDBRootUser(password=root_pwd)
-            LOG.debug('Generating root user password.')
-            root_pwd = utils.generate_random_password()
-        root_user = models.CouchDBUser()
-        root_user.name = 'root'
-        root_user.password = root_pwd
         out, err = utils.execute_with_timeout(
             system.ENABLE_ROOT %
             {'admin_name': self._admin_user().name,
@@ -497,19 +487,24 @@ class CouchDBAdmin(object):
 
         for database in databases:
             dbName = models.CouchDBSchema.deserialize_schema(database).name
-            LOG.debug('Creating CouchDB database %s' % dbName)
+            if self._is_modifiable_database(dbName):
-            try:
+                LOG.debug('Creating CouchDB database %s' % dbName)
-                utils.execute_with_timeout(
+                try:
-                    system.CREATE_DB_COMMAND %
+                    utils.execute_with_timeout(
-                    {'admin_name': self._admin_user().name,
+                        system.CREATE_DB_COMMAND %
-                     'admin_password': self._admin_user().password,
+                        {'admin_name': self._admin_user().name,
-                     'dbname': dbName},
+                         'admin_password': self._admin_user().password,
-                    shell=True)
+                         'dbname': dbName},
-            except exception.ProcessExecutionError:
+                        shell=True)
-                LOG.exception(_(
+                except exception.ProcessExecutionError:
-                    "There was an error creating database: %s.") % dbName)
+                    LOG.exception(_(
+                        "There was an error creating database: %s.") % dbName)
+                    db_create_failed.append(dbName)
+                    pass
+            else:
+                LOG.warning(_('Cannot create database with a reserved name '
+                              '%(db)s') % {'db': dbName})
                 db_create_failed.append(dbName)
-                pass
         if len(db_create_failed) > 0:
             LOG.exception(_("Creating the following databases failed: %s.") %
                           db_create_failed)
@@ -540,21 +535,24 @@ class CouchDBAdmin(object):
 
     def delete_database(self, database):
         '''Delete the specified database.'''
-        dbName = None
+        dbName = models.CouchDBSchema.deserialize_schema(database).name
-        try:
+        if self._is_modifiable_database(dbName):
-            dbName = models.CouchDBSchema.deserialize_schema(database).name
+            try:
-            LOG.debug("Deleting CouchDB database: %s." % dbName)
+                LOG.debug("Deleting CouchDB database: %s." % dbName)
-            utils.execute_with_timeout(
+                utils.execute_with_timeout(
-                system.DELETE_DB_COMMAND %
+                    system.DELETE_DB_COMMAND %
-                {'admin_name': self._admin_user().name,
+                    {'admin_name': self._admin_user().name,
-                 'admin_password': self._admin_user().password,
+                     'admin_password': self._admin_user().password,
-                 'dbname': dbName},
+                     'dbname': dbName},
-                shell=True)
+                    shell=True)
-        except exception.ProcessExecutionError:
+            except exception.ProcessExecutionError:
-            LOG.exception(_(
+                LOG.exception(_(
-                "There was an error while deleting database:%s.") % dbName)
+                    "There was an error while deleting database:%s.") % dbName)
-            raise exception.GuestError(_("Unable to delete database: %s.") %
+                raise exception.GuestError(_("Unable to delete database: %s.")
-                                       dbName)
+                                           % dbName)
+        else:
+            LOG.warning(_('Cannot delete a reserved database '
+                          '%(db)s') % {'db': dbName})
 
 
 class CouchDBCredentials(object):

trove/guestagent/datastore/experimental/postgresql/manager.py

@@ -26,10 +26,10 @@ from .service.status import PgSqlAppStatus
 
 from trove.common import cfg
 from trove.common.notification import EndNotification
-from trove.common import utils
 from trove.guestagent import backup
 from trove.guestagent.datastore.experimental.postgresql import pgutil
 from trove.guestagent.datastore import manager
+from trove.guestagent.db import models
 from trove.guestagent import guest_log
 from trove.guestagent import volume
 
@@ -61,25 +61,24 @@ class Manager(
 
     @property
     def datastore_log_defs(self):
-        owner = 'postgres'
         datastore_dir = '/var/log/postgresql/'
         long_query_time = CONF.get(self.manager).get(
             'guest_log_long_query_time')
         general_log_file = self.build_log_file_name(
-            self.GUEST_LOG_DEFS_GENERAL_LABEL, owner,
+            self.GUEST_LOG_DEFS_GENERAL_LABEL, self.PGSQL_OWNER,
             datastore_dir=datastore_dir)
         general_log_dir, general_log_filename = os.path.split(general_log_file)
         return {
             self.GUEST_LOG_DEFS_GENERAL_LABEL: {
                 self.GUEST_LOG_TYPE_LABEL: guest_log.LogType.USER,
-                self.GUEST_LOG_USER_LABEL: owner,
+                self.GUEST_LOG_USER_LABEL: self.PGSQL_OWNER,
                 self.GUEST_LOG_FILE_LABEL: general_log_file,
                 self.GUEST_LOG_ENABLE_LABEL: {
                     'logging_collector': 'on',
-                    'log_destination': self._quote_str('stderr'),
+                    'log_destination': self._quote('stderr'),
-                    'log_directory': self._quote_str(general_log_dir),
+                    'log_directory': self._quote(general_log_dir),
-                    'log_filename': self._quote_str(general_log_filename),
+                    'log_filename': self._quote(general_log_filename),
-                    'log_statement': self._quote_str('all'),
+                    'log_statement': self._quote('all'),
                     'debug_print_plan': 'on',
                     'log_min_duration_statement': long_query_time,
                 },
@@ -90,9 +89,6 @@ class Manager(
             },
         }
 
-    def _quote_str(self, value):
-        return "'%s'" % value
-
     def do_prepare(self, context, packages, databases, memory_mb, users,
                    device_path, mount_point, backup_info, config_contents,
                    root_password, overrides, cluster_config, snapshot):
@@ -118,11 +114,11 @@ class Manager(
     def _secure(self, context):
         # Create a new administrative user for Trove and also
         # disable the built-in superuser.
-        self.create_database(context, [{'_name': self.ADMIN_USER}])
+        os_admin_db = models.PostgreSQLSchema(self.ADMIN_USER)
-        self._create_admin_user(context)
+        self._create_database(context, os_admin_db)
+        self._create_admin_user(context, databases=[os_admin_db])
         pgutil.PG_ADMIN = self.ADMIN_USER
-        postgres = {'_name': self.PG_BUILTIN_ADMIN,
+        postgres = models.PostgreSQLRootUser()
-                    '_password': utils.generate_random_password()}
         self.alter_user(context, postgres, 'NOSUPERUSER', 'NOLOGIN')
 
     def create_backup(self, context, backup_info):

trove/guestagent/datastore/experimental/postgresql/pgutil.py

@@ -207,7 +207,10 @@ class UserQuery(object):
 
     @classmethod
     def update_name(cls, old, new):
-        """Query to update the name of a user."""
+        """Query to update the name of a user.
+        This statement also results in an automatic permission transfer to the
+        new username.
+        """
 
         return "ALTER USER \"{old}\" RENAME TO \"{new}\"".format(
             old=old,
@@ -231,7 +234,8 @@ class AccessQuery(object):
             "SELECT datname, pg_encoding_to_char(encoding), datcollate "
             "FROM pg_database "
             "WHERE datistemplate = false "
-            "AND 'user {user}=CTc' = ANY (datacl)".format(user=user)
+            "AND 'user \"{user}\"=CTc/{admin}' = ANY (datacl)".format(
+                user=user, admin=PG_ADMIN)
         )
 
     @classmethod

trove/guestagent/datastore/experimental/postgresql/service/access.py

@@ -18,6 +18,7 @@ from oslo_log import log as logging
 from trove.common import cfg
 from trove.common.i18n import _
 from trove.guestagent.datastore.experimental.postgresql import pgutil
+from trove.guestagent.db import models
 
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
@@ -73,21 +74,20 @@ class PgSqlAccess(object):
 
     def list_access(self, context, username, hostname):
         """List database for which the given user as access.
-
+        Return a list of serialized Postgres databases.
-        The username and hostname parameters are strings.
-
-        Return value is a list of dictionaries in the following form:
-
-            [{"_name": "", "_collate": None, "_character_set": None}, ...]
         """
+
+        if self.user_exists(username):
+            return [db.serialize() for db in self._get_databases_for(username)]
+
+        raise exception.UserNotFound(username)
+
+    def _get_databases_for(self, username):
+        """Return all Postgres databases accessible by a given user."""
         results = pgutil.query(
             pgutil.AccessQuery.list(user=username),
             timeout=30,
         )
-
+        return [models.PostgreSQLSchema(
-        # Convert to dictionaries.
+            row[0].strip(), character_set=row[1], collate=row[2])
-        results = (
+            for row in results]
-            {'_name': r[0].strip(), '_collate': None, '_character_set': None}
-            for r in results
-        )
-        return tuple(results)

trove/guestagent/datastore/experimental/postgresql/service/database.py

@@ -13,14 +13,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import itertools
-
 from oslo_log import log as logging
 
 from trove.common import cfg
 from trove.common.i18n import _
 from trove.common.notification import EndNotification
+from trove.common import pagination
 from trove.guestagent.datastore.experimental.postgresql import pgutil
+from trove.guestagent.db import models
 
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
@@ -34,51 +34,58 @@ class PgSqlDatabase(object):
     def create_database(self, context, databases):
         """Create the list of specified databases.
 
-        The databases parameter is a list of dictionaries in the following
+        The databases parameter is a list of serialized Postgres databases.
-        form:
-
-            {"_name": "", "_character_set": "", "_collate": ""}
-
-        Encoding and collation values are validated in
-        trove.guestagent.db.models.
         """
         with EndNotification(context):
             for database in databases:
-                encoding = database.get('_character_set')
+                self._create_database(
-                collate = database.get('_collate')
+                    context,
-                LOG.info(
+                    models.PostgreSQLSchema.deserialize_schema(database))
-                    _("{guest_id}: Creating database {name}.").format(
+
-                        guest_id=CONF.guest_id,
+    def _create_database(self, context, database):
-                        name=database['_name'],
+        """Create a database.
-                    )
+
-                )
+        :param database:          Database to be created.
-                pgutil.psql(
+        :type database:           PostgreSQLSchema
-                    pgutil.DatabaseQuery.create(
+        """
-                        name=database['_name'],
+        LOG.info(
-                        encoding=encoding,
+            _("{guest_id}: Creating database {name}.").format(
-                        collation=collate,
+                guest_id=CONF.guest_id,
-                    ),
+                name=database.name,
-                    timeout=30,
+            )
-                )
+        )
+        pgutil.psql(
+            pgutil.DatabaseQuery.create(
+                name=database.name,
+                encoding=database.character_set,
+                collation=database.collate,
+            ),
+            timeout=30,
+        )
 
     def delete_database(self, context, database):
         """Delete the specified database.
-
-        The database parameter is a dictionary in the following form:
-
-            {"_name": ""}
         """
         with EndNotification(context):
-            LOG.info(
+            self._drop_database(
-                _("{guest_id}: Dropping database {name}.").format(
+                models.PostgreSQLSchema.deserialize_schema(database))
-                    guest_id=CONF.guest_id,
+
-                    name=database['_name'],
+    def _drop_database(self, database):
-                )
+        """Drop a given Postgres database.
-            )
+
-            pgutil.psql(
+        :param database:          Database to be dropped.
-                pgutil.DatabaseQuery.drop(name=database['_name']),
+        :type database:           PostgreSQLSchema
-                timeout=30,
+        """
+        LOG.info(
+            _("{guest_id}: Dropping database {name}.").format(
+                guest_id=CONF.guest_id,
+                name=database.name,
             )
+        )
+        pgutil.psql(
+            pgutil.DatabaseQuery.drop(name=database.name),
+            timeout=30,
+        )
 
     def list_databases(
             self,
@@ -87,42 +94,19 @@ class PgSqlDatabase(object):
             marker=None,
             include_marker=False,
     ):
-        """List databases created on this instance.
+        """List all databases on the instance.
-
+        Return a paginated list of serialized Postgres databases.
-        Return value is a list of dictionaries in the following form:
-
-            [{"_name": "", "_character_set": "", "_collate": ""}, ...]
         """
+        page, next_name = pagination.paginate_object_list(
+            self._get_databases(), 'name', limit, marker, include_marker)
+        return [db.serialize() for db in page], next_name
+
+    def _get_databases(self):
+        """Return all non-system Postgres databases on the instance."""
         results = pgutil.query(
             pgutil.DatabaseQuery.list(ignore=cfg.get_ignored_dbs()),
             timeout=30,
         )
-        # Convert results to dictionaries.
+        return [models.PostgreSQLSchema(
-        results = (
+            row[0].strip(), character_set=row[1], collate=row[2])
-            {'_name': r[0].strip(), '_character_set': r[1], '_collate': r[2]}
+            for row in results]
-            for r in results
-        )
-        # Force __iter__ of generator until marker found.
-        if marker is not None:
-            try:
-                item = next(results)
-                while item['_name'] != marker:
-                    item = next(results)
-            except StopIteration:
-                pass
-
-        remainder = None
-        if limit is not None:
-            remainder = results
-            results = itertools.islice(results, limit)
-
-        results = tuple(results)
-
-        next_marker = None
-        if remainder is not None:
-            try:
-                next_marker = next(remainder)
-            except StopIteration:
-                pass
-
-        return results, next_marker

trove/guestagent/datastore/experimental/postgresql/service/root.py

@@ -14,10 +14,10 @@
 #    under the License.
 
 from trove.common import cfg
-from trove.common import utils
 from trove.guestagent.datastore.experimental.postgresql import pgutil
 from trove.guestagent.datastore.experimental.postgresql.service.users import (
     PgSqlUsers)
+from trove.guestagent.db import models
 
 CONF = cfg.CONF
 
@@ -72,18 +72,15 @@ class PgSqlRoot(PgSqlUsers):
 
             {"_name": "postgres", "_password": "<secret>"}
         """
-        user = {
+        user = models.PostgreSQLRootUser(password=root_password)
-            "_name": "postgres",
-            "_password": root_password or utils.generate_random_password(),
-        }
         query = pgutil.UserQuery.alter_user(
-            user['_name'],
+            user.name,
-            user['_password'],
+            user.password,
             None,
             *self.ADMIN_OPTIONS
         )
         pgutil.psql(query, timeout=30)
-        return user
+        return user.serialize()
 
     def disable_root(self, context):
         """Generate a new random password for the public superuser account.
@@ -93,4 +90,4 @@ class PgSqlRoot(PgSqlUsers):
         self.enable_root(context)
 
     def enable_root_with_password(self, context, root_password=None):
-        self.enable_root(context, root_password)
+        return self.enable_root(context, root_password)

trove/guestagent/datastore/experimental/postgresql/service/users.py

@@ -13,17 +13,19 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import itertools
-
 from oslo_log import log as logging
 
 from trove.common import cfg
+from trove.common import exception
 from trove.common.i18n import _
 from trove.common.notification import EndNotification
+from trove.common import pagination
 from trove.common import utils
 from trove.guestagent.datastore.experimental.postgresql import pgutil
 from trove.guestagent.datastore.experimental.postgresql.service.access import (
     PgSqlAccess)
+from trove.guestagent.db import models
+from trove.guestagent.db.models import PostgreSQLSchema
 
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
@@ -51,32 +53,46 @@ class PgSqlUsers(PgSqlAccess):
             'REPLICATION',
             'LOGIN']
 
-    def _create_admin_user(self, context):
+    def _create_admin_user(self, context, databases=None):
         """Create an administrative user for Trove.
         Force password encryption.
         """
         password = utils.generate_random_password()
-        os_admin = {'_name': self.ADMIN_USER, '_password': password,
+        os_admin = models.PostgreSQLUser(self.ADMIN_USER, password)
-                    '_databases': [{'_name': self.ADMIN_USER}]}
+        if databases:
+            os_admin.databases.extend([db.serialize() for db in databases])
         self._create_user(context, os_admin, True, *self.ADMIN_OPTIONS)
 
     def create_user(self, context, users):
         """Create users and grant privileges for the specified databases.
 
-        The users parameter is a list of dictionaries in the following form:
+        The users parameter is a list of serialized Postgres users.
-
-            {"_name": "", "_password": "", "_databases": [{"_name": ""}, ...]}
         """
         with EndNotification(context):
             for user in users:
-                self._create_user(context, user, None)
+                self._create_user(
+                    context,
+                    models.PostgreSQLUser.deserialize_user(user), None)
 
     def _create_user(self, context, user, encrypt_password=None, *options):
+        """Create a user and grant privileges for the specified databases.
+
+        :param user:              User to be created.
+        :type user:               PostgreSQLUser
+
+        :param encrypt_password:  Store passwords encrypted if True.
+                                  Fallback to configured default
+                                  behavior if None.
+        :type encrypt_password:   boolean
+
+        :param options:           Other user options.
+        :type options:            list
+        """
         LOG.info(
             _("{guest_id}: Creating user {user} {with_clause}.")
             .format(
                 guest_id=CONF.guest_id,
-                user=user['_name'],
+                user=user.name,
                 with_clause=pgutil.UserQuery._build_with_clause(
                     '<SANITIZED>',
                     encrypt_password,
@@ -86,18 +102,23 @@ class PgSqlUsers(PgSqlAccess):
         )
         pgutil.psql(
             pgutil.UserQuery.create(
-                user['_name'],
+                user.name,
-                user['_password'],
+                user.password,
                 encrypt_password,
                 *options
             ),
             timeout=30,
         )
+        self._grant_access(
+            context, user.name,
+            [PostgreSQLSchema.deserialize_schema(db) for db in user.databases])
+
+    def _grant_access(self, context, username, databases):
         self.grant_access(
             context,
-            user['_name'],
+            username,
             None,
-            [d['_name'] for d in user['_databases']],
+            [db.name for db in databases],
         )
 
     def list_users(
@@ -108,121 +129,111 @@ class PgSqlUsers(PgSqlAccess):
             include_marker=False,
     ):
         """List all users on the instance along with their access permissions.
-
+        Return a paginated list of serialized Postgres users.
-        Return value is a list of dictionaries in the following form:
-
-            [{"_name": "", "_password": None, "_host": None,
-              "_databases": [{"_name": ""}, ...]}, ...]
         """
+        page, next_name = pagination.paginate_object_list(
+            self._get_users(context), 'name', limit, marker, include_marker)
+        return [db.serialize() for db in page], next_name
+
+    def _get_users(self, context):
+        """Return all non-system Postgres users on the instance."""
         results = pgutil.query(
             pgutil.UserQuery.list(ignore=cfg.get_ignored_users()),
             timeout=30,
         )
-        # Convert results into dictionaries.
+        return [self._build_user(context, row[0].strip()) for row in results]
-        results = (
-            {
-                '_name': r[0].strip(),
-                '_password': None,
-                '_host': None,
-                '_databases': self.list_access(context, r[0], None),
-            }
-            for r in results
-        )
 
-        # Force __iter__ of generator until marker found.
+    def _build_user(self, context, username):
-        if marker is not None:
+        """Build a model representation of a Postgres user.
-            try:
+        Include all databases it has access to.
-                item = next(results)
+        """
-                while item['_name'] != marker:
+        user = models.PostgreSQLUser(username)
-                    item = next(results)
+        dbs = self.list_access(context, username, None)
-            except StopIteration:
+        for d in dbs:
-                pass
+            user.databases.append(d)
-
+        return user
-        remainder = None
-        if limit is not None:
-            remainder = results
-            results = itertools.islice(results, limit)
-
-        results = tuple(results)
-
-        next_marker = None
-        if remainder is not None:
-            try:
-                next_marker = next(remainder)
-            except StopIteration:
-                pass
-
-        return results, next_marker
 
     def delete_user(self, context, user):
         """Delete the specified user.
-
-        The user parameter is a dictionary in the following form:
-
-            {"_name": ""}
         """
         with EndNotification(context):
-            LOG.info(
+            self._drop_user(models.PostgreSQLUser.deserialize_user(user))
-                _("{guest_id}: Dropping user {name}.").format(
+
-                    guest_id=CONF.guest_id,
+    def _drop_user(self, user):
-                    name=user['_name'],
+        """Drop a given Postgres user.
-                )
+
-            )
+        :param user:              User to be dropped.
-            pgutil.psql(
+        :type user:               PostgreSQLUser
-                pgutil.UserQuery.drop(name=user['_name']),
+        """
-                timeout=30,
+        LOG.info(
+            _("{guest_id}: Dropping user {name}.").format(
+                guest_id=CONF.guest_id,
+                name=user.name,
             )
+        )
+        pgutil.psql(
+            pgutil.UserQuery.drop(name=user.name),
+            timeout=30,
+        )
 
     def get_user(self, context, username, hostname):
-        """Return a single user matching the criteria.
+        """Return a serialized representation of a user with a given name.
+        """
+        user = self._find_user(context, username)
+        return user.serialize() if user is not None else None
 
-        The username and hostname parameter are strings.
+    def _find_user(self, context, username):
-
+        """Lookup a user with a given username.
-        The return value is a dictionary in the following form:
+        Return a new Postgres user instance or raise if no match is found.
-
-            {"_name": "", "_host": None, "_password": None,
-             "_databases": [{"_name": ""}, ...]}
-
-        Where "_databases" is a list of databases the user has access to.
         """
         results = pgutil.query(
             pgutil.UserQuery.get(name=username),
             timeout=30,
         )
-        results = tuple(results)
-        if len(results) < 1:
-            return None
 
-        return {
+        if results:
-            "_name": results[0][0],
+            return self._build_user(context, username)
-            "_host": None,
+
-            "_password": None,
+        return None
-            "_databases": self.list_access(context, username, None),
+
-        }
+    def user_exists(self, username):
+        """Return whether a given user exists on the instance."""
+        results = pgutil.query(
+            pgutil.UserQuery.get(name=username),
+            timeout=30,
+        )
+
+        return bool(results)
 
     def change_passwords(self, context, users):
         """Change the passwords of one or more existing users.
-
+        The users parameter is a list of serialized Postgres users.
-        The users parameter is a list of dictionaries in the following form:
-
-            {"name": "", "password": ""}
         """
         with EndNotification(context):
             for user in users:
-                self.alter_user(context, user, None)
+                self.alter_user(
+                    context,
+                    models.PostgreSQLUser.deserialize_user(user), None)
 
     def alter_user(self, context, user, encrypt_password=None, *options):
         """Change the password and options of an existing users.
 
-        The user parameter is a dictionary of the following form:
+        :param user:              User to be altered.
+        :type user:               PostgreSQLUser
 
-            {"name": "", "password": ""}
+        :param encrypt_password:  Store passwords encrypted if True.
+                                  Fallback to configured default
+                                  behavior if None.
+        :type encrypt_password:   boolean
+
+        :param options:           Other user options.
+        :type options:            list
         """
         LOG.info(
             _("{guest_id}: Altering user {user} {with_clause}.")
             .format(
                 guest_id=CONF.guest_id,
-                user=user['_name'],
+                user=user.name,
                 with_clause=pgutil.UserQuery._build_with_clause(
                     '<SANITIZED>',
                     encrypt_password,
@@ -232,8 +243,8 @@ class PgSqlUsers(PgSqlAccess):
         )
         pgutil.psql(
             pgutil.UserQuery.alter_user(
-                user['_name'],
+                user.name,
-                user['_password'],
+                user.password,
                 encrypt_password,
                 *options),
             timeout=30,
@@ -250,47 +261,42 @@ class PgSqlUsers(PgSqlAccess):
         Each key/value pair in user_attrs is optional.
         """
         with EndNotification(context):
-            if user_attrs.get('password') is not None:
+            user = self._build_user(context, username)
-                self.change_passwords(
+            new_username = user_attrs.get('name')
-                    context,
+            new_password = user_attrs.get('password')
-                    (
-                        {
-                            "name": username,
-                            "password": user_attrs['password'],
-                        },
-                    ),
-                )
 
-            if user_attrs.get('name') is not None:
+            if new_username is not None:
-                access = self.list_access(context, username, None)
+                self._rename_user(context, user, new_username)
-                LOG.info(
+                # Make sure we can retrieve the renamed user.
-                    _("{guest_id}: Changing username for {old} to {new}."
+                user = self._find_user(context, new_username)
-                      ).format(
+                if user is None:
-                          guest_id=CONF.guest_id,
+                    raise exception.TroveError(_(
-                          old=username,
+                        "Renamed user %s could not be found on the instance.")
-                          new=user_attrs['name'],
+                        % new_username)
-                    )
+
-                )
+            if new_password is not None:
-                pgutil.psql(
+                user.password = new_password
-                    pgutil.psql.UserQuery.update_name(
+                self.alter_user(context, user)
-                        old=username,
+
-                        new=user_attrs['name'],
+    def _rename_user(self, context, user, new_username):
-                    ),
+        """Rename a given Postgres user and transfer all access to the
-                    timeout=30,
+        new name.
-                )
+
-                # Regrant all previous access after the name change.
+        :param user:              User to be renamed.
-                LOG.info(
+        :type user:               PostgreSQLUser
-                    _("{guest_id}: Regranting permissions from {old} "
+        """
-                      "to {new}.")
+        LOG.info(
-                    .format(
+            _("{guest_id}: Changing username for {old} to {new}.").format(
-                        guest_id=CONF.guest_id,
+                guest_id=CONF.guest_id,
-                        old=username,
+                old=user.name,
-                        new=user_attrs['name'],
+                new=new_username,
-                    )
+            )
-                )
+        )
-                self.grant_access(
+        # PostgreSQL handles the permission transfer itself.
-                    context,
+        pgutil.psql(
-                    username=user_attrs['name'],
+            pgutil.UserQuery.update_name(
-                    hostname=None,
+                old=user.name,
-                    databases=(db['_name'] for db in access)
+                new=new_username,
-                )
+            ),
+            timeout=30,
+        )

trove/guestagent/db/models.py

@@ -53,11 +53,23 @@ class Base(object):
 class DatastoreSchema(Base):
     """Represents a database schema."""
 
-    def __init__(self):
+    def __init__(self, name, deserializing=False, *args, **kwargs):
         self._name = None
         self._collate = None
         self._character_set = None
 
+        # If both or neither are passed in this is a bug.
+        if not (bool(deserializing) != bool(name)):
+            raise RuntimeError("Bug in DatastoreSchema()")
+        if not deserializing:
+            self.name = name
+
+    def __str__(self):
+        return str(self.name)
+
+    def __repr__(self):
+        return str(self.serialize())
+
     @classmethod
     def deserialize_schema(cls, value):
         if not cls._validate_dict(value):
@@ -65,7 +77,7 @@ class DatastoreSchema(Base):
                                "Required: %(reqs)s")
                              % ({'keys': value.keys(),
                                  'reqs': cls._dict_requirements()}))
-        schema = cls(deserializing=True)
+        schema = cls(name=None, deserializing=True)
         schema.deserialize(value)
         return schema
 
@@ -133,16 +145,8 @@ class MongoDBSchema(DatastoreSchema):
 
     name_regex = re.compile(r'^[a-zA-Z0-9_\-]+$')
 
-    def __init__(self, name=None, deserializing=False):
+    def __init__(self, name, *args, **kwargs):
-        super(MongoDBSchema, self).__init__()
+        super(MongoDBSchema, self).__init__(name, *args, **kwargs)
-        # need one or the other, not both, not none (!= ~ XOR)
-        if not (bool(deserializing) != bool(name)):
-            raise ValueError(_("Bad args. name: %(name)s, "
-                               "deserializing %(deser)s.")
-                             % ({'name': bool(name),
-                                 'deser': bool(deserializing)}))
-        if not deserializing:
-            self.name = name
 
     @property
     def _max_schema_name_length(self):
@@ -165,16 +169,8 @@ class CassandraSchema(DatastoreSchema):
     the first of which is an alpha character.
     """
 
-    def __init__(self, name=None, deserializing=False):
+    def __init__(self, name, *args, **kwargs):
-        super(CassandraSchema, self).__init__()
+        super(CassandraSchema, self).__init__(name, *args, **kwargs)
-
-        if not (bool(deserializing) != bool(name)):
-            raise ValueError(_("Bad args. name: %(name)s, "
-                               "deserializing %(deser)s.")
-                             % ({'name': bool(name),
-                                 'deser': bool(deserializing)}))
-        if not deserializing:
-            self.name = name
 
     @property
     def _max_schema_name_length(self):
@@ -201,17 +197,8 @@ class CouchDBSchema(DatastoreSchema):
 
     name_regex = re.compile(r'^[a-z][a-z0-9_$()+/-]*$')
 
-    def __init__(self, name=None, deserializing=False):
+    def __init__(self, name, *args, **kwargs):
-        super(CouchDBSchema, self).__init__()
+        super(CouchDBSchema, self).__init__(name, *args, **kwargs)
-        self._ignore_dbs = cfg.get_ignored_dbs()
-        # need one or the other, not both, not none (!= ~ XOR)
-        if not (bool(deserializing) != bool(name)):
-            raise ValueError(_("Bad args. name: %(name)s, "
-                               "deserializing %(deser)s.")
-                             % ({'name': bool(name),
-                                 'deser': bool(deserializing)}))
-        if not deserializing:
-            self.name = name
 
     @property
     def _max_schema_name_length(self):
@@ -219,8 +206,6 @@ class CouchDBSchema(DatastoreSchema):
 
     def _is_valid_schema_name(self, value):
         # https://wiki.apache.org/couchdb/HTTP_database_API
-        if value.lower() in self._ignore_dbs:
-            return False
         if re.match(r'^[a-z]*$', value[0]):
             return True
         else:
@@ -240,15 +225,20 @@ class PostgreSQLSchema(DatastoreSchema):
     """
     name_regex = re.compile(u(r'^[\u0001-\u007F\u0080-\uFFFF]+[^\s]$'))
 
-    def __init__(self, name=None, deserializing=False):
+    def __init__(self, name, character_set=None, collate=None,
-        super(PostgreSQLSchema, self).__init__()
+                 *args, **kwargs):
-        if not (bool(deserializing) != bool(name)):
+        super(PostgreSQLSchema, self).__init__(name, *args, **kwargs)
-            raise ValueError(_("Bad args. name: %(name)s, "
+
-                               "deserializing %(deser)s.")
+        self.character_set = character_set
-                             % ({'name': bool(name),
+        self.collate = collate
-                                 'deser': bool(deserializing)}))
+
-        if not deserializing:
+    @DatastoreSchema.collate.setter
-            self.name = name
+    def collate(self, value):
+        self._collate = value
+
+    @DatastoreSchema.character_set.setter
+    def character_set(self, value):
+        self._character_set = value
 
     @property
     def _max_schema_name_length(self):
@@ -586,12 +576,28 @@ class DatastoreUser(Base):
 
     _HOSTNAME_WILDCARD = '%'
 
-    def __init__(self):
+    def __init__(self, name, password, deserializing=False, *args, **kwargs):
         self._name = None
         self._password = None
         self._host = None
         self._databases = []
 
+        # need only one of: deserializing, name, or (name and password)
+        if ((not (bool(deserializing) != bool(name))) or
+                (bool(deserializing) and bool(password))):
+            raise RuntimeError("Bug in DatastoreUser()")
+        if not deserializing:
+            if name:
+                self.name = name
+            if password is not None:
+                self.password = password
+
+    def __str__(self):
+        return str(self.name)
+
+    def __repr__(self):
+        return str(self.serialize())
+
     @classmethod
     def deserialize_user(cls, value):
         if not cls._validate_dict(value):
@@ -599,7 +605,7 @@ class DatastoreUser(Base):
                                "Required: %(reqs)s")
                              % ({'keys': value.keys(),
                                  'reqs': cls._dict_requirements()}))
-        user = cls(deserializing=True)
+        user = cls(name=None, password=None, deserializing=True)
         user.deserialize(value)
         return user
 
@@ -713,24 +719,11 @@ class MongoDBUser(DatastoreUser):
     Trove stores this as <database>.<username>
     """
 
-    def __init__(self, name=None, password=None, deserializing=False):
+    def __init__(self, name=None, password=None, *args, **kwargs):
-        super(MongoDBUser, self).__init__()
-        self._name = None
         self._username = None
         self._database = None
         self._roles = []
-        # need only one of: deserializing, name, or (name and password)
+        super(MongoDBUser, self).__init__(name, password, *args, **kwargs)
-        if ((not (bool(deserializing) != bool(name))) or
-                (bool(deserializing) and bool(password))):
-            raise ValueError(_("Bad args. name: %(name)s, "
-                               "password %(pass)s, "
-                               "deserializing %(deser)s.")
-                             % ({'name': bool(name),
-                                 'pass': bool(password),
-                                 'deser': bool(deserializing)}))
-        if not deserializing:
-            self.name = name
-            self.password = password
 
     @property
     def username(self):
@@ -855,20 +848,8 @@ class MongoDBUser(DatastoreUser):
 class CassandraUser(DatastoreUser):
     """Represents a Cassandra user and its associated properties."""
 
-    def __init__(self, name=None, password=None, deserializing=False):
+    def __init__(self, name, password=None, *args, **kwargs):
-        super(CassandraUser, self).__init__()
+        super(CassandraUser, self).__init__(name, password, *args, **kwargs)
-
-        if ((not (bool(deserializing) != bool(name))) or
-                (bool(deserializing) and bool(password))):
-            raise ValueError(_("Bad args. name: %(name)s, "
-                               "password %(pass)s, "
-                               "deserializing %(deser)s.")
-                             % ({'name': bool(name),
-                                 'pass': bool(password),
-                                 'deser': bool(deserializing)}))
-        if not deserializing:
-            self.name = name
-            self.password = password
 
     def _build_database_schema(self, name):
         return CassandraSchema(name)
@@ -894,60 +875,28 @@ class CassandraUser(DatastoreUser):
 class CouchDBUser(DatastoreUser):
     """Represents a CouchDB user and its associated properties."""
 
-    def __init__(self):
+    def __init__(self, name, password=None, *args, **kwargs):
-        self._name = None
+        super(CouchDBUser, self).__init__(name, password, *args, **kwargs)
-        self._host = None
-        self._password = None
-        self._databases = []
-        self._ignore_users = cfg.get_ignored_users()
 
-    def _is_valid(self, value):
+    def _build_database_schema(self, name):
+        return CouchDBSchema(name)
+
+    @property
+    def _max_username_length(self):
+        return None
+
+    def _is_valid_name(self, value):
         return True
 
-    @property
+    def _is_valid_host_name(self, value):
-    def name(self):
+        return True
-        return self._name
 
-    @name.setter
+    def _is_valid_password(self, value):
-    def name(self, value):
+        return True
-        if not self._is_valid(value):
-            raise ValueError(_("'%s' is not a valid user name.") % value)
-        else:
-            self._name = value
 
-    @property
+    @classmethod
-    def password(self):
+    def _dict_requirements(cls):
-        return self._password
+        return ['_name']
-
-    @password.setter
-    def password(self, value):
-        if not self._is_valid(value):
-            raise ValueError(_("'%s' is not a valid password.") % value)
-        else:
-            self._password = value
-
-    @property
-    def databases(self):
-        return self._databases
-
-    @databases.setter
-    def databases(self, value):
-        mydb = ValidatedMySQLDatabase()
-        mydb.name = value
-        self._databases.append(mydb.serialize())
-
-    @property
-    def host(self):
-        if self._host is None:
-            return '%'
-        return self._host
-
-    @host.setter
-    def host(self, value):
-        if not self._is_valid_host_name(value):
-            raise ValueError(_("'%s' is not a valid hostname.") % value)
-        else:
-            self._host = value
 
 
 class MySQLUser(Base):
@@ -1046,21 +995,8 @@ class MySQLUser(Base):
 class PostgreSQLUser(DatastoreUser):
     """Represents a PostgreSQL user and its associated properties."""
 
-    def __init__(self, name=None, password=None, deserializing=False):
+    def __init__(self, name, password=None, *args, **kwargs):
-        super(PostgreSQLUser, self).__init__()
+        super(PostgreSQLUser, self).__init__(name, password, *args, **kwargs)
-
-        if ((not (bool(deserializing) != bool(name))) or
-                (bool(deserializing) and bool(password))):
-            raise ValueError(_("Bad args. name: %(name)s, "
-                               "password %(pass)s, "
-                               "deserializing %(deser)s.")
-                             % ({'name': bool(name),
-                                 'pass': bool(password),
-                                 'deser': bool(deserializing)}))
-
-        if not deserializing:
-            self.name = name
-            self.password = password
 
     def _build_database_schema(self, name):
         return PostgreSQLSchema(name)
@@ -1080,7 +1016,7 @@ class PostgreSQLUser(DatastoreUser):
 
     @classmethod
     def _dict_requirements(cls):
-        return ['name']
+        return ['_name']
 
 
 class RootUser(MySQLUser):
@@ -1116,6 +1052,17 @@ class CassandraRootUser(CassandraUser):
 class PostgreSQLRootUser(PostgreSQLUser):
     """Represents the PostgreSQL default superuser."""
 
-    def __init__(self, password=None):
+    def __init__(self, password=None, *args, **kwargs):
         password = password if not None else utils.generate_random_password()
-        super(PostgreSQLRootUser, self).__init__("postgres", password=password)
+        super(PostgreSQLRootUser, self).__init__("postgres", password=password,
+                                                 *args, **kwargs)
+
+
+class CouchDBRootUser(CouchDBUser):
+    """Represents the CouchDB default superuser."""
+
+    def __init__(self, password=None, *args, **kwargs):
+        if password is None:
+            password = utils.generate_random_password()
+        super(CouchDBRootUser, self).__init__("root", password=password,
+                                              *args, **kwargs)

trove/tests/unittests/common/test_pagination.py

@@ -15,6 +15,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
+
+from mock import Mock
+
 from trove.common import pagination
 from trove.tests.unittests import trove_testtools
 
@@ -41,7 +44,8 @@ class TestPaginatedDataView(trove_testtools.TestCase):
 
     def _do_paginate_list(self, limit=None, marker=None, include_marker=False):
         li = ['a', 'b', 'c', 'd', 'e']
-        return pagination.paginate_list(li, limit, marker, include_marker)
+        return pagination.paginate_list(li, limit=limit, marker=marker,
+                                        include_marker=include_marker)
 
     def test_paginate_list(self):
         # start list
@@ -75,3 +79,48 @@ class TestPaginatedDataView(trove_testtools.TestCase):
         li_4, marker_4 = self._do_paginate_list(marker='f')
         self.assertEqual([], li_4)
         self.assertIsNone(marker_4)
+
+        li_5, marker_5 = self._do_paginate_list(limit=1, marker='f')
+        self.assertEqual([], li_5)
+        self.assertIsNone(marker_5)
+
+    def test_dict_paginate(self):
+        li = [{'_collate': 'en_US.UTF-8',
+               '_character_set': 'UTF8',
+               '_name': 'db1'},
+              {'_collate': 'en_US.UTF-8',
+               '_character_set': 'UTF8',
+               '_name': 'db3'},
+              {'_collate': 'en_US.UTF-8',
+               '_character_set': 'UTF8',
+               '_name': 'db2'},
+              {'_collate': 'en_US.UTF-8',
+               '_character_set': 'UTF8',
+               '_name': 'db5'},
+              {'_collate': 'en_US.UTF-8',
+               '_character_set': 'UTF8',
+               '_name': 'db4'}
+              ]
+
+        l, m = pagination.paginate_dict_list(li, '_name', limit=1,
+                                             marker='db1',
+                                             include_marker=True)
+        self.assertEqual(l[0], li[0])
+        self.assertEqual(m, 'db1')
+
+    def test_object_paginate(self):
+
+        def build_mock_object(name):
+            o = Mock()
+            o.name = name
+            o.attr = 'attr'
+            return o
+
+        li = [build_mock_object('db1'), build_mock_object('db2'),
+              build_mock_object('db3')]
+
+        l, m = pagination.paginate_object_list(li, 'name', limit=1,
+                                               marker='db1',
+                                               include_marker=True)
+        self.assertEqual(l[0], li[0])
+        self.assertEqual(m, 'db1')