Load projects from yaml file

load_projects command added.
This command should be initialize project_groups and projects.

usage:
storyboard-db-manage --config-file <config> load_projects <projects_file>

Example file added etc/projects.yaml.sample
Tests added.

Change-Id: Ib5cb32064aa0629c45c3249e08a0ca2281acd8f0

etc/projects.yaml.sample

@@ -0,0 +1,6 @@
+- project: Test-Project
+  description: First project
+  group: Group-1
+- project: Test-Project-Two
+  description: Second project
+  group: Group-1

requirements.txt

@@ -5,6 +5,8 @@ Babel>=0.9.6
 iso8601>=0.1.8
 oslo.config>=1.2.0
 pecan>=0.2.0
+python-openid
+PyYAML>=3.1.0
 six>=1.4.1
 SQLAlchemy>=0.8,<=0.8.99
 WSME>=0.5b6

storyboard/db/migration/README

@@ -42,6 +42,11 @@ Downgrade the database by a certain number of revisions:
 $ storyboard-db-manage --config-file /path/to/storyboard.conf \
     downgrade --delta <# of revs>
 
+This utility should be also used to load Project Groups and Projects form a .yaml
+file description. The description sample is provided in etc/projects.yaml.
+$ storyboard-db-manage --config-file /path/to/storyboard.conf \
+    load_projects /path/to/projects.yaml
+
 
 DEVELOPERS:
 A database migration script is required when you submit a change to storyboard

storyboard/db/migration/cli.py

@@ -23,16 +23,12 @@ from alembic import config as alembic_config
 from alembic import util as alembic_util
 from oslo.config import cfg
 
+import storyboard.db.projects_loader as loader
+
+
 gettext.install('storyboard', unicode=1)
 
-_db_opts = [
-    cfg.StrOpt('connection',
-               default='',
-               help=_('URL to database')),
-]
-
-CONF = cfg.ConfigOpts()
-CONF.register_opts(_db_opts, 'database')
+CONF = cfg.CONF
 
 
 def do_alembic_command(config, cmd, *args, **kwargs):
@@ -74,6 +70,10 @@ def do_revision(config, cmd):
                        sql=CONF.command.sql)
 
 
+def do_load_projects(config, cmd):
+    loader.do_load_models(CONF.command.file)
+
+
 def add_command_parsers(subparsers):
     for name in ['current', 'history', 'branches']:
         parser = subparsers.add_parser(name)
@@ -100,6 +100,10 @@ def add_command_parsers(subparsers):
     parser.add_argument('--sql', action='store_true')
     parser.set_defaults(func=do_revision)
 
+    parser = subparsers.add_parser('load_projects')
+    parser.add_argument('file', type=str)
+    parser.set_defaults(func=do_load_projects)
+
 
 command_opt = cfg.SubCommandOpt('command',
                                 title='Command',

storyboard/db/projects_loader.py

@@ -0,0 +1,72 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import six
+import warnings
+import yaml
+
+from oslo.config import cfg
+from sqlalchemy.exc import SADeprecationWarning
+from storyboard.openstack.common.db.sqlalchemy import session as db_session
+
+from storyboard.db.models import Project
+from storyboard.db.models import ProjectGroup
+
+
+warnings.simplefilter("ignore", SADeprecationWarning)
+CONF = cfg.CONF
+
+
+def do_load_models(filename):
+
+    config_file = open(filename)
+    projects_list = yaml.load(config_file)
+
+    project_groups = dict()
+
+    for project in projects_list:
+        group_name = project.get("group") or "default"
+        if group_name not in project_groups:
+            project_groups[group_name] = list()
+
+        project_name = project.get("project")
+        project_description = project.get("description")
+
+        project_groups[group_name].append({"name": project_name,
+                                           "description": project_description})
+
+    session = db_session.get_session(sqlite_fk=True)
+
+    with session.begin():
+        for project_group_name, projects in six.iteritems(project_groups):
+            db_project_group = session.query(ProjectGroup)\
+                .filter_by(name=project_group_name).first()
+            if not db_project_group:
+                db_project_group = ProjectGroup()
+                db_project_group.name = project_group_name
+                db_project_group.projects = []
+
+            for project in projects:
+                db_project = session.query(Project)\
+                    .filter_by(name=project["name"]).first()
+                if not db_project:
+                    db_project = Project()
+                    db_project.name = project["name"]
+                    db_project.description = unicode(project["description"])
+                    session.add(db_project)
+
+                    db_project_group.projects.append(db_project)
+
+            session.add(db_project_group)

storyboard/tests/db/test_load_projects.py

@@ -0,0 +1,65 @@
+# Copyright (c) 2014 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+import mock
+from storyboard.openstack.common.db.sqlalchemy import session as db_session
+import testscenarios
+
+from storyboard.db.migration import cli
+from storyboard.db import models
+from storyboard.tests import base
+
+
+class TestLoadProjects(base.FunctionalTest):
+
+    scenarios = [
+        ('do_load_projects',
+         dict(argv=['prog', 'load_projects',
+                    'etc/projects.yaml.sample'],
+              func_name='load_projects'))
+    ]
+
+    def test_cli(self):
+        with mock.patch.object(sys, 'argv', self.argv):
+            cli.main()
+            session = db_session.get_session(sqlite_fk=True)
+            project_groups = session.query(models.ProjectGroup).all()
+            projects = session.query(models.Project).all()
+
+            self.assertIsNotNone(project_groups)
+            self.assertIsNotNone(projects)
+
+            project_names = ["Test-Project", "Test-Project-Two"]
+            project_ids = []
+            for project in projects:
+                self.assertIn(project.name, project_names)
+                project_ids.append(project.id)
+                project_names.remove(project.name)
+
+            # call again and nothing should change
+            cli.main()
+
+            session = db_session.get_session(sqlite_fk=True)
+            projects = session.query(models.Project).all()
+
+            self.assertIsNotNone(projects)
+            for project in projects:
+                self.assertIn(project.id, project_ids)
+
+
+def load_tests(loader, in_tests, pattern):
+    return testscenarios.load_tests_apply_scenarios(loader, in_tests, pattern)